local-plan-extractor/Makefile at main · digital-land/local-plan-extractor · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
# generated static site pages
DOCS_DIR=docs/

# downloaded data from other sources
CACHE_DIR=var/cache/

# generated CSV datasets
DATASET_DIR=dataset/

SOURCE_DATA=\
	$(CACHE_DIR)organisation.csv\
	$(CACHE_DIR)local-authority-district.csv\
	$(CACHE_DIR)local-planning-authority.csv\
	$(CACHE_DIR)local-planning-authority.geojson\
	$(CACHE_DIR)local-plan-document-type.csv\
	$(CACHE_DIR)local-planning-authority-lookup.csv

CORE_DOCUMENTS=$(wildcard document/*.pdf)

TARGETS=$(patsubst document/%,local-plan/%,$(patsubst %.pdf,%.json,$(CORE_DOCUMENTS)))

.PHONY: all init server test black scrape download download-all extract-dates generate-config clobber clean

all::	$(SOURCE_DATA) $(TARGETS)
	python3 bin/generate-csvs.py --output-dir $(DATASET_DIR)
	python3 bin/render.py

source/%.json:
	@mkdir -p $(dir $@)
	python3 bin/find-local-plan.py $(basename $(notdir $@)) > $@

local-plan/%.json: document/%.pdf
	@mkdir -p $(dir $@)
	python3 bin/local-plan-extractor.py $? > $@

$(CACHE_DIR)prototype.csv:
	@mkdir -p $(CACHE_DIR)
	curl -qfsL 'https://local-plans.prototype.planning.data.gov.uk/local-plans/local-plan-data.csv' > $@

$(CACHE_DIR)organisation.csv:
	@mkdir -p $(CACHE_DIR)
	curl -qfsL "https://files.planning.data.gov.uk/organisation-collection/dataset/organisation.csv" > $@

$(CACHE_DIR)%.csv:
	@mkdir -p $(dir $@)
	curl -qfsL 'https://files.planning.data.gov.uk/dataset/$(notdir $@)' > $@

$(CACHE_DIR)%.geojson:
	@mkdir -p $(dir $@)
	curl -qfsL 'https://files.planning.data.gov.uk/dataset/$(notdir $@)' > $@


init::
	pip3 install -r requirements.txt

# Run the test suite
test::
	python3 -m unittest discover -s tests -v

# Run find-local-plan.py across all authorities (hits Claude API + web — slow)
scrape::
	python3 bin/run-all-authorities.py

# Download the main adopted local plan PDF per authority (default mode)
download::
	python3 bin/download-documents.py

# Download all document URLs across all plans
download-all::
	python3 bin/download-documents.py --all

# Extract adoption dates from local plans using Claude (hits Claude API — slow)
extract-dates::
	python3 bin/extract-adoption-dates.py

# Generate config files for local plans
generate-config::
	python3 bin/generate-local-plan-config-files.py

server::
	python3 -m http.server -d $(DOCS_DIR)

black::
	black bin

clobber::
	rm -f $(TARGETS)
	rm -f $(DATASET_DIR)local-plan.csv $(DATASET_DIR)local-plan-document.csv

clean::
	rm -rf var/