From 99336f5717b5697f66331dfa30d26492394c9561 Mon Sep 17 00:00:00 2001 From: Toby Petty Date: Mon, 10 Jan 2022 12:18:22 +0000 Subject: [PATCH 1/8] Initial attempt at abstracting out digital-land command to run in docker for arbitrary makefile commands --- collection.mk | 52 ++++++++++++++++++++++++++++++++++++++++-- pipeline.mk | 62 +++++++++++++++++++++++++++++++++++++++++++-------- render.mk | 46 +++++++++++++++++++++++++++++++++++--- 3 files changed, 146 insertions(+), 14 deletions(-) diff --git a/collection.mk b/collection.mk index e4cf268..0657fb8 100644 --- a/collection.mk +++ b/collection.mk @@ -16,6 +16,45 @@ ifeq ($(DATASTORE_URL),) DATASTORE_URL=https://collection-dataset.s3.eu-west-2.amazonaws.com/ endif +ifeq ($(DOCKERISED),1) +EXTRA_MOUNTS := +# Run in development mode by default for now +ifneq ($(DEVELOPMENT),0) +EXTRA_MOUNTS += -v $(PWD)/local_collection:/data --workdir /data + +ifdef ($(LOCAL_SPECIFICATION_PATH),) + EXTRA_MOUNTS += -v $(LOCAL_SPECIFICATION_PATH)/specification:/collection/specification +else ifeq ($(LOCAL_SPECIFICATION),1) + EXTRA_MOUNTS += -v $(PWD)/../specification/specificaiton:/collection/specification +endif +ifdef ($(LOCAL_DL_PYTHON_PATH),) + EXTRA_MOUNTS += -v $(LOCAL_DL_PYTHON_PATH):/Src +else ifeq ($(LOCAL_DL_PYTHON),1) + EXTRA_MOUNTS += -v $(PWD)/../digital-land-python:/src +endif +endif + +DOCKER_TAG=latest +ECR_URL=public.ecr.aws/l6z6v3j6/ + +digital-land = docker run -t \ + -u $(shell id -u) \ + -v $(PWD):/pipeline \ + $(EXTRA_MOUNTS) \ + $(ECR_URL)digital-land-python:$(DOCKER_TAG) \ + digital-land \ + --specification-dir /collection/specification + +docker-pull:: +ifndef ($(DISABLE_DOCKER_PULL),) + docker pull $(ECR_URL)digital-land-python:$(DOCKER_TAG) +endif + +init:: docker-pull +else +digital-land = digital-land +endif + # data sources SOURCE_CSV=$(COLLECTION_DIR)source.csv @@ -35,10 +74,10 @@ first-pass:: collect second-pass:: collection collect:: $(SOURCE_CSV) $(ENDPOINT_CSV) - digital-land collect $(ENDPOINT_CSV) + $(digital-land) collect $(ENDPOINT_CSV) collection:: - digital-land collection-save-csv + $(digital-land) collection-save-csv clobber-today:: rm -rf $(LOG_FILES_TODAY) $(COLLECTION_INDEX) @@ -46,12 +85,21 @@ clobber-today:: makerules:: curl -qfsL '$(SOURCE_URL)/makerules/main/collection.mk' > makerules/collection.mk +# These will run as usual unless we're in a dockerised environment and DEVELOPMENT isn't explicitly set to 1 +ifneq ($(DOCKERISED),1) +ifneq ($(DEVELOPMENT),1) +commit-dataset:: + mkdir -p $(DATASET_DIRS) + git add $(DATASET_DIRS) + git diff --quiet && git diff --staged --quiet || (git commit -m "Data $(shell date +%F)"; git push origin $(BRANCH)) + commit-collection:: git add collection git diff --quiet && git diff --staged --quiet || (git commit -m "Collection $(shell date +%F)"; git push origin $(BRANCH)) save-resources:: aws s3 sync s3://collection-dataset/$(REPOSITORY)/$(RESOURCE_DIR) $(RESOURCE_DIR) +endif load-resources:: aws s3 sync $(RESOURCE_DIR) s3://collection-dataset/$(REPOSITORY)/$(RESOURCE_DIR) diff --git a/pipeline.mk b/pipeline.mk index 801ba59..a6dcf39 100644 --- a/pipeline.mk +++ b/pipeline.mk @@ -3,6 +3,45 @@ dataset\ commit-dataset +ifeq ($(DOCKERISED),1) +EXTRA_MOUNTS := +# Run in development mode by default for now +ifneq ($(DEVELOPMENT),0) +EXTRA_MOUNTS += -v $(PWD)/local_collection:/data --workdir /data + +ifdef ($(LOCAL_SPECIFICATION_PATH),) + EXTRA_MOUNTS += -v $(LOCAL_SPECIFICATION_PATH)/specification:/collection/specification +else ifeq ($(LOCAL_SPECIFICATION),1) + EXTRA_MOUNTS += -v $(PWD)/../specification/specificaiton:/collection/specification +endif +ifdef ($(LOCAL_DL_PYTHON_PATH),) + EXTRA_MOUNTS += -v $(LOCAL_DL_PYTHON_PATH):/Src +else ifeq ($(LOCAL_DL_PYTHON),1) + EXTRA_MOUNTS += -v $(PWD)/../digital-land-python:/src +endif +endif + +DOCKER_TAG=latest +ECR_URL=public.ecr.aws/l6z6v3j6/ + +digital-land = docker run -t \ + -u $(shell id -u) \ + -v $(PWD):/pipeline \ + $(EXTRA_MOUNTS) \ + $(ECR_URL)digital-land-python:$(DOCKER_TAG) \ + digital-land \ + --specification-dir /collection/specification + +docker-pull:: +ifndef ($(DISABLE_DOCKER_PULL),) + docker pull $(ECR_URL)digital-land-python:$(DOCKER_TAG) +endif + +init:: docker-pull +else +digital-land = digital-land +endif + # data sources # collected resources ifeq ($(COLLECTION_DIR),) @@ -53,12 +92,12 @@ endif define run-pipeline = mkdir -p $(@D) $(ISSUE_DIR)$(notdir $(@D)) - digital-land --pipeline-name $(notdir $(@D)) $(DIGITAL_LAND_FLAGS) pipeline --issue-dir $(ISSUE_DIR)$(notdir $(@D)) $(PIPELINE_FLAGS) $< $@ + $(digital-land) --pipeline-name $(notdir $(@D)) $(DIGITAL_LAND_FLAGS) pipeline --issue-dir $(ISSUE_DIR)$(notdir $(@D)) $(PIPELINE_FLAGS) $< $@ endef define build-dataset = mkdir -p $(@D) - time digital-land --pipeline-name $(notdir $(basename $@)) load-entries --output-path $(basename $@).sqlite3 $(^) + time $(digital-land) --pipeline-name $(notdir $(basename $@)) load-entries --output-path $(basename $@).sqlite3 $(^) time digital-land --pipeline-name $(notdir $(basename $@)) build-dataset $(basename $@).sqlite3 $@ endef @@ -67,7 +106,7 @@ collection:: collection/pipeline.mk -include collection/pipeline.mk collection/pipeline.mk: collection/resource.csv collection/source.csv - digital-land collection-pipeline-makerules > collection/pipeline.mk + $(digital-land) collection-pipeline-makerules > collection/pipeline.mk # restart the make process to pick-up collected resource files second-pass:: @@ -99,11 +138,6 @@ init:: makerules:: curl -qfsL '$(SOURCE_URL)/makerules/main/pipeline.mk' > makerules/pipeline.mk -commit-dataset:: - mkdir -p $(DATASET_DIRS) - git add $(DATASET_DIRS) - git diff --quiet && git diff --staged --quiet || (git commit -m "Data $(shell date +%F)"; git push origin $(BRANCH)) - fetch-s3:: aws s3 sync s3://collection-dataset/$(REPOSITORY)/$(RESOURCE_DIR) $(RESOURCE_DIR) --no-progress @@ -111,7 +145,15 @@ fetch-transformed-s3:: aws s3 sync s3://collection-dataset/$(REPOSITORY)/$(ISSUE_DIR) $(ISSUE_DIR) --no-progress aws s3 sync s3://collection-dataset/$(REPOSITORY)/$(TRANSFORMED_DIR) $(TRANSFORMED_DIR) --no-progress aws s3 sync s3://collection-dataset/$(REPOSITORY)/$(DATASET_DIR) $(DATASET_DIR) --no-progress - + +# These will run as usual unless we're in a dockerised environment and DEVELOPMENT isn't explicitly set to 1 +ifneq ($(DOCKERISED),1) +ifneq ($(DEVELOPMENT),1) +commit-dataset:: + mkdir -p $(DATASET_DIRS) + git add $(DATASET_DIRS) + git diff --quiet && git diff --staged --quiet || (git commit -m "Data $(shell date +%F)"; git push origin $(BRANCH)) + push-collection-s3:: aws s3 sync $(RESOURCE_DIR) s3://collection-dataset/$(REPOSITORY)/$(RESOURCE_DIR) --no-progress aws s3 cp $(COLLECTION_DIR)/log.csv s3://collection-dataset/$(REPOSITORY)/$(COLLECTION_DIR) --no-progress @@ -126,6 +168,8 @@ push-dataset-s3:: aws s3 sync $(ISSUE_DIR) s3://collection-dataset/$(REPOSITORY)/$(ISSUE_DIR) --no-progress @mkdir -p $(DATASET_DIR) aws s3 sync $(DATASET_DIR) s3://collection-dataset/$(REPOSITORY)/$(DATASET_DIR) --no-progress +endif +endif pipeline-run:: aws batch submit-job --job-name $(REPOSITORY)-$(shell date '+%Y-%m-%d-%H-%M-%S') --job-queue dl-batch-queue --job-definition dl-batch-def --container-overrides '{"environment": [{"name":"BATCH_FILE_URL","value":"https://raw.githubusercontent.com/digital-land/docker-builds/main/pipeline_run.sh"}, {"name" : "REPOSITORY","value" : "$(REPOSITORY)"}]}' diff --git a/render.mk b/render.mk index 7dc304c..7bfa92e 100644 --- a/render.mk +++ b/render.mk @@ -33,6 +33,46 @@ ifeq ($(VIEW_MODEL),) VIEW_MODEL=$(DATASET_DIR)view_model.sqlite3 endif +ifeq ($(DOCKERISED),1) +EXTRA_MOUNTS := +# Run in development mode by default for now +ifneq ($(DEVELOPMENT),0) +EXTRA_MOUNTS += -v $(PWD)/local_collection:/data --workdir /data + +ifdef ($(LOCAL_SPECIFICATION_PATH),) + EXTRA_MOUNTS += -v $(LOCAL_SPECIFICATION_PATH)/specification:/collection/specification +else ifeq ($(LOCAL_SPECIFICATION),1) + EXTRA_MOUNTS += -v $(PWD)/../specification/specificaiton:/collection/specification +endif +ifdef ($(LOCAL_DL_PYTHON_PATH),) + EXTRA_MOUNTS += -v $(LOCAL_DL_PYTHON_PATH):/Src +else ifeq ($(LOCAL_DL_PYTHON),1) + EXTRA_MOUNTS += -v $(PWD)/../digital-land-python:/src +endif +endif + +DOCKER_TAG=latest +ECR_URL=public.ecr.aws/l6z6v3j6/ + +digital-land = docker run -t \ + -u $(shell id -u) \ + -v $(PWD):/pipeline \ + $(EXTRA_MOUNTS) \ + $(ECR_URL)digital-land-python:$(DOCKER_TAG) \ + digital-land \ + --specification-dir /collection/specification + +docker-pull:: +ifndef ($(DISABLE_DOCKER_PULL),) + docker pull $(ECR_URL)digital-land-python:$(DOCKER_TAG) +endif + +init:: docker-pull +else +digital-land = digital-land +endif + + TEMPLATE_FILES=$(wildcard templates/*) @@ -53,7 +93,7 @@ render:: $(TEMPLATE_FILES) $(SPECIFICATION_FILES) $(DATASET_FILES) $(DATASET_PAT ifneq ($(RENDER_COMMAND),) $(RENDER_COMMAND) else - digital-land --pipeline-name $(DATASET) render --dataset-path $(DATASET_PATH) $(RENDER_FLAGS) + $(digital-land) --pipeline-name $(DATASET) render --dataset-path $(DATASET_PATH) $(RENDER_FLAGS) endif @touch ./docs/.nojekyll @@ -62,10 +102,10 @@ server: cd docs && python3 -m http.server clobber clean:: clobber-dataset clobber-docs - + clobber-dataset:: rm -rf $(DATASET_PATH) - + clobber-docs:: rm -rf $(DOCS_DIR) From d9d9bd3c9cc92903b61a025e322256a66354de67 Mon Sep 17 00:00:00 2001 From: Toby Petty Date: Mon, 10 Jan 2022 14:24:45 +0000 Subject: [PATCH 2/8] Apply refactoring and indentation --- collection.mk | 43 +++++++++++++++++++++++++------------------ pipeline.mk | 42 ++++++++++++++++++++++++------------------ render.mk | 38 +++++++++++++++++++++++--------------- 3 files changed, 72 insertions(+), 51 deletions(-) diff --git a/collection.mk b/collection.mk index 0657fb8..e5c2232 100644 --- a/collection.mk +++ b/collection.mk @@ -16,22 +16,30 @@ ifeq ($(DATASTORE_URL),) DATASTORE_URL=https://collection-dataset.s3.eu-west-2.amazonaws.com/ endif -ifeq ($(DOCKERISED),1) -EXTRA_MOUNTS := -# Run in development mode by default for now -ifneq ($(DEVELOPMENT),0) -EXTRA_MOUNTS += -v $(PWD)/local_collection:/data --workdir /data - -ifdef ($(LOCAL_SPECIFICATION_PATH),) - EXTRA_MOUNTS += -v $(LOCAL_SPECIFICATION_PATH)/specification:/collection/specification -else ifeq ($(LOCAL_SPECIFICATION),1) - EXTRA_MOUNTS += -v $(PWD)/../specification/specificaiton:/collection/specification -endif -ifdef ($(LOCAL_DL_PYTHON_PATH),) - EXTRA_MOUNTS += -v $(LOCAL_DL_PYTHON_PATH):/Src -else ifeq ($(LOCAL_DL_PYTHON),1) - EXTRA_MOUNTS += -v $(PWD)/../digital-land-python:/src +ifndef ($(DOCKERISED),) + DOCKERISED = 0 + DEVELOPMENT = 0 +else + # Run in development mode by default for now + ifndef ($(DEVELOPMENT),) + DEVELOPMENT = 1 + endif endif + +EXTRA_MOUNTS := +ifeq ($(and $(DOCKERISED),$(DEVELOPMENT))) + EXTRA_MOUNTS += -v $(PWD)/local_collection:/data --workdir /data + + ifdef ($(LOCAL_SPECIFICATION_PATH),) + EXTRA_MOUNTS += -v $(LOCAL_SPECIFICATION_PATH)/specification:/collection/specification + else ifeq ($(LOCAL_SPECIFICATION),1) + EXTRA_MOUNTS += -v $(PWD)/../specification/specificaiton:/collection/specification + endif + ifdef ($(LOCAL_DL_PYTHON_PATH),) + EXTRA_MOUNTS += -v $(LOCAL_DL_PYTHON_PATH):/Src + else ifeq ($(LOCAL_DL_PYTHON),1) + EXTRA_MOUNTS += -v $(PWD)/../digital-land-python:/src + endif endif DOCKER_TAG=latest @@ -85,9 +93,8 @@ clobber-today:: makerules:: curl -qfsL '$(SOURCE_URL)/makerules/main/collection.mk' > makerules/collection.mk -# These will run as usual unless we're in a dockerised environment and DEVELOPMENT isn't explicitly set to 1 -ifneq ($(DOCKERISED),1) -ifneq ($(DEVELOPMENT),1) +# These will run as usual unless we're in a dockerised environment and DEVELOPMENT isn't explicitly set to 1 i.e if DEVELOPMENT != 1 or DOCKERISED != 1 +ifneq ($(and $(DOCKERISED),$(DEVELOPMENT))) commit-dataset:: mkdir -p $(DATASET_DIRS) git add $(DATASET_DIRS) diff --git a/pipeline.mk b/pipeline.mk index a6dcf39..304bb75 100644 --- a/pipeline.mk +++ b/pipeline.mk @@ -3,22 +3,30 @@ dataset\ commit-dataset -ifeq ($(DOCKERISED),1) +ifndef ($(DOCKERISED),) + DOCKERISED = 0 + DEVELOPMENT = 0 +else + # Run in development mode by default for now + ifndef ($(DEVELOPMENT),) + DEVELOPMENT = 1 + endif +endif + EXTRA_MOUNTS := -# Run in development mode by default for now -ifneq ($(DEVELOPMENT),0) -EXTRA_MOUNTS += -v $(PWD)/local_collection:/data --workdir /data +ifeq ($(and $(DOCKERISED),$(DEVELOPMENT))) + EXTRA_MOUNTS += -v $(PWD)/local_collection:/data --workdir /data -ifdef ($(LOCAL_SPECIFICATION_PATH),) - EXTRA_MOUNTS += -v $(LOCAL_SPECIFICATION_PATH)/specification:/collection/specification -else ifeq ($(LOCAL_SPECIFICATION),1) - EXTRA_MOUNTS += -v $(PWD)/../specification/specificaiton:/collection/specification -endif -ifdef ($(LOCAL_DL_PYTHON_PATH),) - EXTRA_MOUNTS += -v $(LOCAL_DL_PYTHON_PATH):/Src -else ifeq ($(LOCAL_DL_PYTHON),1) - EXTRA_MOUNTS += -v $(PWD)/../digital-land-python:/src -endif + ifdef ($(LOCAL_SPECIFICATION_PATH),) + EXTRA_MOUNTS += -v $(LOCAL_SPECIFICATION_PATH)/specification:/collection/specification + else ifeq ($(LOCAL_SPECIFICATION),1) + EXTRA_MOUNTS += -v $(PWD)/../specification/specificaiton:/collection/specification + endif + ifdef ($(LOCAL_DL_PYTHON_PATH),) + EXTRA_MOUNTS += -v $(LOCAL_DL_PYTHON_PATH):/Src + else ifeq ($(LOCAL_DL_PYTHON),1) + EXTRA_MOUNTS += -v $(PWD)/../digital-land-python:/src + endif endif DOCKER_TAG=latest @@ -146,9 +154,8 @@ fetch-transformed-s3:: aws s3 sync s3://collection-dataset/$(REPOSITORY)/$(TRANSFORMED_DIR) $(TRANSFORMED_DIR) --no-progress aws s3 sync s3://collection-dataset/$(REPOSITORY)/$(DATASET_DIR) $(DATASET_DIR) --no-progress -# These will run as usual unless we're in a dockerised environment and DEVELOPMENT isn't explicitly set to 1 -ifneq ($(DOCKERISED),1) -ifneq ($(DEVELOPMENT),1) +# These will run as usual unless we're in a dockerised environment and DEVELOPMENT isn't explicitly set to 1 i.e if DEVELOPMENT != 1 or DOCKERISED != 1 +ifneq ($and($(DOCKERISED), $(DEVELOPMENT))) commit-dataset:: mkdir -p $(DATASET_DIRS) git add $(DATASET_DIRS) @@ -169,7 +176,6 @@ push-dataset-s3:: @mkdir -p $(DATASET_DIR) aws s3 sync $(DATASET_DIR) s3://collection-dataset/$(REPOSITORY)/$(DATASET_DIR) --no-progress endif -endif pipeline-run:: aws batch submit-job --job-name $(REPOSITORY)-$(shell date '+%Y-%m-%d-%H-%M-%S') --job-queue dl-batch-queue --job-definition dl-batch-def --container-overrides '{"environment": [{"name":"BATCH_FILE_URL","value":"https://raw.githubusercontent.com/digital-land/docker-builds/main/pipeline_run.sh"}, {"name" : "REPOSITORY","value" : "$(REPOSITORY)"}]}' diff --git a/render.mk b/render.mk index 7bfa92e..b3ef858 100644 --- a/render.mk +++ b/render.mk @@ -33,22 +33,30 @@ ifeq ($(VIEW_MODEL),) VIEW_MODEL=$(DATASET_DIR)view_model.sqlite3 endif -ifeq ($(DOCKERISED),1) -EXTRA_MOUNTS := -# Run in development mode by default for now -ifneq ($(DEVELOPMENT),0) -EXTRA_MOUNTS += -v $(PWD)/local_collection:/data --workdir /data - -ifdef ($(LOCAL_SPECIFICATION_PATH),) - EXTRA_MOUNTS += -v $(LOCAL_SPECIFICATION_PATH)/specification:/collection/specification -else ifeq ($(LOCAL_SPECIFICATION),1) - EXTRA_MOUNTS += -v $(PWD)/../specification/specificaiton:/collection/specification -endif -ifdef ($(LOCAL_DL_PYTHON_PATH),) - EXTRA_MOUNTS += -v $(LOCAL_DL_PYTHON_PATH):/Src -else ifeq ($(LOCAL_DL_PYTHON),1) - EXTRA_MOUNTS += -v $(PWD)/../digital-land-python:/src +ifndef ($(DOCKERISED),) + DOCKERISED = 0 + DEVELOPMENT = 0 +else + # Run in development mode by default for now + ifndef ($(DEVELOPMENT),) + DEVELOPMENT = 1 + endif endif + +EXTRA_MOUNTS := +ifeq ($(and $(DOCKERISED),$(DEVELOPMENT))) + EXTRA_MOUNTS += -v $(PWD)/local_collection:/data --workdir /data + + ifdef ($(LOCAL_SPECIFICATION_PATH),) + EXTRA_MOUNTS += -v $(LOCAL_SPECIFICATION_PATH)/specification:/collection/specification + else ifeq ($(LOCAL_SPECIFICATION),1) + EXTRA_MOUNTS += -v $(PWD)/../specification/specificaiton:/collection/specification + endif + ifdef ($(LOCAL_DL_PYTHON_PATH),) + EXTRA_MOUNTS += -v $(LOCAL_DL_PYTHON_PATH):/Src + else ifeq ($(LOCAL_DL_PYTHON),1) + EXTRA_MOUNTS += -v $(PWD)/../digital-land-python:/src + endif endif DOCKER_TAG=latest From 6257d0b19afefef7e86881311877fb133e54c834 Mon Sep 17 00:00:00 2001 From: Toby Petty Date: Tue, 11 Jan 2022 17:44:09 +0000 Subject: [PATCH 3/8] Refactor dockerisation into docker.mk --- collection.mk | 51 +++----------------------------------------- docker.mk | 58 +++++++++++++++++++++++++++++++++++++++++++++++++++ makerules.mk | 3 ++- pipeline.mk | 49 ++----------------------------------------- render.mk | 49 +------------------------------------------ 5 files changed, 66 insertions(+), 144 deletions(-) create mode 100644 docker.mk diff --git a/collection.mk b/collection.mk index e5c2232..4909603 100644 --- a/collection.mk +++ b/collection.mk @@ -4,6 +4,8 @@ commit-collection\ clobber-today +include makerules/docker.mk + ifeq ($(COLLECTION_DIR),) COLLECTION_DIR=collection/ endif @@ -16,53 +18,6 @@ ifeq ($(DATASTORE_URL),) DATASTORE_URL=https://collection-dataset.s3.eu-west-2.amazonaws.com/ endif -ifndef ($(DOCKERISED),) - DOCKERISED = 0 - DEVELOPMENT = 0 -else - # Run in development mode by default for now - ifndef ($(DEVELOPMENT),) - DEVELOPMENT = 1 - endif -endif - -EXTRA_MOUNTS := -ifeq ($(and $(DOCKERISED),$(DEVELOPMENT))) - EXTRA_MOUNTS += -v $(PWD)/local_collection:/data --workdir /data - - ifdef ($(LOCAL_SPECIFICATION_PATH),) - EXTRA_MOUNTS += -v $(LOCAL_SPECIFICATION_PATH)/specification:/collection/specification - else ifeq ($(LOCAL_SPECIFICATION),1) - EXTRA_MOUNTS += -v $(PWD)/../specification/specificaiton:/collection/specification - endif - ifdef ($(LOCAL_DL_PYTHON_PATH),) - EXTRA_MOUNTS += -v $(LOCAL_DL_PYTHON_PATH):/Src - else ifeq ($(LOCAL_DL_PYTHON),1) - EXTRA_MOUNTS += -v $(PWD)/../digital-land-python:/src - endif -endif - -DOCKER_TAG=latest -ECR_URL=public.ecr.aws/l6z6v3j6/ - -digital-land = docker run -t \ - -u $(shell id -u) \ - -v $(PWD):/pipeline \ - $(EXTRA_MOUNTS) \ - $(ECR_URL)digital-land-python:$(DOCKER_TAG) \ - digital-land \ - --specification-dir /collection/specification - -docker-pull:: -ifndef ($(DISABLE_DOCKER_PULL),) - docker pull $(ECR_URL)digital-land-python:$(DOCKER_TAG) -endif - -init:: docker-pull -else -digital-land = digital-land -endif - # data sources SOURCE_CSV=$(COLLECTION_DIR)source.csv @@ -94,7 +49,7 @@ makerules:: curl -qfsL '$(SOURCE_URL)/makerules/main/collection.mk' > makerules/collection.mk # These will run as usual unless we're in a dockerised environment and DEVELOPMENT isn't explicitly set to 1 i.e if DEVELOPMENT != 1 or DOCKERISED != 1 -ifneq ($(and $(DOCKERISED),$(DEVELOPMENT))) +ifeq ($(DEVELOPMENT),0) commit-dataset:: mkdir -p $(DATASET_DIRS) git add $(DATASET_DIRS) diff --git a/docker.mk b/docker.mk new file mode 100644 index 0000000..4d2ca8a --- /dev/null +++ b/docker.mk @@ -0,0 +1,58 @@ +ifneq ($(DOCKERISED),1) +DOCKERISED = 0 +DEVELOPMENT = 0 +else +# Run in development mode by default for now +ifeq ($(DEVELOPMENT),0) +DEVELOPMENT = 0 +else +DEVELOPMENT = 1 +endif +endif +$(info DOCKERISED is $(DOCKERISED)) +$(info DEVELOPMENT is $(DEVELOPMENT)) + +EXTRA_MOUNTS := +# ifeq ($(and $(DOCKERISED),$(DEVELOPMENT))) +ifeq ($(DOCKERISED),1) +ifeq ($(DEVELOPMENT),1) +EXTRA_MOUNTS += -v $(PWD)/local_collection:/data --workdir /data + +ifdef ($(LOCAL_SPECIFICATION_PATH),) +EXTRA_MOUNTS += -v $(LOCAL_SPECIFICATION_PATH)/specification:/collection/specification +else ifeq ($(LOCAL_SPECIFICATION),1) +EXTRA_MOUNTS += -v $(PWD)/../specification/specificaiton:/collection/specification +endif + +ifdef ($(LOCAL_DL_PYTHON_PATH),) +EXTRA_MOUNTS += -v $(LOCAL_DL_PYTHON_PATH):/Src +else ifeq ($(LOCAL_DL_PYTHON),1) +EXTRA_MOUNTS += -v $(PWD)/../digital-land-python:/src +endif + +endif +$(info EXTRA_MOUNTS is $(EXTRA_MOUNTS)) + +DOCKER_TAG=latest +ECR_URL=public.ecr.aws/l6z6v3j6/ + +COLLECTION_DIR=/pipeline/collection/ + +digital-land = docker run -t \ + -e LOCAL_USER_ID=$(shell id -u) \ + -v $(PWD):/pipeline \ + $(EXTRA_MOUNTS) \ + $(ECR_URL)digital-land-python:$(DOCKER_TAG) \ + digital-land \ + --specification-dir /collection/specification + +docker-pull:: +ifndef ($(DISABLE_DOCKER_PULL),) + docker pull $(ECR_URL)digital-land-python:$(DOCKER_TAG) +endif + +init:: docker-pull +else +digital-land = digital-land +endif + diff --git a/makerules.mk b/makerules.mk index 8b0a7cb..15c8861 100644 --- a/makerules.mk +++ b/makerules.mk @@ -1,5 +1,6 @@ SOURCE_URL=https://raw.githubusercontent.com/digital-land/ +include makerules/docker.mk # deduce the repository ifeq ($(REPOSITORY),) REPOSITORY=$(shell basename -s .git `git config --get remote.origin.url`) @@ -58,7 +59,7 @@ second-pass:: @: # initialise -ifeq (,$(wildcard /.dockerenv )) +ifeq ($(DOCKERISED),0) init:: pip install --upgrade pip ifneq (,$(wildcard requirements.txt)) diff --git a/pipeline.mk b/pipeline.mk index 304bb75..5bc359e 100644 --- a/pipeline.mk +++ b/pipeline.mk @@ -3,52 +3,7 @@ dataset\ commit-dataset -ifndef ($(DOCKERISED),) - DOCKERISED = 0 - DEVELOPMENT = 0 -else - # Run in development mode by default for now - ifndef ($(DEVELOPMENT),) - DEVELOPMENT = 1 - endif -endif - -EXTRA_MOUNTS := -ifeq ($(and $(DOCKERISED),$(DEVELOPMENT))) - EXTRA_MOUNTS += -v $(PWD)/local_collection:/data --workdir /data - - ifdef ($(LOCAL_SPECIFICATION_PATH),) - EXTRA_MOUNTS += -v $(LOCAL_SPECIFICATION_PATH)/specification:/collection/specification - else ifeq ($(LOCAL_SPECIFICATION),1) - EXTRA_MOUNTS += -v $(PWD)/../specification/specificaiton:/collection/specification - endif - ifdef ($(LOCAL_DL_PYTHON_PATH),) - EXTRA_MOUNTS += -v $(LOCAL_DL_PYTHON_PATH):/Src - else ifeq ($(LOCAL_DL_PYTHON),1) - EXTRA_MOUNTS += -v $(PWD)/../digital-land-python:/src - endif -endif - -DOCKER_TAG=latest -ECR_URL=public.ecr.aws/l6z6v3j6/ - -digital-land = docker run -t \ - -u $(shell id -u) \ - -v $(PWD):/pipeline \ - $(EXTRA_MOUNTS) \ - $(ECR_URL)digital-land-python:$(DOCKER_TAG) \ - digital-land \ - --specification-dir /collection/specification - -docker-pull:: -ifndef ($(DISABLE_DOCKER_PULL),) - docker pull $(ECR_URL)digital-land-python:$(DOCKER_TAG) -endif - -init:: docker-pull -else -digital-land = digital-land -endif +include makerules/docker.mk # data sources # collected resources @@ -155,7 +110,7 @@ fetch-transformed-s3:: aws s3 sync s3://collection-dataset/$(REPOSITORY)/$(DATASET_DIR) $(DATASET_DIR) --no-progress # These will run as usual unless we're in a dockerised environment and DEVELOPMENT isn't explicitly set to 1 i.e if DEVELOPMENT != 1 or DOCKERISED != 1 -ifneq ($and($(DOCKERISED), $(DEVELOPMENT))) +ifeq ($(DEVELOPMENT),0) commit-dataset:: mkdir -p $(DATASET_DIRS) git add $(DATASET_DIRS) diff --git a/render.mk b/render.mk index b3ef858..fc68053 100644 --- a/render.mk +++ b/render.mk @@ -33,54 +33,7 @@ ifeq ($(VIEW_MODEL),) VIEW_MODEL=$(DATASET_DIR)view_model.sqlite3 endif -ifndef ($(DOCKERISED),) - DOCKERISED = 0 - DEVELOPMENT = 0 -else - # Run in development mode by default for now - ifndef ($(DEVELOPMENT),) - DEVELOPMENT = 1 - endif -endif - -EXTRA_MOUNTS := -ifeq ($(and $(DOCKERISED),$(DEVELOPMENT))) - EXTRA_MOUNTS += -v $(PWD)/local_collection:/data --workdir /data - - ifdef ($(LOCAL_SPECIFICATION_PATH),) - EXTRA_MOUNTS += -v $(LOCAL_SPECIFICATION_PATH)/specification:/collection/specification - else ifeq ($(LOCAL_SPECIFICATION),1) - EXTRA_MOUNTS += -v $(PWD)/../specification/specificaiton:/collection/specification - endif - ifdef ($(LOCAL_DL_PYTHON_PATH),) - EXTRA_MOUNTS += -v $(LOCAL_DL_PYTHON_PATH):/Src - else ifeq ($(LOCAL_DL_PYTHON),1) - EXTRA_MOUNTS += -v $(PWD)/../digital-land-python:/src - endif -endif - -DOCKER_TAG=latest -ECR_URL=public.ecr.aws/l6z6v3j6/ - -digital-land = docker run -t \ - -u $(shell id -u) \ - -v $(PWD):/pipeline \ - $(EXTRA_MOUNTS) \ - $(ECR_URL)digital-land-python:$(DOCKER_TAG) \ - digital-land \ - --specification-dir /collection/specification - -docker-pull:: -ifndef ($(DISABLE_DOCKER_PULL),) - docker pull $(ECR_URL)digital-land-python:$(DOCKER_TAG) -endif - -init:: docker-pull -else -digital-land = digital-land -endif - - +include makerules/docker.mk TEMPLATE_FILES=$(wildcard templates/*) From 079b15bc30ede40869b1ee35e7cf3442992ebe95 Mon Sep 17 00:00:00 2001 From: Toby Petty Date: Wed, 12 Jan 2022 10:40:18 +0000 Subject: [PATCH 4/8] Fixed issue with missing /pipeline/collection/*.csv targets --- docker.mk | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docker.mk b/docker.mk index 4d2ca8a..da09bac 100644 --- a/docker.mk +++ b/docker.mk @@ -38,6 +38,12 @@ ECR_URL=public.ecr.aws/l6z6v3j6/ COLLECTION_DIR=/pipeline/collection/ +/pipeline/collection/resource.csv: + +/pipeline/collection/source.csv: + +/pipeline/collection/endpoint.csv: + digital-land = docker run -t \ -e LOCAL_USER_ID=$(shell id -u) \ -v $(PWD):/pipeline \ From 89cfbf2d2c07a7320998fa65ea4832e9113429b4 Mon Sep 17 00:00:00 2001 From: Toby Petty Date: Wed, 12 Jan 2022 10:47:07 +0000 Subject: [PATCH 5/8] Updated comments --- collection.mk | 2 +- pipeline.mk | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/collection.mk b/collection.mk index 4909603..d42c0a4 100644 --- a/collection.mk +++ b/collection.mk @@ -48,7 +48,7 @@ clobber-today:: makerules:: curl -qfsL '$(SOURCE_URL)/makerules/main/collection.mk' > makerules/collection.mk -# These will run as usual unless we're in a dockerised environment and DEVELOPMENT isn't explicitly set to 1 i.e if DEVELOPMENT != 1 or DOCKERISED != 1 +# These will run as usual if DEVELOPMENT isn't explicitly set to 1 ifeq ($(DEVELOPMENT),0) commit-dataset:: mkdir -p $(DATASET_DIRS) diff --git a/pipeline.mk b/pipeline.mk index 5bc359e..8bd05df 100644 --- a/pipeline.mk +++ b/pipeline.mk @@ -109,7 +109,7 @@ fetch-transformed-s3:: aws s3 sync s3://collection-dataset/$(REPOSITORY)/$(TRANSFORMED_DIR) $(TRANSFORMED_DIR) --no-progress aws s3 sync s3://collection-dataset/$(REPOSITORY)/$(DATASET_DIR) $(DATASET_DIR) --no-progress -# These will run as usual unless we're in a dockerised environment and DEVELOPMENT isn't explicitly set to 1 i.e if DEVELOPMENT != 1 or DOCKERISED != 1 +# These will run as usual if DEVELOPMENT isn't explicitly set to 1 ifeq ($(DEVELOPMENT),0) commit-dataset:: mkdir -p $(DATASET_DIRS) From 0554a42ffb35f5c2cf36462848a8a791d245f0a9 Mon Sep 17 00:00:00 2001 From: Toby Petty Date: Wed, 12 Jan 2022 11:20:06 +0000 Subject: [PATCH 6/8] Fix typo --- docker.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker.mk b/docker.mk index da09bac..e2f3d84 100644 --- a/docker.mk +++ b/docker.mk @@ -21,7 +21,7 @@ EXTRA_MOUNTS += -v $(PWD)/local_collection:/data --workdir /data ifdef ($(LOCAL_SPECIFICATION_PATH),) EXTRA_MOUNTS += -v $(LOCAL_SPECIFICATION_PATH)/specification:/collection/specification else ifeq ($(LOCAL_SPECIFICATION),1) -EXTRA_MOUNTS += -v $(PWD)/../specification/specificaiton:/collection/specification +EXTRA_MOUNTS += -v $(PWD)/../specification/specification:/collection/specification endif ifdef ($(LOCAL_DL_PYTHON_PATH),) From c3f0a709f67686f40392bb2867ae975c31b4caa9 Mon Sep 17 00:00:00 2001 From: Toby Petty Date: Wed, 12 Jan 2022 11:48:12 +0000 Subject: [PATCH 7/8] Do not modify workdir, instead for DEVELOPMENT=1, mount over output file directories --- collection.mk | 4 ++-- docker.mk | 17 ++++++++++++++--- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/collection.mk b/collection.mk index d42c0a4..7bf0908 100644 --- a/collection.mk +++ b/collection.mk @@ -37,10 +37,10 @@ first-pass:: collect second-pass:: collection collect:: $(SOURCE_CSV) $(ENDPOINT_CSV) - $(digital-land) collect $(ENDPOINT_CSV) + $(digital-land) collect $(ENDPOINT_CSV) --collection-dir $(COLLECTION_DIR) collection:: - $(digital-land) collection-save-csv + $(digital-land) collection-save-csv --collection-dir $(COLLECTION_DIR) clobber-today:: rm -rf $(LOG_FILES_TODAY) $(COLLECTION_INDEX) diff --git a/docker.mk b/docker.mk index e2f3d84..cbc02a7 100644 --- a/docker.mk +++ b/docker.mk @@ -13,10 +13,21 @@ $(info DOCKERISED is $(DOCKERISED)) $(info DEVELOPMENT is $(DEVELOPMENT)) EXTRA_MOUNTS := +EXTRA_DL_ARGS := # ifeq ($(and $(DOCKERISED),$(DEVELOPMENT))) ifeq ($(DOCKERISED),1) ifeq ($(DEVELOPMENT),1) -EXTRA_MOUNTS += -v $(PWD)/local_collection:/data --workdir /data +EXTRA_MOUNTS += -v $(PWD)/local_collection/collection/log:/pipeline/collection/log +EXTRA_MOUNTS += -v $(PWD)/local_collection/collection/resource:/pipeline/collection/resource +ifneq (,$(wildcard ./fixed)) +EXTRA_MOUNTS += -v $(PWD)/local_collection/fixed:/pipeline/fixed +endif +ifneq (,$(wildcard ./harmonised)) +EXTRA_MOUNTS += -v $(PWD)/local_collection/harmonised:/pipeline/harmonised +endif +ifneq (,$(wildcard ./harmonised)) +EXTRA_MOUNTS += -v $(PWD)/local_collection/transformed:/pipeline/transformed +endif ifdef ($(LOCAL_SPECIFICATION_PATH),) EXTRA_MOUNTS += -v $(LOCAL_SPECIFICATION_PATH)/specification:/collection/specification @@ -36,7 +47,7 @@ $(info EXTRA_MOUNTS is $(EXTRA_MOUNTS)) DOCKER_TAG=latest ECR_URL=public.ecr.aws/l6z6v3j6/ -COLLECTION_DIR=/pipeline/collection/ +EXTRA_DL_ARGS += --specification-dir /collection/specification /pipeline/collection/resource.csv: @@ -50,7 +61,7 @@ digital-land = docker run -t \ $(EXTRA_MOUNTS) \ $(ECR_URL)digital-land-python:$(DOCKER_TAG) \ digital-land \ - --specification-dir /collection/specification + $(EXTRA_DL_ARGS) docker-pull:: ifndef ($(DISABLE_DOCKER_PULL),) From db9152c4c1b864e4e905b3b0869715ef00792ace Mon Sep 17 00:00:00 2001 From: Toby Petty Date: Wed, 12 Jan 2022 13:16:58 +0000 Subject: [PATCH 8/8] Trying to work around issues with some commands running in a containeried way and some not Issue is aruond directory paths being different for dockerised and non-dockerised case Nicest solution might be a shell command Not sure that running the makefile outside the container is the best approach anymore --- docker.mk | 18 ++++++++++++++++-- pipeline.mk | 42 +++++++++++++++++++++--------------------- 2 files changed, 37 insertions(+), 23 deletions(-) diff --git a/docker.mk b/docker.mk index cbc02a7..1e5e95f 100644 --- a/docker.mk +++ b/docker.mk @@ -55,11 +55,22 @@ EXTRA_DL_ARGS += --specification-dir /collection/specification /pipeline/collection/endpoint.csv: -digital-land = docker run -t \ +dockerised = docker run -t \ -e LOCAL_USER_ID=$(shell id -u) \ + -e AWS_ACCESS_KEY_ID \ + -e AWS_DEFAULT_REGION \ + -e AWS_REGION \ + -e AWS_SECRET_ACCESS_KEY \ + -e AWS_SECURITY_TOKEN \ + -e AWS_SESSION_EXPIRATION \ + -e AWS_SESSION_TOKEN \ -v $(PWD):/pipeline \ $(EXTRA_MOUNTS) \ - $(ECR_URL)digital-land-python:$(DOCKER_TAG) \ + $(ECR_URL)digital-land-python:$(DOCKER_TAG) + +shell_cmd = $(dockerised) bash + +digital-land = $(dockerised) \ digital-land \ $(EXTRA_DL_ARGS) @@ -70,6 +81,9 @@ endif init:: docker-pull else +shell_cmd = $(SHELL) digital-land = digital-land endif +debug_shell: + $(shell_cmd) diff --git a/pipeline.mk b/pipeline.mk index 8bd05df..748c5fa 100644 --- a/pipeline.mk +++ b/pipeline.mk @@ -61,7 +61,7 @@ endef define build-dataset = mkdir -p $(@D) time $(digital-land) --pipeline-name $(notdir $(basename $@)) load-entries --output-path $(basename $@).sqlite3 $(^) - time digital-land --pipeline-name $(notdir $(basename $@)) build-dataset $(basename $@).sqlite3 $@ + time $(digital-land) --pipeline-name $(notdir $(basename $@)) build-dataset $(basename $@).sqlite3 $@ endef collection:: collection/pipeline.mk @@ -69,7 +69,7 @@ collection:: collection/pipeline.mk -include collection/pipeline.mk collection/pipeline.mk: collection/resource.csv collection/source.csv - $(digital-land) collection-pipeline-makerules > collection/pipeline.mk + $(shell_cmd) -c "digital-land $(EXTRA_DL_ARGS) collection-pipeline-makerules > collection/pipeline.mk" # restart the make process to pick-up collected resource files second-pass:: @@ -88,26 +88,26 @@ endif endif clobber:: - rm -rf $(TRANSFORMED_DIR) $(ISSUE_DIR) $(DATASET_DIR) + $(shell_cmd) rm -rf $(TRANSFORMED_DIR) $(ISSUE_DIR) $(DATASET_DIR) clean:: - rm -rf ./var + $(shell_cmd) rm -rf ./var # local copies of the organisation dataset needed by harmonise init:: @mkdir -p $(CACHE_DIR) - curl -qfs "https://raw.githubusercontent.com/digital-land/organisation-dataset/main/collection/organisation.csv" > $(CACHE_DIR)organisation.csv + $(shell_cmd) curl -qfs "https://raw.githubusercontent.com/digital-land/organisation-dataset/main/collection/organisation.csv" > $(CACHE_DIR)organisation.csv makerules:: - curl -qfsL '$(SOURCE_URL)/makerules/main/pipeline.mk' > makerules/pipeline.mk + $(shell_cmd) curl -qfsL '$(SOURCE_URL)/makerules/main/pipeline.mk' > makerules/pipeline.mk fetch-s3:: - aws s3 sync s3://collection-dataset/$(REPOSITORY)/$(RESOURCE_DIR) $(RESOURCE_DIR) --no-progress + $(shell_cmd) aws s3 sync s3://collection-dataset/$(REPOSITORY)/$(RESOURCE_DIR) $(RESOURCE_DIR) --no-progress fetch-transformed-s3:: - aws s3 sync s3://collection-dataset/$(REPOSITORY)/$(ISSUE_DIR) $(ISSUE_DIR) --no-progress - aws s3 sync s3://collection-dataset/$(REPOSITORY)/$(TRANSFORMED_DIR) $(TRANSFORMED_DIR) --no-progress - aws s3 sync s3://collection-dataset/$(REPOSITORY)/$(DATASET_DIR) $(DATASET_DIR) --no-progress + $(shell_cmd) aws s3 sync s3://collection-dataset/$(REPOSITORY)/$(ISSUE_DIR) $(ISSUE_DIR) --no-progress + $(shell_cmd) aws s3 sync s3://collection-dataset/$(REPOSITORY)/$(TRANSFORMED_DIR) $(TRANSFORMED_DIR) --no-progress + $(shell_cmd) aws s3 sync s3://collection-dataset/$(REPOSITORY)/$(DATASET_DIR) $(DATASET_DIR) --no-progress # These will run as usual if DEVELOPMENT isn't explicitly set to 1 ifeq ($(DEVELOPMENT),0) @@ -117,19 +117,19 @@ commit-dataset:: git diff --quiet && git diff --staged --quiet || (git commit -m "Data $(shell date +%F)"; git push origin $(BRANCH)) push-collection-s3:: - aws s3 sync $(RESOURCE_DIR) s3://collection-dataset/$(REPOSITORY)/$(RESOURCE_DIR) --no-progress - aws s3 cp $(COLLECTION_DIR)/log.csv s3://collection-dataset/$(REPOSITORY)/$(COLLECTION_DIR) --no-progress - aws s3 cp $(COLLECTION_DIR)/resource.csv s3://collection-dataset/$(REPOSITORY)/$(COLLECTION_DIR) --no-progress - aws s3 cp $(COLLECTION_DIR)/source.csv s3://collection-dataset/$(REPOSITORY)/$(COLLECTION_DIR) --no-progress - aws s3 cp $(COLLECTION_DIR)/endpoint.csv s3://collection-dataset/$(REPOSITORY)/$(COLLECTION_DIR) --no-progress + $(shell_cmd) aws s3 sync $(RESOURCE_DIR) s3://collection-dataset/$(REPOSITORY)/$(RESOURCE_DIR) --no-progress + $(shell_cmd) aws s3 cp $(COLLECTION_DIR)/log.csv s3://collection-dataset/$(REPOSITORY)/$(COLLECTION_DIR) --no-progress + $(shell_cmd) aws s3 cp $(COLLECTION_DIR)/resource.csv s3://collection-dataset/$(REPOSITORY)/$(COLLECTION_DIR) --no-progress + $(shell_cmd) aws s3 cp $(COLLECTION_DIR)/source.csv s3://collection-dataset/$(REPOSITORY)/$(COLLECTION_DIR) --no-progress + $(shell_cmd) aws s3 cp $(COLLECTION_DIR)/endpoint.csv s3://collection-dataset/$(REPOSITORY)/$(COLLECTION_DIR) --no-progress push-dataset-s3:: - @mkdir -p $(TRANSFORMED_DIR) - aws s3 sync $(TRANSFORMED_DIR) s3://collection-dataset/$(REPOSITORY)/$(TRANSFORMED_DIR) --no-progress - @mkdir -p $(ISSUE_DIR) - aws s3 sync $(ISSUE_DIR) s3://collection-dataset/$(REPOSITORY)/$(ISSUE_DIR) --no-progress - @mkdir -p $(DATASET_DIR) - aws s3 sync $(DATASET_DIR) s3://collection-dataset/$(REPOSITORY)/$(DATASET_DIR) --no-progress + @$(shell_cmd) mkdir -p $(TRANSFORMED_DIR) + $(shell_cmd) aws s3 sync $(TRANSFORMED_DIR) s3://collection-dataset/$(REPOSITORY)/$(TRANSFORMED_DIR) --no-progress + @$(shell_cmd) mkdir -p $(ISSUE_DIR) + $(shell_cmd) aws s3 sync $(ISSUE_DIR) s3://collection-dataset/$(REPOSITORY)/$(ISSUE_DIR) --no-progress + @$(shell_cmd) mkdir -p $(DATASET_DIR) + $(shell_cmd) aws s3 sync $(DATASET_DIR) s3://collection-dataset/$(REPOSITORY)/$(DATASET_DIR) --no-progress endif pipeline-run::