From 610b783a7e807fb4f530a0f6faaede0ac644a200 Mon Sep 17 00:00:00 2001 From: kena vyas Date: Wed, 14 May 2025 15:07:22 +0100 Subject: [PATCH 1/7] create build-performance and update dockerfile --- Dockerfile | 2 +- task/build-performance.sh | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 task/build-performance.sh diff --git a/Dockerfile b/Dockerfile index f8be847..eed8901 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,5 +21,5 @@ RUN pip install awscli RUN pip install --upgrade pip RUN pip3 install --upgrade -r requirements.txt -ENTRYPOINT ["./run.sh"] +CMD ["./run.sh"] diff --git a/task/build-performance.sh b/task/build-performance.sh new file mode 100644 index 0000000..5c2fbfe --- /dev/null +++ b/task/build-performance.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +set -e + +ENTRY_DATE=$(date +%Y-%m-%d) +echo "Performance build for entry date: $ENTRY_DATE" + +# run the script to generate the provision-quality dataset +echo "Running generate_provision_quality.py for entry date: $ENTRY_DATE" +python3 -c "from digital_land.utils.generate_provision_quality import generate_provision_quality; generate_provision_quality()" + +OUTPUT_FILE="/tmp/performance/provision-quality/entry-date=${ENTRY_DATE}/provision-quality.parquet" + +if [[ -n "$COLLECTION_DATASET_BUCKET_NAME" ]]; then + if [[ -f "$OUTPUT_FILE" ]]; then + echo "Uploading $OUTPUT_FILE to s3" + aws s3 cp "$OUTPUT_FILE" "s3://${COLLECTION_DATASET_BUCKET_NAME}/performance/provision-quality/entry-date=${ENTRY_DATE}/provision-quality.parquet" + else + echo "ERROR: Output file $OUTPUT_FILE not found. Skipping S3 upload." + exit 1 + fi +else + echo "COLLECTION_DATASET_BUCKET_NAME not set. Skipping S3 upload." +fi + +echo "Performance build complete" \ No newline at end of file From 5c85f9f2996e02f11d6c45f20ba05672076ff23f Mon Sep 17 00:00:00 2001 From: kena vyas Date: Wed, 14 May 2025 15:09:33 +0100 Subject: [PATCH 2/7] add geopandas --- task/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/task/requirements.txt b/task/requirements.txt index 023af0e..fbf1ccb 100644 --- a/task/requirements.txt +++ b/task/requirements.txt @@ -1 +1,2 @@ -e git+https://github.com/digital-land/pipeline.git@main#egg=digital-land +geopandas \ No newline at end of file From 4ea7d6d1411ddfe0ed9959a6c2b3c34cd802ea49 Mon Sep 17 00:00:00 2001 From: kena vyas Date: Wed, 14 May 2025 15:13:58 +0100 Subject: [PATCH 3/7] update reqs --- task/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/task/requirements.txt b/task/requirements.txt index fbf1ccb..19c4bbb 100644 --- a/task/requirements.txt +++ b/task/requirements.txt @@ -1,2 +1,2 @@ --e git+https://github.com/digital-land/pipeline.git@main#egg=digital-land +-e git+https://github.com/digital-land/pipeline.git@provision-quality-dataset#egg=digital-land geopandas \ No newline at end of file From 177f2b49b930439af24c28efd49db7c7778c6a85 Mon Sep 17 00:00:00 2001 From: kena vyas Date: Wed, 14 May 2025 15:23:22 +0100 Subject: [PATCH 4/7] make script executable --- task/build-performance.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 task/build-performance.sh diff --git a/task/build-performance.sh b/task/build-performance.sh old mode 100644 new mode 100755 From 649206b1d44f19bca63cb650e55fdcd096073139 Mon Sep 17 00:00:00 2001 From: kena vyas Date: Thu, 15 May 2025 11:57:52 +0100 Subject: [PATCH 5/7] invoke digital-land command --- task/build-performance.sh | 3 +-- task/requirements.txt | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/task/build-performance.sh b/task/build-performance.sh index 5c2fbfe..c188a9b 100755 --- a/task/build-performance.sh +++ b/task/build-performance.sh @@ -7,8 +7,7 @@ echo "Performance build for entry date: $ENTRY_DATE" # run the script to generate the provision-quality dataset echo "Running generate_provision_quality.py for entry date: $ENTRY_DATE" -python3 -c "from digital_land.utils.generate_provision_quality import generate_provision_quality; generate_provision_quality()" - +digital-land generate-provision-quality OUTPUT_FILE="/tmp/performance/provision-quality/entry-date=${ENTRY_DATE}/provision-quality.parquet" if [[ -n "$COLLECTION_DATASET_BUCKET_NAME" ]]; then diff --git a/task/requirements.txt b/task/requirements.txt index 19c4bbb..624368f 100644 --- a/task/requirements.txt +++ b/task/requirements.txt @@ -1,2 +1 @@ --e git+https://github.com/digital-land/pipeline.git@provision-quality-dataset#egg=digital-land -geopandas \ No newline at end of file +-e git+https://github.com/digital-land/pipeline.git@provision-quality-dataset#egg=digital-land \ No newline at end of file From a24023de3111c64259a8ec478b8298977f189432 Mon Sep 17 00:00:00 2001 From: kena vyas Date: Thu, 15 May 2025 12:40:10 +0100 Subject: [PATCH 6/7] add --specification-dir --- task/build-performance.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/task/build-performance.sh b/task/build-performance.sh index c188a9b..cd0932a 100755 --- a/task/build-performance.sh +++ b/task/build-performance.sh @@ -7,7 +7,7 @@ echo "Performance build for entry date: $ENTRY_DATE" # run the script to generate the provision-quality dataset echo "Running generate_provision_quality.py for entry date: $ENTRY_DATE" -digital-land generate-provision-quality +digital-land generate-provision-quality --specification-dir specification/ OUTPUT_FILE="/tmp/performance/provision-quality/entry-date=${ENTRY_DATE}/provision-quality.parquet" if [[ -n "$COLLECTION_DATASET_BUCKET_NAME" ]]; then From 299c5c2967d4db738e13454947a2adbc4430b051 Mon Sep 17 00:00:00 2001 From: kena vyas Date: Thu, 15 May 2025 13:40:48 +0100 Subject: [PATCH 7/7] use generate_provision_quality from commands.py --- task/build-performance.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/task/build-performance.sh b/task/build-performance.sh index cd0932a..482d777 100755 --- a/task/build-performance.sh +++ b/task/build-performance.sh @@ -7,7 +7,7 @@ echo "Performance build for entry date: $ENTRY_DATE" # run the script to generate the provision-quality dataset echo "Running generate_provision_quality.py for entry date: $ENTRY_DATE" -digital-land generate-provision-quality --specification-dir specification/ +python3 -c "from digital_land.commands import generate_provision_quality; generate_provision_quality()" OUTPUT_FILE="/tmp/performance/provision-quality/entry-date=${ENTRY_DATE}/provision-quality.parquet" if [[ -n "$COLLECTION_DATASET_BUCKET_NAME" ]]; then