diff --git a/Dockerfile b/Dockerfile index f8be847..eed8901 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,5 +21,5 @@ RUN pip install awscli RUN pip install --upgrade pip RUN pip3 install --upgrade -r requirements.txt -ENTRYPOINT ["./run.sh"] +CMD ["./run.sh"] diff --git a/task/build-performance.sh b/task/build-performance.sh new file mode 100755 index 0000000..482d777 --- /dev/null +++ b/task/build-performance.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +set -e + +ENTRY_DATE=$(date +%Y-%m-%d) +echo "Performance build for entry date: $ENTRY_DATE" + +# run the script to generate the provision-quality dataset +echo "Running generate_provision_quality.py for entry date: $ENTRY_DATE" +python3 -c "from digital_land.commands import generate_provision_quality; generate_provision_quality()" +OUTPUT_FILE="/tmp/performance/provision-quality/entry-date=${ENTRY_DATE}/provision-quality.parquet" + +if [[ -n "$COLLECTION_DATASET_BUCKET_NAME" ]]; then + if [[ -f "$OUTPUT_FILE" ]]; then + echo "Uploading $OUTPUT_FILE to s3" + aws s3 cp "$OUTPUT_FILE" "s3://${COLLECTION_DATASET_BUCKET_NAME}/performance/provision-quality/entry-date=${ENTRY_DATE}/provision-quality.parquet" + else + echo "ERROR: Output file $OUTPUT_FILE not found. Skipping S3 upload." + exit 1 + fi +else + echo "COLLECTION_DATASET_BUCKET_NAME not set. Skipping S3 upload." +fi + +echo "Performance build complete" \ No newline at end of file diff --git a/task/requirements.txt b/task/requirements.txt index 023af0e..624368f 100644 --- a/task/requirements.txt +++ b/task/requirements.txt @@ -1 +1 @@ --e git+https://github.com/digital-land/pipeline.git@main#egg=digital-land +-e git+https://github.com/digital-land/pipeline.git@provision-quality-dataset#egg=digital-land \ No newline at end of file