From fd02d99f6308c56a6b75f32f0559861c660de169 Mon Sep 17 00:00:00 2001 From: Tom Brooks Date: Mon, 1 Jun 2026 11:49:16 +0100 Subject: [PATCH 1/2] fix assmble.sh to only retain expectations file for the collection it is running --- bin/assemble.sh | 2 +- makerules/pipeline.mk | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/assemble.sh b/bin/assemble.sh index e670353..178a8d7 100755 --- a/bin/assemble.sh +++ b/bin/assemble.sh @@ -164,7 +164,7 @@ echo "Dataset flattened files created" # Fetch previous expectations from S3, to retain historic results on days expectations are skipped if [ -n "$COLLECTION_DATASET_BUCKET_NAME" ]; then echo "Fetching previous expectation results from S3..." - aws s3 sync s3://${COLLECTION_DATASET_BUCKET_NAME}/${OUTPUT_LOG_DIR}expectation/ ${OUTPUT_LOG_DIR}expectation/ --no-progress + aws s3 sync s3://${COLLECTION_DATASET_BUCKET_NAME}/${OUTPUT_LOG_DIR}expectation/dataset=${DATASET_NAME}/ ${OUTPUT_LOG_DIR}expectation/dataset=${DATASET_NAME}/ --no-progress fi diff --git a/makerules/pipeline.mk b/makerules/pipeline.mk index 75c38da..6550227 100644 --- a/makerules/pipeline.mk +++ b/makerules/pipeline.mk @@ -258,7 +258,7 @@ endif save-expectations:: @mkdir -p $(OUTPUT_LOG_DIR) - aws s3 sync $(OUTPUT_LOG_DIR) s3://$(COLLECTION_DATASET_BUCKET_NAME)/$(OUTPUT_LOG_DIR) --no-progress + aws s3 cp $(OUTPUT_LOG_DIR)expectation/dataset=$(DATASET_NAME)/$(DATASET_NAME).parquet s3://$(COLLECTION_DATASET_BUCKET_NAME)/$(OUTPUT_LOG_DIR)expectation/dataset=$(DATASET_NAME)/$(DATASET_NAME).parquet save-performance:: @mkdir -p $(PERFORMANCE_DIR) From fd5e59c7895cea2b5e22c9a13ef06aa542952311 Mon Sep 17 00:00:00 2001 From: Tom Brooks Date: Mon, 1 Jun 2026 12:26:09 +0100 Subject: [PATCH 2/2] ensure issue files are updated as they should be --- makerules/pipeline.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/makerules/pipeline.mk b/makerules/pipeline.mk index 6550227..75c38da 100644 --- a/makerules/pipeline.mk +++ b/makerules/pipeline.mk @@ -258,7 +258,7 @@ endif save-expectations:: @mkdir -p $(OUTPUT_LOG_DIR) - aws s3 cp $(OUTPUT_LOG_DIR)expectation/dataset=$(DATASET_NAME)/$(DATASET_NAME).parquet s3://$(COLLECTION_DATASET_BUCKET_NAME)/$(OUTPUT_LOG_DIR)expectation/dataset=$(DATASET_NAME)/$(DATASET_NAME).parquet + aws s3 sync $(OUTPUT_LOG_DIR) s3://$(COLLECTION_DATASET_BUCKET_NAME)/$(OUTPUT_LOG_DIR) --no-progress save-performance:: @mkdir -p $(PERFORMANCE_DIR)