From 8f9737888f55765e8b9d8078cea2c4c7764c054f Mon Sep 17 00:00:00 2001 From: BALDO Frederick Date: Mon, 15 Jan 2024 14:43:33 +0100 Subject: [PATCH 1/5] [MacOS] Fix pipeline setup --- .gitlab-ci.yml | 30 ++++-------------------------- darwin.gitlab-ci.yml | 15 +++++---------- linux.gitlab-ci.yml | 18 ++++++++---------- mac.conf | 4 ++-- prepare.sh | 4 ++-- windows.gitlab-ci.yml | 4 +--- 6 files changed, 22 insertions(+), 53 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 188e82c7..4baabd04 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,29 +1,3 @@ -include: - - local: linux.gitlab-ci.yml - rules: - - if: $PLATFORM == 'linux' - when: always - - if: $PLATFORM == 'windows' - when: never - - if: $PLATFORM == 'darwin' - when: never - - local: windows.gitlab-ci.yml - rules: - - if: $PLATFORM == 'windows' - when: always - - if: $PLATFORM == 'linux' - when: never - - if: $PLATFORM == 'darwin' - when: never - - local: darwin.gitlab-ci.yml - rules: - - if: $PLATFORM == 'darwin' - when: always - - if: $PLATFORM == 'linux' - when: never - - if: $PLATFORM == 'windows' - when: never - default: image: docker-hosted.snap-ci.ovh/snap-installer:10.0.0-RC1-snapshot cache: @@ -35,6 +9,7 @@ default: variables: VERSION: 10.0.0-RC1 + # Can be 'linux' | 'windows' | 'darwin' PLATFORM: linux PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" # $DOCKER_AUTH_CONFIG should have been setup as projet variable @@ -61,3 +36,6 @@ variables: stages: - test - report + +include: + - local: $PLATFORM.gitlab-ci.yml \ No newline at end of file diff --git a/darwin.gitlab-ci.yml b/darwin.gitlab-ci.yml index ea2c95f6..94ce2ec3 100644 --- a/darwin.gitlab-ci.yml +++ b/darwin.gitlab-ci.yml @@ -2,10 +2,7 @@ test_mac: tags: - mac stage: test - rules: - - if: $PLATFORM == 'darwin' - when: always - - when: never + # rules: # - if: $CI_COMMIT_TAG # variables: # DL_URL: https://nexus.snap-ci.ovh/repository/snap-maven-releases/org/esa/snap/installers/snap_all_macos/RC10/snap_all_macos-RC10.dmg @@ -28,7 +25,7 @@ test_mac: -DdeployAtEnd=false -Dmaven.wagon.http.ssl.insecure=true -Dmaven.wagon.http.ssl.allowall=true - DL_URL: https://nexus.snap-ci.ovh/repository/snap-maven-releases/org/esa/snap/installers-snapshot/snap_all_macos_archive/$VERSION/snap_all_macos_archive-$VERSION.dmg + DL_URL: "https://nexus.snap-ci.ovh/repository/snap-maven-releases/org/esa/snap/installers-snapshot/snap_all_macos_archive/${VERSION}/snap_all_macos_archive-$VERSION.dmg" before_script: - mvn $MAVEN_CLI_OPTS package install - ls "${CI_PROJECT_DIR}/gpt-tests-executer/target" @@ -39,13 +36,13 @@ test_mac: - pip3 install -r $CI_PROJECT_DIR/requirements.txt - curl --progress-bar "${DL_URL}" >> "${CI_PROJECT_DIR}/${SNAP_INSTALLER}" - hdiutil attach ${CI_PROJECT_DIR}/${SNAP_INSTALLER} - - "/Volumes/esa-snap 1/ESA SNAP.app/Contents/MacOS/JavaApplicationStub" -q -console -dir $SNAP_DIR + - '"/Volumes/esa-snap/ESA SNAP.app/Contents/MacOS/JavaApplicationStub" -dir ${SNAP_DIR} -q -console -overwrite' - ls $SNAP_DIR/bin - python3 $CI_PROJECT_DIR/pygpt/filter_json.py gpt-tests-resources/tests $SCOPE $REPORT_DIR script: - $CI_PROJECT_DIR/prepare.sh "${REPORT_DIR}" "${TEST_DATA_DIR}" "${TEST_DATA_LIST}" "${S3_BUCKET}" "${S3_ARGS}" "${CI_PROJECT_DIR}" after_script: - - hdiutil eject /dev/disk3 + - hdiutil detach /dev/disk3 artifacts: when: always expire_in: "3 days" @@ -57,9 +54,7 @@ report: tags: - mac rules: - - if: $PLATFORM == 'darwin' && $SCOPE != 'CItest' - when: always - - when: never + - if: $SCOPE != 'CItest' - if: $CI_COMMIT_BRANCH == 'master' variables: VERSION: snap:master diff --git a/linux.gitlab-ci.yml b/linux.gitlab-ci.yml index f4b8b8a7..053140d9 100644 --- a/linux.gitlab-ci.yml +++ b/linux.gitlab-ci.yml @@ -56,7 +56,7 @@ test_CItest: variables: TEST_DATA_LIST: singleTestData.txt rules: - - if: $SCOPE == 'CItest' && $PLATFORM == 'linux' + - if: $SCOPE == 'CItest' when: always - when: never parallel: 2 @@ -67,7 +67,7 @@ test_s3tbx: variables: TEST_DATA_LIST: singleTestData.txt rules: - - if: $SCOPE == 's3tbx' && $PLATFORM == 'linux' + - if: $SCOPE == 's3tbx' when: always - when: never parallel: 22 @@ -76,7 +76,7 @@ test_s3tbx: test_s2tbx: tags: [kube] rules: - - if: $SCOPE == 's2tbx' && $PLATFORM == 'linux' + - if: $SCOPE == 's2tbx' when: always - when: never parallel: 100 @@ -85,7 +85,7 @@ test_s2tbx: test_s1tbx: tags: [kube] rules: - - if: $SCOPE == 's1tbx' && $PLATFORM == 'linux' + - if: $SCOPE == 's1tbx' when: always - when: never parallel: 34 @@ -94,7 +94,7 @@ test_s1tbx: test_snap: tags: [kube] rules: - - if: $SCOPE == 'snap' && $PLATFORM == 'linux' + - if: $SCOPE == 'snap' when: always - when: never parallel: 26 @@ -103,7 +103,7 @@ test_snap: test_weekly: tags: [kube] rules: - - if: $SCOPE == 'weekly' && $PLATFORM == 'linux' + - if: $SCOPE == 'weekly' when: always - when: never parallel: 117 @@ -112,7 +112,7 @@ test_weekly: test_daily: tags: [kube] rules: - - if: $SCOPE == 'daily' && $PLATFORM == 'linux' + - if: $SCOPE == 'daily' when: always - when: never parallel: 181 @@ -121,9 +121,7 @@ test_daily: report: stage: report rules: - - if: $PLATFORM == 'linux' && $SCOPE != 'CItest' - when: always - - when: never + - if: $SCOPE != 'CItest' - if: $CI_COMMIT_BRANCH == 'master' variables: VERSION: snap:master diff --git a/mac.conf b/mac.conf index 7ea7c4b2..865d62bd 100644 --- a/mac.conf +++ b/mac.conf @@ -1,6 +1,6 @@ testFolder=./gpt-tests-resources/tests graphFolder=./gpt-tests-resources/graphs -inputFolder=testData +inputFolder=/Volumes/snap-ext/SNAP/testData expectedOutputFolder=./gpt-tests-resources/expectedOutputs -tempFolder=./tmp +tempFolder=/Users/otb/tmp snapBin=/Users/otb/Applications/snap/bin \ No newline at end of file diff --git a/prepare.sh b/prepare.sh index e603f05a..bf84259b 100755 --- a/prepare.sh +++ b/prepare.sh @@ -16,10 +16,10 @@ CI_PROJECT_DIR=$6 while IFS="" read -r test || [ -n "$test" ] do echo "Start get test data for ${test}" - python3 pygpt/get_test_data_list.py "${test}" "${REPORT_DIR}" + python3 pygpt/get_test_data_list.py "${test}" "${REPORT_DIR}" "${TEST_DATA_DIR}" cat "${REPORT_DIR}/${TEST_DATA_LIST}" # Download test data - ./download_test_data.sh "${REPORT_DIR}" "${TEST_DATA_DIR}" "${TEST_DATA_LIST}" "${S3_BUCKET}" "${S3_ARGS}" + "${CI_PROJECT_DIR}/download_test_data.sh" "${REPORT_DIR}" "${TEST_DATA_DIR}" "${TEST_DATA_LIST}" "${S3_BUCKET}" "${S3_ARGS}" echo "Running ${test}" python3 pygpt/snap_gpt_test.py "${JAVA_HOME}/bin/java" "${JAVA_OPTIONS} -cp gpt-tests-executer/target/gpt-test-exec.jar" \ diff --git a/windows.gitlab-ci.yml b/windows.gitlab-ci.yml index 70420519..5442b748 100644 --- a/windows.gitlab-ci.yml +++ b/windows.gitlab-ci.yml @@ -54,9 +54,7 @@ report: - windows cache: [] rules: - - if: $PLATFORM == 'windows' && $SCOPE != 'CItest' - when: always - - when: never + - if: $SCOPE != 'CItest' variables: REPORT_DIR: result # output upload and download progress every 2 seconds From a92959eaa70e9d5c5b371eb7d40747be70fb47b7 Mon Sep 17 00:00:00 2001 From: BALDO Frederick Date: Tue, 23 Jan 2024 14:26:50 +0100 Subject: [PATCH 2/5] [MacOS] Copy files instead of run installer --- darwin.gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/darwin.gitlab-ci.yml b/darwin.gitlab-ci.yml index 94ce2ec3..d467e20b 100644 --- a/darwin.gitlab-ci.yml +++ b/darwin.gitlab-ci.yml @@ -36,7 +36,9 @@ test_mac: - pip3 install -r $CI_PROJECT_DIR/requirements.txt - curl --progress-bar "${DL_URL}" >> "${CI_PROJECT_DIR}/${SNAP_INSTALLER}" - hdiutil attach ${CI_PROJECT_DIR}/${SNAP_INSTALLER} - - '"/Volumes/esa-snap/ESA SNAP.app/Contents/MacOS/JavaApplicationStub" -dir ${SNAP_DIR} -q -console -overwrite' + # - '"/Volumes/esa-snap/ESA SNAP.app/Contents/MacOS/JavaApplicationStub" -dir ${SNAP_DIR} -q -console -overwrite' + - rm -rf $SNAP_DIR && ls /Volumes/esa-snap + - cp -rf "/Volumes/esa-snap/ESA SNAP.app/Contents/Resources/app" $SNAP_DIR && ls $SNAP_DIR - ls $SNAP_DIR/bin - python3 $CI_PROJECT_DIR/pygpt/filter_json.py gpt-tests-resources/tests $SCOPE $REPORT_DIR script: From 93cdd9eb3f0ed290be00a24d51268077ed1c0ebb Mon Sep 17 00:00:00 2001 From: BALDO Frederick Date: Wed, 24 Jan 2024 14:49:35 +0100 Subject: [PATCH 3/5] Re-move to installing snap --- darwin.gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/darwin.gitlab-ci.yml b/darwin.gitlab-ci.yml index d467e20b..057d922a 100644 --- a/darwin.gitlab-ci.yml +++ b/darwin.gitlab-ci.yml @@ -36,9 +36,9 @@ test_mac: - pip3 install -r $CI_PROJECT_DIR/requirements.txt - curl --progress-bar "${DL_URL}" >> "${CI_PROJECT_DIR}/${SNAP_INSTALLER}" - hdiutil attach ${CI_PROJECT_DIR}/${SNAP_INSTALLER} - # - '"/Volumes/esa-snap/ESA SNAP.app/Contents/MacOS/JavaApplicationStub" -dir ${SNAP_DIR} -q -console -overwrite' - - rm -rf $SNAP_DIR && ls /Volumes/esa-snap - - cp -rf "/Volumes/esa-snap/ESA SNAP.app/Contents/Resources/app" $SNAP_DIR && ls $SNAP_DIR + - '"/Volumes/esa-snap/ESA SNAP.app/Contents/MacOS/JavaApplicationStub" -dir ${SNAP_DIR} -q -console -overwrite' + # - rm -rf $SNAP_DIR && ls /Volumes/esa-snap + # - cp -rf "/Volumes/esa-snap/ESA SNAP.app/Contents/Resources/app" $SNAP_DIR && ls $SNAP_DIR - ls $SNAP_DIR/bin - python3 $CI_PROJECT_DIR/pygpt/filter_json.py gpt-tests-resources/tests $SCOPE $REPORT_DIR script: From 691cfe259d846d00c6cea2aa7b10fedf55cf14f6 Mon Sep 17 00:00:00 2001 From: BALDO Frederick Date: Wed, 24 Jan 2024 15:02:56 +0100 Subject: [PATCH 4/5] fix --- darwin.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/darwin.gitlab-ci.yml b/darwin.gitlab-ci.yml index 057d922a..d74ed1fb 100644 --- a/darwin.gitlab-ci.yml +++ b/darwin.gitlab-ci.yml @@ -36,7 +36,7 @@ test_mac: - pip3 install -r $CI_PROJECT_DIR/requirements.txt - curl --progress-bar "${DL_URL}" >> "${CI_PROJECT_DIR}/${SNAP_INSTALLER}" - hdiutil attach ${CI_PROJECT_DIR}/${SNAP_INSTALLER} - - '"/Volumes/esa-snap/ESA SNAP.app/Contents/MacOS/JavaApplicationStub" -dir ${SNAP_DIR} -q -console -overwrite' + - 'INSTALL4J_LOG=yes "/Volumes/esa-snap/ESA SNAP.app/Contents/MacOS/JavaApplicationStub" -dir ${SNAP_DIR} -q -console -overwrite' # - rm -rf $SNAP_DIR && ls /Volumes/esa-snap # - cp -rf "/Volumes/esa-snap/ESA SNAP.app/Contents/Resources/app" $SNAP_DIR && ls $SNAP_DIR - ls $SNAP_DIR/bin From 1f183d585661720792a2448e827d7f4e5533b3f0 Mon Sep 17 00:00:00 2001 From: BALDO Frederick Date: Wed, 24 Jan 2024 16:33:15 +0100 Subject: [PATCH 5/5] fix --- darwin.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/darwin.gitlab-ci.yml b/darwin.gitlab-ci.yml index d74ed1fb..2fe00bdb 100644 --- a/darwin.gitlab-ci.yml +++ b/darwin.gitlab-ci.yml @@ -36,7 +36,7 @@ test_mac: - pip3 install -r $CI_PROJECT_DIR/requirements.txt - curl --progress-bar "${DL_URL}" >> "${CI_PROJECT_DIR}/${SNAP_INSTALLER}" - hdiutil attach ${CI_PROJECT_DIR}/${SNAP_INSTALLER} - - 'INSTALL4J_LOG=yes "/Volumes/esa-snap/ESA SNAP.app/Contents/MacOS/JavaApplicationStub" -dir ${SNAP_DIR} -q -console -overwrite' + - 'INSTALL4J_LOG=yes "/Volumes/esa-snap/ESA SNAP.app/Contents/MacOS/JavaApplicationStub" -dir ${SNAP_DIR} -q -console -overwrite -Dsnap.mainClass=org.esa.snap.main.Main' # - rm -rf $SNAP_DIR && ls /Volumes/esa-snap # - cp -rf "/Volumes/esa-snap/ESA SNAP.app/Contents/Resources/app" $SNAP_DIR && ls $SNAP_DIR - ls $SNAP_DIR/bin