From 00efbc141525372cf09e90f86c8be8a0e02efdfa Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 28 Oct 2025 04:07:36 -0400 Subject: [PATCH 01/50] feat(deployment): Containerize CLP tools via Docker Compose. --- .../clp_package_utils/general.py | 1 - .../scripts/archive_manager.py | 1 - .../clp_package_utils/scripts/compress.py | 2 +- .../scripts/compress_from_s3.py | 2 +- .../scripts/dataset_manager.py | 5 +- .../clp_package_utils/scripts/decompress.py | 7 +-- .../clp_package_utils/scripts/search.py | 2 +- .../package-template/src/sbin/.common-env.sh | 53 +++++++++++++++++++ .../src/sbin/admin-tools/archive-manager.sh | 5 +- .../src/sbin/admin-tools/dataset-manager.sh | 5 +- .../src/sbin/compress-from-s3.sh | 5 +- .../package-template/src/sbin/compress.sh | 5 +- .../package-template/src/sbin/decompress.sh | 5 +- .../package-template/src/sbin/search.sh | 5 +- .../package-template/src/sbin/start-clp.sh | 5 +- .../package-template/src/sbin/stop-clp.sh | 5 +- .../package/docker-compose.runtime.yaml | 21 ++++++++ 17 files changed, 106 insertions(+), 28 deletions(-) create mode 100755 components/package-template/src/sbin/.common-env.sh create mode 100644 tools/deployment/package/docker-compose.runtime.yaml diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index a1a35dcd9b..9d3693fa84 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -396,7 +396,6 @@ def generate_container_start_cmd( "--rm", "--network", "host", "-w", str(CONTAINER_CLP_HOME), - "-e", f"PYTHONPATH={clp_site_packages_dir}", "-u", f"{os.getuid()}:{os.getgid()}", "--name", container_name, "--log-driver", "local" diff --git a/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py b/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py index 5ccc351a61..f108e29f59 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py +++ b/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py @@ -220,7 +220,6 @@ def main(argv: List[str]) -> int: ) necessary_mounts: List[Optional[DockerMount]] = [ - mounts.clp_home, mounts.logs_dir, mounts.archives_output_dir, ] diff --git a/components/clp-package-utils/clp_package_utils/scripts/compress.py b/components/clp-package-utils/clp_package_utils/scripts/compress.py index 89d8f7365f..ce23daa740 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/compress.py +++ b/components/clp-package-utils/clp_package_utils/scripts/compress.py @@ -230,7 +230,7 @@ def main(argv): container_clp_config, clp_config, get_container_config_filename(container_name) ) - necessary_mounts = [mounts.clp_home, mounts.data_dir, mounts.logs_dir, mounts.input_logs_dir] + necessary_mounts = [mounts.data_dir, mounts.logs_dir, mounts.input_logs_dir] # Write compression logs to a file while True: diff --git a/components/clp-package-utils/clp_package_utils/scripts/compress_from_s3.py b/components/clp-package-utils/clp_package_utils/scripts/compress_from_s3.py index 3f7309d400..fb4ebaa7af 100644 --- a/components/clp-package-utils/clp_package_utils/scripts/compress_from_s3.py +++ b/components/clp-package-utils/clp_package_utils/scripts/compress_from_s3.py @@ -286,7 +286,7 @@ def main(argv): container_clp_config, clp_config, get_container_config_filename(container_name) ) - necessary_mounts = [mounts.clp_home, mounts.data_dir, mounts.logs_dir] + necessary_mounts = [mounts.data_dir, mounts.logs_dir] while True: container_url_list_filename = f"{uuid.uuid4()}.txt" diff --git a/components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py b/components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py index 7b895340e6..9c1b943ab6 100644 --- a/components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py +++ b/components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py @@ -131,10 +131,7 @@ def main(argv: List[str]) -> int: container_clp_config, clp_config, get_container_config_filename(container_name) ) - necessary_mounts = [ - mounts.clp_home, - mounts.logs_dir, - ] + necessary_mounts = [mounts.logs_dir] if clp_config.archive_output.storage.type == StorageType.FS: necessary_mounts.append(mounts.archives_output_dir) diff --git a/components/clp-package-utils/clp_package_utils/scripts/decompress.py b/components/clp-package-utils/clp_package_utils/scripts/decompress.py index d5c4c059dd..86b61f8170 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/decompress.py +++ b/components/clp-package-utils/clp_package_utils/scripts/decompress.py @@ -1,5 +1,6 @@ import argparse import logging +import os import pathlib import shlex import subprocess @@ -110,7 +111,6 @@ def handle_extract_file_cmd( extraction_dir.mkdir(exist_ok=True) container_extraction_dir = pathlib.Path("/") / "mnt" / "extraction-dir" necessary_mounts = [ - mounts.clp_home, mounts.data_dir, mounts.logs_dir, mounts.archives_output_dir, @@ -205,7 +205,7 @@ def handle_extract_stream_cmd( generated_config_path_on_container, generated_config_path_on_host = dump_container_config( container_clp_config, clp_config, get_container_config_filename(container_name) ) - necessary_mounts = [mounts.clp_home, mounts.logs_dir] + necessary_mounts = [mounts.logs_dir] extra_env_vars = { CLP_DB_USER_ENV_VAR_NAME: clp_config.database.username, CLP_DB_PASS_ENV_VAR_NAME: clp_config.database.password, @@ -298,8 +298,9 @@ def main(argv): file_extraction_parser.add_argument( "-f", "--files-from", help="A file listing all files to extract." ) + default_extraction_dir = pathlib.Path(os.environ.get("CLP_PWD_HOST", ".")) file_extraction_parser.add_argument( - "-d", "--extraction-dir", metavar="DIR", default=".", help="Extract files into DIR." + "-d", "--extraction-dir", metavar="DIR", default=default_extraction_dir, help="Extract files into DIR." ) # IR extraction command parser diff --git a/components/clp-package-utils/clp_package_utils/scripts/search.py b/components/clp-package-utils/clp_package_utils/scripts/search.py index 03aa3817b9..05d4e28d4b 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/search.py +++ b/components/clp-package-utils/clp_package_utils/scripts/search.py @@ -128,7 +128,7 @@ def main(argv): generated_config_path_on_container, generated_config_path_on_host = dump_container_config( container_clp_config, clp_config, get_container_config_filename(container_name) ) - necessary_mounts = [mounts.clp_home, mounts.logs_dir] + necessary_mounts = [mounts.logs_dir] extra_env_vars = { CLP_DB_USER_ENV_VAR_NAME: clp_config.database.username, CLP_DB_PASS_ENV_VAR_NAME: clp_config.database.password, diff --git a/components/package-template/src/sbin/.common-env.sh b/components/package-template/src/sbin/.common-env.sh new file mode 100755 index 0000000000..2948b8011c --- /dev/null +++ b/components/package-template/src/sbin/.common-env.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +package_root=$(readlink -f "$script_dir/..") + +if [[ -n "${CLP_HOME:-}" ]]; then + export CLP_HOME="$CLP_HOME" +else + export CLP_HOME="$package_root" +fi + +image_id_file="$CLP_HOME/clp-package-image.id" +version_file="$CLP_HOME/VERSION" + +if [[ -f "$image_id_file" ]]; then + image_id="$(tr -d '[:space:]' < "$image_id_file")" + export CLP_PACKAGE_CONTAINER_IMAGE_REF="$image_id" +elif [[ -f "$version_file" ]]; then + version="$(tr -d '[:space:]' < "$version_file")" + export CLP_PACKAGE_CONTAINER_IMAGE_REF="clp-package:$version" +else + echo "Error: Neither clp-package-image.id nor VERSION file exists." >&2 + return 1 2>/dev/null || exit 1 +fi + +uid="$(id --user 2>/dev/null || echo "1000")" +gid="$(getent group docker | cut -d: -f3 2>/dev/null || echo "999")" +export CLP_FIRST_PARTY_SERVICE_UID_GID="$uid:$gid" + +export CLP_PWD_HOST="$(pwd)" + +if [[ -z "${CLP_DOCKER_PLUGIN_DIR:-}" ]]; then + for dir in \ + "$HOME/.docker/cli-plugins" \ + "/mnt/wsl/docker-desktop/cli-tools/usr/local/lib/docker/cli-plugins" \ + "/usr/local/lib/docker/cli-plugins" \ + "/usr/libexec/docker/cli-plugins"; do + + compose_plugin_path="$dir/docker-compose" + if [[ -f "$compose_plugin_path" ]]; then + export CLP_DOCKER_PLUGIN_DIR="$dir" + break + fi + done + if [[ -z "${CLP_DOCKER_PLUGIN_DIR:-}" ]]; then + echo "Warning: Docker plugin directory not found; Docker Compose may not work inside container." >&2 + fi +fi + +socket="$(docker context inspect --format '{{.Endpoints.docker.Host}}' 2>/dev/null | sed -E 's|^unix://||')" +if [[ -S "$socket" ]]; then + export CLP_DOCKER_SOCK_PATH="$socket" +fi diff --git a/components/package-template/src/sbin/admin-tools/archive-manager.sh b/components/package-template/src/sbin/admin-tools/archive-manager.sh index 8dc9cdecb5..3949330e33 100755 --- a/components/package-template/src/sbin/admin-tools/archive-manager.sh +++ b/components/package-template/src/sbin/admin-tools/archive-manager.sh @@ -1,9 +1,10 @@ #!/usr/bin/env bash script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" -package_root="$script_dir/../.." +common_env_path="$script_dir/../.common-env.sh" +source "$common_env_path" -PYTHONPATH=$(readlink -f "$package_root/lib/python3/site-packages") \ +docker compose -f "$CLP_HOME/docker-compose.runtime.yaml" run --rm clp-runtime \ python3 \ -m clp_package_utils.scripts.archive_manager \ "$@" diff --git a/components/package-template/src/sbin/admin-tools/dataset-manager.sh b/components/package-template/src/sbin/admin-tools/dataset-manager.sh index e0ecb767a1..281b44f72b 100755 --- a/components/package-template/src/sbin/admin-tools/dataset-manager.sh +++ b/components/package-template/src/sbin/admin-tools/dataset-manager.sh @@ -1,9 +1,10 @@ #!/usr/bin/env bash script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" -package_root="$script_dir/../.." +common_env_path="$script_dir/../.common-env.sh" +source "$common_env_path" -PYTHONPATH=$(readlink -f "$package_root/lib/python3/site-packages") \ +docker compose -f "$CLP_HOME/docker-compose.runtime.yaml" run --rm clp-runtime \ python3 \ -m clp_package_utils.scripts.dataset_manager \ "$@" diff --git a/components/package-template/src/sbin/compress-from-s3.sh b/components/package-template/src/sbin/compress-from-s3.sh index 7d3bc8e455..8aede3a885 100755 --- a/components/package-template/src/sbin/compress-from-s3.sh +++ b/components/package-template/src/sbin/compress-from-s3.sh @@ -1,9 +1,10 @@ #!/usr/bin/env bash script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" -package_root="$script_dir/.." +common_env_path="$script_dir/.common-env.sh" +source "$common_env_path" -PYTHONPATH=$(readlink -f "$package_root/lib/python3/site-packages") \ +docker compose -f "$CLP_HOME/docker-compose.runtime.yaml" run --rm clp-runtime \ python3 \ -m clp_package_utils.scripts.compress_from_s3 \ "$@" diff --git a/components/package-template/src/sbin/compress.sh b/components/package-template/src/sbin/compress.sh index c9b7c1b9b5..1cc1b61cf5 100755 --- a/components/package-template/src/sbin/compress.sh +++ b/components/package-template/src/sbin/compress.sh @@ -1,9 +1,10 @@ #!/usr/bin/env bash script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" -package_root="$script_dir/.." +common_env_path="$script_dir/.common-env.sh" +source "$common_env_path" -PYTHONPATH=$(readlink -f "$package_root/lib/python3/site-packages") \ +docker compose -f "$CLP_HOME/docker-compose.runtime.yaml" run --rm clp-runtime \ python3 \ -m clp_package_utils.scripts.compress \ "$@" diff --git a/components/package-template/src/sbin/decompress.sh b/components/package-template/src/sbin/decompress.sh index a8a343bb44..046398e818 100755 --- a/components/package-template/src/sbin/decompress.sh +++ b/components/package-template/src/sbin/decompress.sh @@ -1,9 +1,10 @@ #!/usr/bin/env bash script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" -package_root="$script_dir/.." +common_env_path="$script_dir/.common-env.sh" +source "$common_env_path" -PYTHONPATH=$(readlink -f "$package_root/lib/python3/site-packages") \ +docker compose -f "$CLP_HOME/docker-compose.runtime.yaml" run --rm clp-runtime \ python3 \ -m clp_package_utils.scripts.decompress \ "$@" diff --git a/components/package-template/src/sbin/search.sh b/components/package-template/src/sbin/search.sh index d3987265a1..87af94ffd5 100755 --- a/components/package-template/src/sbin/search.sh +++ b/components/package-template/src/sbin/search.sh @@ -1,9 +1,10 @@ #!/usr/bin/env bash script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" -package_root="$script_dir/.." +common_env_path="$script_dir/.common-env.sh" +source "$common_env_path" -PYTHONPATH=$(readlink -f "$package_root/lib/python3/site-packages") \ +docker compose -f "$CLP_HOME/docker-compose.runtime.yaml" run --rm clp-runtime \ python3 \ -m clp_package_utils.scripts.search \ "$@" diff --git a/components/package-template/src/sbin/start-clp.sh b/components/package-template/src/sbin/start-clp.sh index b4ecabd802..71c3121419 100755 --- a/components/package-template/src/sbin/start-clp.sh +++ b/components/package-template/src/sbin/start-clp.sh @@ -1,9 +1,10 @@ #!/usr/bin/env bash script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" -package_root="$script_dir/.." +common_env_path="$script_dir/.common-env.sh" +source "$common_env_path" -PYTHONPATH=$(readlink -f "$package_root/lib/python3/site-packages") \ +docker compose -f "$CLP_HOME/docker-compose.runtime.yaml" run --rm clp-runtime \ python3 \ -m clp_package_utils.scripts.start_clp \ "$@" diff --git a/components/package-template/src/sbin/stop-clp.sh b/components/package-template/src/sbin/stop-clp.sh index b6ee88a945..fcc3d68553 100755 --- a/components/package-template/src/sbin/stop-clp.sh +++ b/components/package-template/src/sbin/stop-clp.sh @@ -1,9 +1,10 @@ #!/usr/bin/env bash script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" -package_root="$script_dir/.." +common_env_path="$script_dir/.common-env.sh" +source "$common_env_path" -PYTHONPATH=$(readlink -f "$package_root/lib/python3/site-packages") \ +docker compose -f "$CLP_HOME/docker-compose.runtime.yaml" run --rm clp-runtime \ python3 \ -m clp_package_utils.scripts.stop_clp \ "$@" diff --git a/tools/deployment/package/docker-compose.runtime.yaml b/tools/deployment/package/docker-compose.runtime.yaml new file mode 100644 index 0000000000..d7d0511db2 --- /dev/null +++ b/tools/deployment/package/docker-compose.runtime.yaml @@ -0,0 +1,21 @@ +x-service-defaults: &service_defaults + image: "${CLP_PACKAGE_CONTAINER_IMAGE_REF:-clp-package}" + logging: + driver: "local" + stop_grace_period: "60s" + +services: + clp-runtime: + <<: *service_defaults + hostname: "clp_runtime" + user: "${CLP_FIRST_PARTY_SERVICE_UID_GID:-1000:999}" + environment: + CLP_HOME: "${CLP_HOME}" + CLP_PWD_HOST: "${CLP_PWD_HOST:-${PWD}}" + volumes: + - "${CLP_DOCKER_PLUGIN_DIR}:/usr/local/lib/docker/cli-plugins:ro" + - "${CLP_DOCKER_SOCK_PATH}:/var/run/docker.sock" + - "${CLP_HOME}:${CLP_HOME}" + - "/usr/bin/docker:/usr/bin/docker:ro" + stdin_open: true + tty: true From ed3dde2b20f27c3a9ad06aa1a9257008397e708e Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 28 Oct 2025 04:12:50 -0400 Subject: [PATCH 02/50] revert unrelated change --- components/clp-package-utils/clp_package_utils/general.py | 1 + 1 file changed, 1 insertion(+) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 9d3693fa84..a1a35dcd9b 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -396,6 +396,7 @@ def generate_container_start_cmd( "--rm", "--network", "host", "-w", str(CONTAINER_CLP_HOME), + "-e", f"PYTHONPATH={clp_site_packages_dir}", "-u", f"{os.getuid()}:{os.getgid()}", "--name", container_name, "--log-driver", "local" From d30745778ffe29ac752f9de2299f9840cb439d6a Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 28 Oct 2025 04:19:15 -0400 Subject: [PATCH 03/50] fix(docker): Simplify clp-runtime service configuration in Docker Compose --- .../deployment/package/docker-compose.runtime.yaml | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/tools/deployment/package/docker-compose.runtime.yaml b/tools/deployment/package/docker-compose.runtime.yaml index d7d0511db2..60f2ecc328 100644 --- a/tools/deployment/package/docker-compose.runtime.yaml +++ b/tools/deployment/package/docker-compose.runtime.yaml @@ -1,13 +1,11 @@ -x-service-defaults: &service_defaults - image: "${CLP_PACKAGE_CONTAINER_IMAGE_REF:-clp-package}" - logging: - driver: "local" - stop_grace_period: "60s" - services: clp-runtime: - <<: *service_defaults hostname: "clp_runtime" + image: "${CLP_PACKAGE_CONTAINER_IMAGE_REF:-clp-package}" + logging: + driver: "local" + stdin_open: true + tty: true user: "${CLP_FIRST_PARTY_SERVICE_UID_GID:-1000:999}" environment: CLP_HOME: "${CLP_HOME}" @@ -17,5 +15,3 @@ services: - "${CLP_DOCKER_SOCK_PATH}:/var/run/docker.sock" - "${CLP_HOME}:${CLP_HOME}" - "/usr/bin/docker:/usr/bin/docker:ro" - stdin_open: true - tty: true From f871424c3da584de3625377a997b93383a8aa82b Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 28 Oct 2025 06:07:07 -0400 Subject: [PATCH 04/50] debloat the package by avoiding duplicating assets already in the container image. --- taskfile.yaml | 124 +++++++++++---------- taskfiles/docker-images.yaml | 14 +-- tools/docker-images/clp-package/Dockerfile | 16 ++- 3 files changed, 85 insertions(+), 69 deletions(-) diff --git a/taskfile.yaml b/taskfile.yaml index e1a43e77a5..56525e49e9 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -27,6 +27,7 @@ vars: G_NODEJS_22_BIN_DIR: "{{.G_NODEJS_22_BUILD_DIR}}/bin" G_PACKAGE_BUILD_DIR: "{{.G_BUILD_DIR}}/clp-package" G_PACKAGE_VENV_DIR: "{{.G_BUILD_DIR}}/package-venv" + G_PYTHON_LIBS_DIR: "{{.G_BUILD_DIR}}/python-libs" G_WEBUI_BUILD_DIR: "{{.G_BUILD_DIR}}/webui" G_SPIDER_BUILD_DIR: "{{.G_BUILD_DIR}}/spider" @@ -43,6 +44,8 @@ vars: # Checksum files G_PACKAGE_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/package.md5" + G_PYTHON_LIBS_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/python-libs.md5" + G_WEBUI_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/webui.md5" G_WEBUI_CLIENT_NODE_MODULES_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/webui-client-node-modules.md5" G_WEBUI_COMMON_NODE_MODULES_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/webui-common-node-modules.md5" G_WEBUI_LOG_VIEWER_NODE_MODULES_CHECKSUM_FILE: @@ -52,7 +55,7 @@ vars: tasks: default: - deps: ["docker-images:package"] + deps: ["package"] clean: cmds: @@ -107,80 +110,40 @@ tasks: CHECKSUM_FILE: "{{.G_PACKAGE_CHECKSUM_FILE}}" OUTPUT_DIR: "{{.G_PACKAGE_BUILD_DIR}}" sources: - - "{{.G_BUILD_DIR}}/package-venv.md5" - - "{{.G_BUILD_DIR}}/webui.md5" - - "{{.G_CORE_COMPONENT_BUILD_DIR}}/clg" - - "{{.G_CORE_COMPONENT_BUILD_DIR}}/clo" - - "{{.G_CORE_COMPONENT_BUILD_DIR}}/clp" - - "{{.G_CORE_COMPONENT_BUILD_DIR}}/clp-s" - - "{{.G_CORE_COMPONENT_BUILD_DIR}}/indexer" - - "{{.G_CORE_COMPONENT_BUILD_DIR}}/log-converter" - - "{{.G_CORE_COMPONENT_BUILD_DIR}}/reducer-server" - - "{{.G_SPIDER_BUILD_DIR}}/spider-build/src/spider/spider_scheduler" - - "{{.G_SPIDER_BUILD_DIR}}/spider-build/src/spider/spider_worker" + - "{{.G_BUILD_DIR}}/clp-package-image.id" + - "{{.G_WEBUI_BUILD_DIR}}/client/settings.json" + - "{{.G_WEBUI_BUILD_DIR}}/server/dist/settings.json" - "{{.TASKFILE}}" - - "components/clp-mcp-server/dist/*.whl" - - "components/clp-package-utils/dist/*.whl" - - "components/clp-py-utils/dist/*.whl" - - "components/job-orchestration/dist/*.whl" - - "components/package-template/src/**/*" - "tools/deployment/package/**/*" generates: ["{{.CHECKSUM_FILE}}"] deps: - - "core" - - "clp-mcp-server" - - "clp-package-utils" - - "clp-py-utils" - - "deps:spider" - - "init" - - "job-orchestration" - - "package-venv" + - "docker-images:package" - task: "utils:checksum:validate" vars: CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"] - - "webui" cmds: - "rm -rf '{{.OUTPUT_DIR}}'" - - "rsync -a components/package-template/src/ '{{.OUTPUT_DIR}}'" - - "mkdir -p '{{.OUTPUT_DIR}}/lib/python3/site-packages'" - - |- - . "{{.G_PACKAGE_VENV_DIR}}/bin/activate" - pip3 install --upgrade \ - components/clp-mcp-server/dist/*.whl \ - components/clp-package-utils/dist/*.whl \ - components/clp-py-utils/dist/*.whl \ - components/job-orchestration/dist/*.whl \ - -t "{{.OUTPUT_DIR}}/lib/python3/site-packages" - - "mkdir -p '{{.OUTPUT_DIR}}/bin'" - >- - rsync -a - "{{.G_CORE_COMPONENT_BUILD_DIR}}/clg" - "{{.G_CORE_COMPONENT_BUILD_DIR}}/clo" - "{{.G_CORE_COMPONENT_BUILD_DIR}}/clp" - "{{.G_CORE_COMPONENT_BUILD_DIR}}/clp-s" - "{{.G_CORE_COMPONENT_BUILD_DIR}}/indexer" - "{{.G_CORE_COMPONENT_BUILD_DIR}}/log-converter" - "{{.G_CORE_COMPONENT_BUILD_DIR}}/reducer-server" - "{{.G_SPIDER_BUILD_DIR}}/spider-build/src/spider/spider_scheduler" - "{{.G_SPIDER_BUILD_DIR}}/spider-build/src/spider/spider_worker" - "{{.OUTPUT_DIR}}/bin/" + rsync --archive + "components/package-template/src/" + "{{.OUTPUT_DIR}}" - >- - rsync -a - "{{.G_NODEJS_22_BIN_DIR}}/node" - "{{.OUTPUT_DIR}}/bin/node-22" - - "mkdir -p '{{.OUTPUT_DIR}}/var/www/'" + rsync --archive --mkpath + "{{.G_WEBUI_BUILD_DIR}}/client/settings.json" + "{{.OUTPUT_DIR}}/var/www/webui/client/" - >- - rsync -a - "{{.G_WEBUI_BUILD_DIR}}/" - "{{.OUTPUT_DIR}}/var/www/webui" - - |- - cd "{{.OUTPUT_DIR}}/var/www/webui" - PATH="{{.G_NODEJS_22_BIN_DIR}}":$PATH npm ci --omit=dev + rsync --archive --mkpath + "{{.G_WEBUI_BUILD_DIR}}/server/dist/settings.json" + "{{.OUTPUT_DIR}}/var/www/webui/server/dist/" - >- - rsync -a + rsync --archive "tools/deployment/package/" "{{.OUTPUT_DIR}}" + - >- + rsync --archive + "{{.G_BUILD_DIR}}/clp-package-image.id" + "{{.OUTPUT_DIR}}" - "echo '{{.G_PACKAGE_VERSION}}' > '{{.OUTPUT_DIR}}/VERSION'" # This command must be last - task: "utils:checksum:compute" @@ -290,6 +253,9 @@ tasks: rsync -a package.json "{{.OUTPUT_DIR}}/server/" - |- rsync -a package.json package-lock.json "{{.OUTPUT_DIR}}/" + - |- + cd "{{.OUTPUT_DIR}}" + PATH="{{.G_NODEJS_22_BIN_DIR}}":$PATH npm ci --omit=dev - task: "utils:checksum:compute" vars: CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" @@ -503,6 +469,46 @@ tasks: --directory '{{.G_BUILD_DIR}}' --dereference '{{.VERSIONED_PACKAGE_NAME}}' + python-libs: + internal: true + vars: + CHECKSUM_FILE: "{{.G_PYTHON_LIBS_CHECKSUM_FILE}}" + OUTPUT_DIR: "{{.G_PYTHON_LIBS_DIR}}" + sources: + - "{{.G_BUILD_DIR}}/package-venv.md5" + - "{{.TASKFILE}}" + - "components/clp-mcp-server/dist/*.whl" + - "components/clp-package-utils/dist/*.whl" + - "components/clp-py-utils/dist/*.whl" + - "components/job-orchestration/dist/*.whl" + generates: ["{{.CHECKSUM_FILE}}"] + deps: + - "init" + - "clp-mcp-server" + - "clp-package-utils" + - "clp-py-utils" + - "job-orchestration" + - "package-venv" + - task: "utils:checksum:validate" + vars: + CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" + INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"] + cmds: + - "rm -rf '{{.OUTPUT_DIR}}'" + - "mkdir -p '{{.OUTPUT_DIR}}'" + - |- + . "{{.G_PACKAGE_VENV_DIR}}/bin/activate" + pip3 install --upgrade \ + components/clp-mcp-server/dist/*.whl \ + components/clp-package-utils/dist/*.whl \ + components/clp-py-utils/dist/*.whl \ + components/job-orchestration/dist/*.whl \ + -t "{{.OUTPUT_DIR}}" + - task: "utils:checksum:compute" + vars: + CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" + INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"] + package-venv: internal: true vars: diff --git a/taskfiles/docker-images.yaml b/taskfiles/docker-images.yaml index 821c1a91ae..488b943ff6 100644 --- a/taskfiles/docker-images.yaml +++ b/taskfiles/docker-images.yaml @@ -2,14 +2,12 @@ version: "3" tasks: package: - vars: - SRC_DIR: "{{.ROOT_DIR}}/tools/docker-images/clp-package" - dir: "{{.SRC_DIR}}" - sources: - - "{{.G_PACKAGE_CHECKSUM_FILE}}" - - "{{.SRC_DIR}}/**/*" + dir: "{{.ROOT_DIR}}/tools/docker-images/clp-package" deps: - - ":package" + - ":init" + - ":core" + - ":deps:spider" + - ":python-libs" + - ":webui" cmds: - "./build.sh" - - "rsync --archive '{{.G_BUILD_DIR}}/clp-package-image.id' '{{.G_PACKAGE_BUILD_DIR}}'" diff --git a/tools/docker-images/clp-package/Dockerfile b/tools/docker-images/clp-package/Dockerfile index 6728a5f0c9..6dc00016d1 100644 --- a/tools/docker-images/clp-package/Dockerfile +++ b/tools/docker-images/clp-package/Dockerfile @@ -27,6 +27,18 @@ RUN useradd --uid ${UID} --shell /bin/bash --home-dir ${CLP_HOME} ${USER} USER ${USER} WORKDIR ${CLP_HOME} -COPY --link --chown=${UID} ./build/clp-package ${CLP_HOME} +COPY --link --chown=${UID} ./components/package-template/src/ . +COPY --link --chown=${UID} ./build/core/clg bin/ +COPY --link --chown=${UID} ./build/core/clo bin/ +COPY --link --chown=${UID} ./build/core/clp bin/ +COPY --link --chown=${UID} ./build/core/clp-s bin/ +COPY --link --chown=${UID} ./build/core/indexer bin/ +COPY --link --chown=${UID} ./build/core/log-converter bin/ +COPY --link --chown=${UID} ./build/core/reducer-server bin/ COPY --link --chown=${UID} ./build/deps/cpp/mariadb-connector-cpp-install/lib/*/libmariadbcpp.so* \ - ${CLP_HOME}/lib/ + lib/ +COPY --link --chown=${UID} ./build/spider/spider-build/src/spider/spider_scheduler bin/ +COPY --link --chown=${UID} ./build/spider/spider-build/src/spider/spider_worker bin/ +COPY --link --chown=${UID} ./build/nodejs-22/bin/node bin/node-22 +COPY --link --chown=${UID} ./build/python-libs/ lib/python3/site-packages/ +COPY --link --chown=${UID} ./build/webui/ var/www/webui/ From b03539a10f025e38c25c8847449b6abe3bdee814 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 28 Oct 2025 06:31:11 -0400 Subject: [PATCH 05/50] fix(env): Update container image reference format in .common-env.sh --- components/package-template/src/sbin/.common-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/package-template/src/sbin/.common-env.sh b/components/package-template/src/sbin/.common-env.sh index 2948b8011c..9a997383be 100755 --- a/components/package-template/src/sbin/.common-env.sh +++ b/components/package-template/src/sbin/.common-env.sh @@ -17,7 +17,7 @@ if [[ -f "$image_id_file" ]]; then export CLP_PACKAGE_CONTAINER_IMAGE_REF="$image_id" elif [[ -f "$version_file" ]]; then version="$(tr -d '[:space:]' < "$version_file")" - export CLP_PACKAGE_CONTAINER_IMAGE_REF="clp-package:$version" + export CLP_PACKAGE_CONTAINER_IMAGE_REF="ghcr.io/y-scope/clp/clp-package:v$version" else echo "Error: Neither clp-package-image.id nor VERSION file exists." >&2 return 1 2>/dev/null || exit 1 From 1b65afe50723d4ba8a62644e6ff864f90b0a302d Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 28 Oct 2025 20:58:43 -0400 Subject: [PATCH 06/50] feat(core): Add resolve_host_path utility for path resolution in container environments. Update components to utilize resolve_host_path for consistent path handling between host and container; adjust validations to support use of host-mounted paths. --- .../clp_package_utils/controller.py | 82 ++++++++++++------- .../clp_package_utils/general.py | 32 +++++--- .../scripts/archive_manager.py | 4 +- .../clp_package_utils/scripts/compress.py | 23 ++++-- .../scripts/compress_from_s3.py | 25 ++++-- .../scripts/dataset_manager.py | 4 +- .../clp_package_utils/scripts/decompress.py | 16 +++- .../clp_package_utils/scripts/search.py | 6 +- .../clp_package_utils/scripts/start_clp.py | 13 +-- .../clp-py-utils/clp_py_utils/clp_config.py | 19 +++-- components/clp-py-utils/clp_py_utils/core.py | 27 +++++- .../package/docker-compose.runtime.yaml | 2 + 12 files changed, 175 insertions(+), 78 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 6566fab12f..a2645538f5 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -54,6 +54,7 @@ validate_results_cache_config, validate_webui_config, ) +from clp_py_utils.core import resolve_host_path LOG_FILE_ACCESS_MODE = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH @@ -119,9 +120,12 @@ def _set_up_env_for_database(self) -> EnvVarsDict: logs_dir = self._clp_config.logs_directory / component_name validate_db_config(self._clp_config, conf_logging_file, data_dir, logs_dir) - data_dir.mkdir(exist_ok=True, parents=True) - logs_dir.mkdir(exist_ok=True, parents=True) - _chown_paths_if_root(data_dir, logs_dir) + resolved_data_dir = resolve_host_path(data_dir) + resolved_logs_dir = resolve_host_path(logs_dir) + + resolved_data_dir.mkdir(exist_ok=True, parents=True) + resolved_logs_dir.mkdir(exist_ok=True, parents=True) + _chown_paths_if_root(resolved_data_dir, resolved_logs_dir) env_vars = EnvVarsDict() @@ -166,8 +170,10 @@ def _set_up_env_for_queue(self) -> EnvVarsDict: logs_dir = self._clp_config.logs_directory / component_name validate_queue_config(self._clp_config, logs_dir) - logs_dir.mkdir(exist_ok=True, parents=True) - _chown_paths_if_root(logs_dir) + resolved_logs_dir = resolve_host_path(logs_dir) + + resolved_logs_dir.mkdir(exist_ok=True, parents=True) + _chown_paths_if_root(resolved_logs_dir) env_vars = EnvVarsDict() @@ -204,9 +210,12 @@ def _set_up_env_for_redis(self) -> EnvVarsDict: logs_dir = self._clp_config.logs_directory / component_name validate_redis_config(self._clp_config, conf_file, data_dir, logs_dir) - data_dir.mkdir(exist_ok=True, parents=True) - logs_dir.mkdir(exist_ok=True, parents=True) - _chown_paths_if_root(data_dir, logs_dir) + resolved_data_dir = resolve_host_path(data_dir) + resolved_logs_dir = resolve_host_path(logs_dir) + + resolved_data_dir.mkdir(exist_ok=True, parents=True) + resolved_logs_dir.mkdir(exist_ok=True, parents=True) + _chown_paths_if_root(resolved_data_dir, resolved_logs_dir) env_vars = EnvVarsDict() @@ -252,9 +261,12 @@ def _set_up_env_for_results_cache(self) -> EnvVarsDict: logs_dir = self._clp_config.logs_directory / component_name validate_results_cache_config(self._clp_config, conf_file, data_dir, logs_dir) - data_dir.mkdir(exist_ok=True, parents=True) - logs_dir.mkdir(exist_ok=True, parents=True) - _chown_paths_if_root(data_dir, logs_dir) + resolved_data_dir = resolve_host_path(data_dir) + resolved_logs_dir = resolve_host_path(logs_dir) + + resolved_data_dir.mkdir(exist_ok=True, parents=True) + resolved_logs_dir.mkdir(exist_ok=True, parents=True) + _chown_paths_if_root(resolved_data_dir, resolved_logs_dir) env_vars = EnvVarsDict() @@ -291,7 +303,8 @@ def _set_up_env_for_compression_scheduler(self) -> EnvVarsDict: logger.info(f"Setting up environment for {component_name}...") logs_dir = self._clp_config.logs_directory / component_name - logs_dir.mkdir(parents=True, exist_ok=True) + resolved_logs_dir = resolve_host_path(logs_dir) + resolved_logs_dir.mkdir(parents=True, exist_ok=True) env_vars = EnvVarsDict() @@ -314,7 +327,8 @@ def _set_up_env_for_query_scheduler(self) -> EnvVarsDict: logger.info(f"Setting up environment for {component_name}...") logs_dir = self._clp_config.logs_directory / component_name - logs_dir.mkdir(parents=True, exist_ok=True) + resolved_logs_dir = resolve_host_path(logs_dir) + resolved_logs_dir.mkdir(parents=True, exist_ok=True) env_vars = EnvVarsDict() @@ -336,7 +350,8 @@ def _set_up_env_for_compression_worker(self, num_workers: int) -> EnvVarsDict: logger.info(f"Setting up environment for {component_name}...") logs_dir = self._clp_config.logs_directory / component_name - logs_dir.mkdir(parents=True, exist_ok=True) + resolved_logs_dir = resolve_host_path(logs_dir) + resolved_logs_dir.mkdir(parents=True, exist_ok=True) env_vars = EnvVarsDict() @@ -365,7 +380,8 @@ def _set_up_env_for_query_worker(self, num_workers: int) -> EnvVarsDict: logger.info(f"Setting up environment for {component_name}...") logs_dir = self._clp_config.logs_directory / component_name - logs_dir.mkdir(parents=True, exist_ok=True) + resolved_logs_dir = resolve_host_path(logs_dir) + resolved_logs_dir.mkdir(parents=True, exist_ok=True) env_vars = EnvVarsDict() @@ -392,7 +408,8 @@ def _set_up_env_for_reducer(self, num_workers: int) -> EnvVarsDict: logger.info(f"Setting up environment for {component_name}...") logs_dir = self._clp_config.logs_directory / component_name - logs_dir.mkdir(parents=True, exist_ok=True) + resolved_logs_dir = resolve_host_path(logs_dir) + resolved_logs_dir.mkdir(parents=True, exist_ok=True) env_vars = EnvVarsDict() @@ -426,8 +443,12 @@ def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict: server_settings_json_path = ( self._clp_home / "var" / "www" / "webui" / "server" / "dist" / "settings.json" ) + resolved_client_settings_json_path = resolve_host_path(client_settings_json_path) + resolved_server_settings_json_path = resolve_host_path(server_settings_json_path) validate_webui_config( - self._clp_config, client_settings_json_path, server_settings_json_path + self._clp_config, + client_settings_json_path, + server_settings_json_path, ) # Read, update, and write back client's and server's settings.json @@ -455,9 +476,9 @@ def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict: "SqlDbCompressionJobsTableName": COMPRESSION_JOBS_TABLE_NAME, } client_settings_json = self._read_and_update_settings_json( - client_settings_json_path, client_settings_json_updates + resolved_client_settings_json_path, client_settings_json_updates ) - with open(client_settings_json_path, "w") as client_settings_json_file: + with open(resolved_client_settings_json_path, "w") as client_settings_json_file: client_settings_json_file.write(json.dumps(client_settings_json)) server_settings_json_updates = { @@ -510,9 +531,9 @@ def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict: server_settings_json_updates["PrestoPort"] = None server_settings_json = self._read_and_update_settings_json( - server_settings_json_path, server_settings_json_updates + resolved_server_settings_json_path, server_settings_json_updates ) - with open(server_settings_json_path, "w") as settings_json_file: + with open(resolved_server_settings_json_path, "w") as settings_json_file: settings_json_file.write(json.dumps(server_settings_json)) env_vars = EnvVarsDict() @@ -544,7 +565,8 @@ def _set_up_env_for_mcp_server(self) -> EnvVarsDict: logs_dir = self._clp_config.logs_directory / component_name validate_mcp_server_config(self._clp_config, logs_dir) - logs_dir.mkdir(parents=True, exist_ok=True) + resolved_logs_dir = resolve_host_path(logs_dir) + resolved_logs_dir.mkdir(parents=True, exist_ok=True) env_vars = EnvVarsDict() @@ -581,7 +603,8 @@ def _set_up_env_for_garbage_collector(self) -> EnvVarsDict: logger.info(f"Setting up environment for {component_name}...") logs_dir = self._clp_config.logs_directory / component_name - logs_dir.mkdir(parents=True, exist_ok=True) + resolved_logs_dir = resolve_host_path(logs_dir) + resolved_logs_dir.mkdir(parents=True, exist_ok=True) env_vars = EnvVarsDict() @@ -696,8 +719,10 @@ def set_up_env(self) -> None: env_vars["CLP_LOGS_INPUT_DIR_HOST"] = str(self._clp_config.logs_input.directory) # Output config - archive_output_dir_str = str(self._clp_config.archive_output.get_directory()) - stream_output_dir_str = str(self._clp_config.stream_output.get_directory()) + archive_output_dir = self._clp_config.archive_output.get_directory() + stream_output_dir = self._clp_config.stream_output.get_directory() + archive_output_dir_str = str(archive_output_dir) + stream_output_dir_str = str(stream_output_dir) if self._clp_config.archive_output.storage.type == StorageType.FS: env_vars["CLP_ARCHIVE_OUTPUT_DIR_HOST"] = archive_output_dir_str if self._clp_config.archive_output.storage.type == StorageType.S3: @@ -804,13 +829,14 @@ def get_or_create_instance_id(clp_config: CLPConfig) -> str: :return: The instance ID. """ instance_id_file_path = clp_config.logs_directory / "instance-id" + resolved_instance_id_file_path = resolve_host_path(instance_id_file_path) - if instance_id_file_path.exists(): - with open(instance_id_file_path, "r") as f: + if resolved_instance_id_file_path.exists(): + with open(resolved_instance_id_file_path, "r") as f: instance_id = f.readline() else: instance_id = str(uuid.uuid4())[-4:] - with open(instance_id_file_path, "w") as f: + with open(resolved_instance_id_file_path, "w") as f: f.write(instance_id) return instance_id diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index a1a35dcd9b..21b2fa0e43 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -30,6 +30,7 @@ WEBUI_COMPONENT_NAME, WorkerConfig, ) +from clp_py_utils.core import resolve_host_path from clp_py_utils.clp_metadata_db_utils import ( MYSQL_TABLE_NAME_MAX_LEN, TABLE_SUFFIX_MAX_LEN, @@ -355,7 +356,8 @@ def dump_container_config( """ config_file_path_on_host = clp_config.logs_directory / config_filename config_file_path_on_container = container_clp_config.logs_directory / config_filename - with open(config_file_path_on_host, "w") as f: + resolved_config_file_path_on_host = resolve_host_path(config_file_path_on_host) + with open(resolved_config_file_path_on_host, "w") as f: yaml.safe_dump(container_clp_config.dump_to_primitive_dict(), f) return config_file_path_on_container, config_file_path_on_host @@ -427,8 +429,9 @@ def validate_config_key_existence(config, key): def load_config_file( config_file_path: pathlib.Path, default_config_file_path: pathlib.Path, clp_home: pathlib.Path ): - if config_file_path.exists(): - raw_clp_config = read_yaml_config_file(config_file_path) + resolved_config_file_path = resolve_host_path(config_file_path) + if resolved_config_file_path.exists(): + raw_clp_config = read_yaml_config_file(resolved_config_file_path) if raw_clp_config is None: clp_config = CLPConfig() else: @@ -464,16 +467,17 @@ def validate_credentials_file_path( clp_config: CLPConfig, clp_home: pathlib.Path, generate_default_file: bool ): credentials_file_path = clp_config.credentials_file_path - if not credentials_file_path.exists(): + resolved_credentials_file_path = resolve_host_path(credentials_file_path) + if not resolved_credentials_file_path.exists(): if ( make_config_path_absolute(clp_home, CLP_DEFAULT_CREDENTIALS_FILE_PATH) == credentials_file_path and generate_default_file ): - generate_credentials_file(credentials_file_path) + generate_credentials_file(resolved_credentials_file_path) else: raise ValueError(f"Credentials file path '{credentials_file_path}' does not exist.") - elif not credentials_file_path.is_file(): + elif not resolved_credentials_file_path.is_file(): raise ValueError(f"Credentials file path '{credentials_file_path}' is not a file.") @@ -504,7 +508,8 @@ def validate_db_config( data_dir: pathlib.Path, logs_dir: pathlib.Path, ): - if not component_config.exists(): + resolved_component_config = resolve_host_path(component_config) + if not resolved_component_config.exists(): raise ValueError(f"{DB_COMPONENT_NAME} configuration file missing: '{component_config}'.") _validate_data_directory(data_dir, DB_COMPONENT_NAME) _validate_log_directory(logs_dir, DB_COMPONENT_NAME) @@ -524,7 +529,8 @@ def validate_redis_config( data_dir: pathlib.Path, logs_dir: pathlib.Path, ): - if not component_config.exists(): + resolved_component_config = resolve_host_path(component_config) + if not resolved_component_config.exists(): raise ValueError( f"{REDIS_COMPONENT_NAME} configuration file missing: '{component_config}'." ) @@ -551,7 +557,8 @@ def validate_results_cache_config( data_dir: pathlib.Path, logs_dir: pathlib.Path, ): - if not component_config.exists(): + resolved_component_config = resolve_host_path(component_config) + if not resolved_component_config.exists(): raise ValueError( f"{RESULTS_CACHE_COMPONENT_NAME} configuration file missing: '{component_config}'." ) @@ -583,7 +590,8 @@ def validate_webui_config( server_settings_json_path: pathlib.Path, ): for path in [client_settings_json_path, server_settings_json_path]: - if not path.exists(): + resolved_path = resolve_host_path(path) + if not resolved_path.exists(): raise ValueError(f"{WEBUI_COMPONENT_NAME} {path} is not a valid path to settings.json") validate_port(f"{WEBUI_COMPONENT_NAME}.port", clp_config.webui.host, clp_config.webui.port) @@ -765,7 +773,7 @@ def _is_docker_compose_project_running(project_name: str) -> bool: def _validate_data_directory(data_dir: pathlib.Path, component_name: str) -> None: try: - validate_path_could_be_dir(data_dir) + validate_path_could_be_dir(data_dir, True) except ValueError as ex: raise ValueError(f"{component_name} data directory is invalid: {ex}") @@ -779,6 +787,6 @@ def _validate_log_directory(logs_dir: pathlib.Path, component_name: str): :raise ValueError: If the path is invalid or can't be a directory. """ try: - validate_path_could_be_dir(logs_dir) + validate_path_could_be_dir(logs_dir, True) except ValueError as ex: raise ValueError(f"{component_name} logs directory is invalid: {ex}") diff --git a/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py b/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py index f108e29f59..6ba9881c48 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py +++ b/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py @@ -28,6 +28,7 @@ validate_and_load_db_credentials_file, validate_dataset_name, ) +from clp_py_utils.core import resolve_host_path # Command/Argument Constants FIND_COMMAND: Final[str] = "find" @@ -280,7 +281,8 @@ def main(argv: List[str]) -> int: logger.debug(f"Docker command failed: {shlex.join(cmd)}") # Remove generated files - generated_config_path_on_host.unlink() + resolved_generated_config_path_on_host = resolve_host_path(generated_config_path_on_host) + resolved_generated_config_path_on_host.unlink() return ret_code diff --git a/components/clp-package-utils/clp_package_utils/scripts/compress.py b/components/clp-package-utils/clp_package_utils/scripts/compress.py index ce23daa740..c2bdc89ec9 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/compress.py +++ b/components/clp-package-utils/clp_package_utils/scripts/compress.py @@ -29,6 +29,7 @@ validate_and_load_db_credentials_file, validate_dataset_name, ) +from clp_py_utils.core import resolve_host_path logger = logging.getLogger(__name__) @@ -56,7 +57,10 @@ def _generate_logs_list( return len(parsed_args.paths) != 0 no_path_found = True - with open(host_logs_list_path, "r") as host_logs_list_file: + resolved_host_logs_list_path = resolve_host_path( + pathlib.Path(host_logs_list_path) + ) + with open(resolved_host_logs_list_path, "r") as host_logs_list_file: for line in host_logs_list_file: stripped_path_str = line.rstrip() if "" == stripped_path_str: @@ -235,15 +239,16 @@ def main(argv): # Write compression logs to a file while True: # Get unused output path - container_logs_list_filename = f"{uuid.uuid4()}.txt" - container_logs_list_path = clp_config.logs_directory / container_logs_list_filename + logs_list_filename = f"{uuid.uuid4()}.txt" + logs_list_path_on_host = clp_config.logs_directory / logs_list_filename + resolved_logs_list_path_on_host = resolve_host_path(logs_list_path_on_host) logs_list_path_on_container = ( - container_clp_config.logs_directory / container_logs_list_filename + container_clp_config.logs_directory / logs_list_filename ) - if not container_logs_list_path.exists(): + if not resolved_logs_list_path_on_host.exists(): break - if not _generate_logs_list(container_logs_list_path, parsed_args): + if not _generate_logs_list(resolved_logs_list_path_on_host, parsed_args): logger.error("No filesystem paths given for compression.") return -1 @@ -266,9 +271,11 @@ def main(argv): logger.error("Compression failed.") logger.debug(f"Docker command failed: {shlex.join(cmd)}") else: - container_logs_list_path.unlink() + resolved_logs_list_path_on_host.unlink() + + resolved_generated_config_path_on_host = resolve_host_path(generated_config_path_on_host) + resolved_generated_config_path_on_host.unlink() - generated_config_path_on_host.unlink() return ret_code diff --git a/components/clp-package-utils/clp_package_utils/scripts/compress_from_s3.py b/components/clp-package-utils/clp_package_utils/scripts/compress_from_s3.py index fb4ebaa7af..a50a48be00 100644 --- a/components/clp-package-utils/clp_package_utils/scripts/compress_from_s3.py +++ b/components/clp-package-utils/clp_package_utils/scripts/compress_from_s3.py @@ -29,6 +29,7 @@ validate_and_load_db_credentials_file, validate_dataset_name, ) +from clp_py_utils.core import resolve_host_path logger = logging.getLogger(__name__) @@ -56,7 +57,10 @@ def _generate_url_list( return len(parsed_args.inputs) != 0 no_url_found = True - with open(parsed_args.inputs_from, "r") as input_file: + resolved_inputs_from_path = resolve_host_path( + pathlib.Path(parsed_args.inputs_from) + ) + with open(resolved_inputs_from_path, "r") as input_file: for line in input_file: stripped_url = line.strip() if "" == stripped_url: @@ -289,15 +293,16 @@ def main(argv): necessary_mounts = [mounts.data_dir, mounts.logs_dir] while True: - container_url_list_filename = f"{uuid.uuid4()}.txt" - container_url_list_path = clp_config.logs_directory / container_url_list_filename + url_list_filename = f"{uuid.uuid4()}.txt" + url_list_path_on_host = clp_config.logs_directory / url_list_filename + resolved_url_list_path_on_host = resolve_host_path(url_list_path_on_host) url_list_path_on_container = ( - container_clp_config.logs_directory / container_url_list_filename + container_clp_config.logs_directory / url_list_filename ) - if not container_url_list_path.exists(): + if not resolved_url_list_path_on_host.exists(): break - if not _generate_url_list(parsed_args.subcommand, container_url_list_path, parsed_args): + if not _generate_url_list(parsed_args.subcommand, resolved_url_list_path_on_host, parsed_args): logger.error("No S3 URLs given for compression.") return -1 @@ -320,9 +325,13 @@ def main(argv): logger.error("Compression failed.") logger.debug(f"Docker command failed: {shlex.join(cmd)}") else: - container_url_list_path.unlink() + resolved_url_list_path_on_host.unlink() + + resolved_generated_config_path_on_host = resolve_host_path( + generated_config_path_on_host + ) + resolved_generated_config_path_on_host.unlink() - generated_config_path_on_host.unlink() return ret_code diff --git a/components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py b/components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py index 9c1b943ab6..f13d77698f 100644 --- a/components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py +++ b/components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py @@ -14,6 +14,7 @@ StorageEngine, StorageType, ) +from clp_py_utils.core import resolve_host_path from clp_py_utils.s3_utils import generate_container_auth_options from clp_package_utils.general import ( @@ -184,7 +185,8 @@ def main(argv: List[str]) -> int: logger.debug(f"Docker command failed: {shlex.join(cmd)}") # Remove generated files - generated_config_path_on_host.unlink() + resolved_generated_config_path_on_host = resolve_host_path(generated_config_path_on_host) + resolved_generated_config_path_on_host.unlink() return ret_code diff --git a/components/clp-package-utils/clp_package_utils/scripts/decompress.py b/components/clp-package-utils/clp_package_utils/scripts/decompress.py index 86b61f8170..b44b90270c 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/decompress.py +++ b/components/clp-package-utils/clp_package_utils/scripts/decompress.py @@ -35,6 +35,7 @@ validate_dataset_name, validate_path_could_be_dir, ) +from clp_py_utils.core import resolve_host_path logger = logging.getLogger(__file__) @@ -80,7 +81,7 @@ def handle_extract_file_cmd( # Validate extraction directory extraction_dir = pathlib.Path(parsed_args.extraction_dir).resolve() try: - validate_path_could_be_dir(extraction_dir) + validate_path_could_be_dir(extraction_dir, True) except ValueError as ex: logger.error(f"extraction-dir is invalid: {ex}") return -1 @@ -108,7 +109,8 @@ def handle_extract_file_cmd( ) # Set up mounts - extraction_dir.mkdir(exist_ok=True) + resolved_extraction_dir = resolve_host_path(extraction_dir) + resolved_extraction_dir.mkdir(exist_ok=True) container_extraction_dir = pathlib.Path("/") / "mnt" / "extraction-dir" necessary_mounts = [ mounts.data_dir, @@ -161,7 +163,10 @@ def handle_extract_file_cmd( logger.debug(f"Docker command failed: {shlex.join(cmd)}") # Remove generated files - generated_config_path_on_host.unlink() + resolved_generated_config_path_on_host = resolve_host_path( + generated_config_path_on_host + ) + resolved_generated_config_path_on_host.unlink() return ret_code @@ -266,7 +271,10 @@ def handle_extract_stream_cmd( logger.debug(f"Docker command failed: {shlex.join(cmd)}") # Remove generated files - generated_config_path_on_host.unlink() + resolved_generated_config_path_on_host = resolve_host_path( + generated_config_path_on_host + ) + resolved_generated_config_path_on_host.unlink() return ret_code diff --git a/components/clp-package-utils/clp_package_utils/scripts/search.py b/components/clp-package-utils/clp_package_utils/scripts/search.py index 05d4e28d4b..42c57af901 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/search.py +++ b/components/clp-package-utils/clp_package_utils/scripts/search.py @@ -26,6 +26,7 @@ validate_and_load_db_credentials_file, validate_dataset_name, ) +from clp_py_utils.core import resolve_host_path logger = logging.getLogger(__file__) @@ -180,7 +181,10 @@ def main(argv): logger.debug(f"Docker command failed: {shlex.join(cmd)}") # Remove generated files - generated_config_path_on_host.unlink() + resolved_generated_config_path_on_host = resolve_host_path( + generated_config_path_on_host + ) + resolved_generated_config_path_on_host.unlink() return ret_code diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 15a44b7458..633d52ec1c 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -16,6 +16,7 @@ validate_output_storage_config, validate_retention_config, ) +from clp_py_utils.core import resolve_host_path logger = logging.getLogger(__file__) @@ -72,11 +73,13 @@ def main(argv): try: # Create necessary directories. - clp_config.data_directory.mkdir(parents=True, exist_ok=True) - clp_config.logs_directory.mkdir(parents=True, exist_ok=True) - clp_config.tmp_directory.mkdir(parents=True, exist_ok=True) - clp_config.archive_output.get_directory().mkdir(parents=True, exist_ok=True) - clp_config.stream_output.get_directory().mkdir(parents=True, exist_ok=True) + resolve_host_path(clp_config.data_directory).mkdir(parents=True, exist_ok=True) + resolve_host_path(clp_config.logs_directory).mkdir(parents=True, exist_ok=True) + resolve_host_path(clp_config.tmp_directory).mkdir(parents=True, exist_ok=True) + resolve_host_path(clp_config.archive_output.get_directory()).mkdir(parents=True, + exist_ok=True) + resolve_host_path(clp_config.stream_output.get_directory()).mkdir(parents=True, + exist_ok=True) except: logger.exception("Failed to create necessary directories.") return -1 diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 0da114232b..0b152124a1 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -19,6 +19,7 @@ get_config_value, make_config_path_absolute, read_yaml_config_file, + resolve_host_path, validate_path_could_be_dir, ) from .serialization_utils import serialize_path, serialize_str_enum @@ -672,9 +673,10 @@ def validate_logs_input_config(self): # NOTE: This can't be a pydantic validator since input_logs_dir might be a # package-relative path that will only be resolved after pydantic validation input_logs_dir = self.logs_input.directory - if not input_logs_dir.exists(): + resolved_input_logs_dir = resolve_host_path(input_logs_dir) + if not resolved_input_logs_dir.exists(): raise ValueError(f"logs_input.directory '{input_logs_dir}' doesn't exist.") - if not input_logs_dir.is_dir(): + if not resolved_input_logs_dir.is_dir(): raise ValueError(f"logs_input.directory '{input_logs_dir}' is not a directory.") if StorageType.S3 == logs_input_type and StorageEngine.CLP_S != self.package.storage_engine: raise ValueError( @@ -692,7 +694,7 @@ def validate_archive_output_config(self): f" = '{StorageEngine.CLP_S}'" ) try: - validate_path_could_be_dir(self.archive_output.get_directory()) + validate_path_could_be_dir(self.archive_output.get_directory(), True) except ValueError as ex: raise ValueError(f"archive_output.storage's directory is invalid: {ex}") @@ -706,25 +708,25 @@ def validate_stream_output_config(self): f" = '{StorageEngine.CLP_S}'" ) try: - validate_path_could_be_dir(self.stream_output.get_directory()) + validate_path_could_be_dir(self.stream_output.get_directory(), True) except ValueError as ex: raise ValueError(f"stream_output.storage's directory is invalid: {ex}") def validate_data_dir(self): try: - validate_path_could_be_dir(self.data_directory) + validate_path_could_be_dir(self.data_directory, True) except ValueError as ex: raise ValueError(f"data_directory is invalid: {ex}") def validate_logs_dir(self): try: - validate_path_could_be_dir(self.logs_directory) + validate_path_could_be_dir(self.logs_directory, True) except ValueError as ex: raise ValueError(f"logs_directory is invalid: {ex}") def validate_tmp_dir(self): try: - validate_path_could_be_dir(self.tmp_directory) + validate_path_could_be_dir(self.tmp_directory, True) except ValueError as ex: raise ValueError(f"tmp_directory is invalid: {ex}") @@ -749,7 +751,8 @@ def validate_aws_config_dir(self): raise ValueError( "aws_config_directory must be set when using profile authentication" ) - if not self.aws_config_directory.exists(): + if not resolve_host_path(self.aws_config_directory).exists(): + print(resolve_host_path(self.aws_config_directory)) raise ValueError( f"aws_config_directory does not exist: '{self.aws_config_directory}'" ) diff --git a/components/clp-py-utils/clp_py_utils/core.py b/components/clp-py-utils/clp_py_utils/core.py index 95266fa216..aed5b125df 100644 --- a/components/clp-py-utils/clp_py_utils/core.py +++ b/components/clp-py-utils/clp_py_utils/core.py @@ -1,8 +1,11 @@ +import os import pathlib import yaml from yaml.parser import ParserError +CONTAINER_HOST_ROOT_DIR = pathlib.Path("/") / "mnt" / "host" + class FileMetadata: __slots__ = ("path", "size", "estimated_uncompressed_size") @@ -62,8 +65,28 @@ def read_yaml_config_file(yaml_config_file_path: pathlib.Path): return config -def validate_path_could_be_dir(path: pathlib.Path): - part = path +def resolve_host_path(path: pathlib.Path) -> pathlib.Path: + """ + Translates a host path to its container-mount equivalent. + + :param path: The host path. + :return: The translated container path. + """ + path = path.absolute() + resolved = CONTAINER_HOST_ROOT_DIR / path.relative_to("/") + + try: + if resolved.is_symlink(): + target_path = (resolved.parent / resolved.readlink()).resolve() + resolved = CONTAINER_HOST_ROOT_DIR / target_path.relative_to("/") + except OSError: + pass + + return resolved + + +def validate_path_could_be_dir(path: pathlib.Path, use_host_mount: bool): + part = resolve_host_path(path) if use_host_mount else path while True: if part.exists(): if not part.is_dir(): diff --git a/tools/deployment/package/docker-compose.runtime.yaml b/tools/deployment/package/docker-compose.runtime.yaml index 60f2ecc328..6895f85ba7 100644 --- a/tools/deployment/package/docker-compose.runtime.yaml +++ b/tools/deployment/package/docker-compose.runtime.yaml @@ -8,10 +8,12 @@ services: tty: true user: "${CLP_FIRST_PARTY_SERVICE_UID_GID:-1000:999}" environment: + HOME: "${HOME}" CLP_HOME: "${CLP_HOME}" CLP_PWD_HOST: "${CLP_PWD_HOST:-${PWD}}" volumes: - "${CLP_DOCKER_PLUGIN_DIR}:/usr/local/lib/docker/cli-plugins:ro" - "${CLP_DOCKER_SOCK_PATH}:/var/run/docker.sock" - "${CLP_HOME}:${CLP_HOME}" + - "/:/mnt/host" - "/usr/bin/docker:/usr/bin/docker:ro" From 3b4d44c212bbb8a311b0f676daf5ae1749564366 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 28 Oct 2025 21:01:24 -0400 Subject: [PATCH 07/50] lint --- .../clp_package_utils/controller.py | 2 +- .../clp_package_utils/general.py | 2 +- .../clp_package_utils/scripts/archive_manager.py | 2 +- .../clp_package_utils/scripts/compress.py | 10 +++------- .../scripts/compress_from_s3.py | 14 ++++---------- .../clp_package_utils/scripts/decompress.py | 16 ++++++++-------- .../clp_package_utils/scripts/search.py | 6 ++---- .../clp_package_utils/scripts/start_clp.py | 12 +++++++----- 8 files changed, 27 insertions(+), 37 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index a2645538f5..91ff02ab38 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -37,6 +37,7 @@ get_datasets_table_name, get_files_table_name, ) +from clp_py_utils.core import resolve_host_path from clp_package_utils.general import ( check_docker_dependencies, @@ -54,7 +55,6 @@ validate_results_cache_config, validate_webui_config, ) -from clp_py_utils.core import resolve_host_path LOG_FILE_ACCESS_MODE = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 21b2fa0e43..32668eec34 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -30,7 +30,6 @@ WEBUI_COMPONENT_NAME, WorkerConfig, ) -from clp_py_utils.core import resolve_host_path from clp_py_utils.clp_metadata_db_utils import ( MYSQL_TABLE_NAME_MAX_LEN, TABLE_SUFFIX_MAX_LEN, @@ -39,6 +38,7 @@ get_config_value, make_config_path_absolute, read_yaml_config_file, + resolve_host_path, validate_path_could_be_dir, ) from strenum import KebabCaseStrEnum diff --git a/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py b/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py index 6ba9881c48..af64c4aeda 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py +++ b/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py @@ -14,6 +14,7 @@ StorageEngine, StorageType, ) +from clp_py_utils.core import resolve_host_path from clp_package_utils.general import ( CLPConfig, @@ -28,7 +29,6 @@ validate_and_load_db_credentials_file, validate_dataset_name, ) -from clp_py_utils.core import resolve_host_path # Command/Argument Constants FIND_COMMAND: Final[str] = "find" diff --git a/components/clp-package-utils/clp_package_utils/scripts/compress.py b/components/clp-package-utils/clp_package_utils/scripts/compress.py index c2bdc89ec9..5ba7a73f92 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/compress.py +++ b/components/clp-package-utils/clp_package_utils/scripts/compress.py @@ -15,6 +15,7 @@ StorageEngine, StorageType, ) +from clp_py_utils.core import resolve_host_path from clp_package_utils.general import ( CONTAINER_INPUT_LOGS_ROOT_DIR, @@ -29,7 +30,6 @@ validate_and_load_db_credentials_file, validate_dataset_name, ) -from clp_py_utils.core import resolve_host_path logger = logging.getLogger(__name__) @@ -57,9 +57,7 @@ def _generate_logs_list( return len(parsed_args.paths) != 0 no_path_found = True - resolved_host_logs_list_path = resolve_host_path( - pathlib.Path(host_logs_list_path) - ) + resolved_host_logs_list_path = resolve_host_path(pathlib.Path(host_logs_list_path)) with open(resolved_host_logs_list_path, "r") as host_logs_list_file: for line in host_logs_list_file: stripped_path_str = line.rstrip() @@ -242,9 +240,7 @@ def main(argv): logs_list_filename = f"{uuid.uuid4()}.txt" logs_list_path_on_host = clp_config.logs_directory / logs_list_filename resolved_logs_list_path_on_host = resolve_host_path(logs_list_path_on_host) - logs_list_path_on_container = ( - container_clp_config.logs_directory / logs_list_filename - ) + logs_list_path_on_container = container_clp_config.logs_directory / logs_list_filename if not resolved_logs_list_path_on_host.exists(): break diff --git a/components/clp-package-utils/clp_package_utils/scripts/compress_from_s3.py b/components/clp-package-utils/clp_package_utils/scripts/compress_from_s3.py index a50a48be00..1abf2a060c 100644 --- a/components/clp-package-utils/clp_package_utils/scripts/compress_from_s3.py +++ b/components/clp-package-utils/clp_package_utils/scripts/compress_from_s3.py @@ -14,6 +14,7 @@ StorageEngine, StorageType, ) +from clp_py_utils.core import resolve_host_path from clp_package_utils.general import ( dump_container_config, @@ -29,7 +30,6 @@ validate_and_load_db_credentials_file, validate_dataset_name, ) -from clp_py_utils.core import resolve_host_path logger = logging.getLogger(__name__) @@ -57,9 +57,7 @@ def _generate_url_list( return len(parsed_args.inputs) != 0 no_url_found = True - resolved_inputs_from_path = resolve_host_path( - pathlib.Path(parsed_args.inputs_from) - ) + resolved_inputs_from_path = resolve_host_path(pathlib.Path(parsed_args.inputs_from)) with open(resolved_inputs_from_path, "r") as input_file: for line in input_file: stripped_url = line.strip() @@ -296,9 +294,7 @@ def main(argv): url_list_filename = f"{uuid.uuid4()}.txt" url_list_path_on_host = clp_config.logs_directory / url_list_filename resolved_url_list_path_on_host = resolve_host_path(url_list_path_on_host) - url_list_path_on_container = ( - container_clp_config.logs_directory / url_list_filename - ) + url_list_path_on_container = container_clp_config.logs_directory / url_list_filename if not resolved_url_list_path_on_host.exists(): break @@ -327,9 +323,7 @@ def main(argv): else: resolved_url_list_path_on_host.unlink() - resolved_generated_config_path_on_host = resolve_host_path( - generated_config_path_on_host - ) + resolved_generated_config_path_on_host = resolve_host_path(generated_config_path_on_host) resolved_generated_config_path_on_host.unlink() return ret_code diff --git a/components/clp-package-utils/clp_package_utils/scripts/decompress.py b/components/clp-package-utils/clp_package_utils/scripts/decompress.py index b44b90270c..e86eeef46e 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/decompress.py +++ b/components/clp-package-utils/clp_package_utils/scripts/decompress.py @@ -16,6 +16,7 @@ StorageEngine, StorageType, ) +from clp_py_utils.core import resolve_host_path from clp_package_utils.general import ( DockerMount, @@ -35,7 +36,6 @@ validate_dataset_name, validate_path_could_be_dir, ) -from clp_py_utils.core import resolve_host_path logger = logging.getLogger(__file__) @@ -163,9 +163,7 @@ def handle_extract_file_cmd( logger.debug(f"Docker command failed: {shlex.join(cmd)}") # Remove generated files - resolved_generated_config_path_on_host = resolve_host_path( - generated_config_path_on_host - ) + resolved_generated_config_path_on_host = resolve_host_path(generated_config_path_on_host) resolved_generated_config_path_on_host.unlink() return ret_code @@ -271,9 +269,7 @@ def handle_extract_stream_cmd( logger.debug(f"Docker command failed: {shlex.join(cmd)}") # Remove generated files - resolved_generated_config_path_on_host = resolve_host_path( - generated_config_path_on_host - ) + resolved_generated_config_path_on_host = resolve_host_path(generated_config_path_on_host) resolved_generated_config_path_on_host.unlink() return ret_code @@ -308,7 +304,11 @@ def main(argv): ) default_extraction_dir = pathlib.Path(os.environ.get("CLP_PWD_HOST", ".")) file_extraction_parser.add_argument( - "-d", "--extraction-dir", metavar="DIR", default=default_extraction_dir, help="Extract files into DIR." + "-d", + "--extraction-dir", + metavar="DIR", + default=default_extraction_dir, + help="Extract files into DIR.", ) # IR extraction command parser diff --git a/components/clp-package-utils/clp_package_utils/scripts/search.py b/components/clp-package-utils/clp_package_utils/scripts/search.py index 42c57af901..bba5364db2 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/search.py +++ b/components/clp-package-utils/clp_package_utils/scripts/search.py @@ -13,6 +13,7 @@ StorageEngine, StorageType, ) +from clp_py_utils.core import resolve_host_path from clp_package_utils.general import ( dump_container_config, @@ -26,7 +27,6 @@ validate_and_load_db_credentials_file, validate_dataset_name, ) -from clp_py_utils.core import resolve_host_path logger = logging.getLogger(__file__) @@ -181,9 +181,7 @@ def main(argv): logger.debug(f"Docker command failed: {shlex.join(cmd)}") # Remove generated files - resolved_generated_config_path_on_host = resolve_host_path( - generated_config_path_on_host - ) + resolved_generated_config_path_on_host = resolve_host_path(generated_config_path_on_host) resolved_generated_config_path_on_host.unlink() return ret_code diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 633d52ec1c..5e76bdca74 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -4,6 +4,7 @@ import sys from clp_py_utils.clp_config import CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH +from clp_py_utils.core import resolve_host_path from clp_package_utils.controller import DockerComposeController, get_or_create_instance_id from clp_package_utils.general import ( @@ -16,7 +17,6 @@ validate_output_storage_config, validate_retention_config, ) -from clp_py_utils.core import resolve_host_path logger = logging.getLogger(__file__) @@ -76,10 +76,12 @@ def main(argv): resolve_host_path(clp_config.data_directory).mkdir(parents=True, exist_ok=True) resolve_host_path(clp_config.logs_directory).mkdir(parents=True, exist_ok=True) resolve_host_path(clp_config.tmp_directory).mkdir(parents=True, exist_ok=True) - resolve_host_path(clp_config.archive_output.get_directory()).mkdir(parents=True, - exist_ok=True) - resolve_host_path(clp_config.stream_output.get_directory()).mkdir(parents=True, - exist_ok=True) + resolve_host_path(clp_config.archive_output.get_directory()).mkdir( + parents=True, exist_ok=True + ) + resolve_host_path(clp_config.stream_output.get_directory()).mkdir( + parents=True, exist_ok=True + ) except: logger.exception("Failed to create necessary directories.") return -1 From 74a392106152afc7f723e88c4de334c725e0a8b9 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 28 Oct 2025 21:06:08 -0400 Subject: [PATCH 08/50] refactor(utils): Simplify path validation by removing host mount parameter; unify with resolve_host_path. --- .../clp-package-utils/clp_package_utils/general.py | 4 ++-- .../clp_package_utils/scripts/decompress.py | 4 ++-- components/clp-py-utils/clp_py_utils/clp_config.py | 10 +++++----- components/clp-py-utils/clp_py_utils/core.py | 4 ++-- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 32668eec34..6662092259 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -773,7 +773,7 @@ def _is_docker_compose_project_running(project_name: str) -> bool: def _validate_data_directory(data_dir: pathlib.Path, component_name: str) -> None: try: - validate_path_could_be_dir(data_dir, True) + validate_path_could_be_dir(resolve_host_path(data_dir)) except ValueError as ex: raise ValueError(f"{component_name} data directory is invalid: {ex}") @@ -787,6 +787,6 @@ def _validate_log_directory(logs_dir: pathlib.Path, component_name: str): :raise ValueError: If the path is invalid or can't be a directory. """ try: - validate_path_could_be_dir(logs_dir, True) + validate_path_could_be_dir(resolve_host_path(logs_dir)) except ValueError as ex: raise ValueError(f"{component_name} logs directory is invalid: {ex}") diff --git a/components/clp-package-utils/clp_package_utils/scripts/decompress.py b/components/clp-package-utils/clp_package_utils/scripts/decompress.py index e86eeef46e..ffd8516af8 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/decompress.py +++ b/components/clp-package-utils/clp_package_utils/scripts/decompress.py @@ -80,8 +80,9 @@ def handle_extract_file_cmd( # Validate extraction directory extraction_dir = pathlib.Path(parsed_args.extraction_dir).resolve() + resolved_extraction_dir = resolve_host_path(extraction_dir) try: - validate_path_could_be_dir(extraction_dir, True) + validate_path_could_be_dir(resolved_extraction_dir) except ValueError as ex: logger.error(f"extraction-dir is invalid: {ex}") return -1 @@ -109,7 +110,6 @@ def handle_extract_file_cmd( ) # Set up mounts - resolved_extraction_dir = resolve_host_path(extraction_dir) resolved_extraction_dir.mkdir(exist_ok=True) container_extraction_dir = pathlib.Path("/") / "mnt" / "extraction-dir" necessary_mounts = [ diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 0b152124a1..d605830aad 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -694,7 +694,7 @@ def validate_archive_output_config(self): f" = '{StorageEngine.CLP_S}'" ) try: - validate_path_could_be_dir(self.archive_output.get_directory(), True) + validate_path_could_be_dir(resolve_host_path(self.archive_output.get_directory())) except ValueError as ex: raise ValueError(f"archive_output.storage's directory is invalid: {ex}") @@ -708,25 +708,25 @@ def validate_stream_output_config(self): f" = '{StorageEngine.CLP_S}'" ) try: - validate_path_could_be_dir(self.stream_output.get_directory(), True) + validate_path_could_be_dir(resolve_host_path(self.stream_output.get_directory())) except ValueError as ex: raise ValueError(f"stream_output.storage's directory is invalid: {ex}") def validate_data_dir(self): try: - validate_path_could_be_dir(self.data_directory, True) + validate_path_could_be_dir(resolve_host_path(self.data_directory)) except ValueError as ex: raise ValueError(f"data_directory is invalid: {ex}") def validate_logs_dir(self): try: - validate_path_could_be_dir(self.logs_directory, True) + validate_path_could_be_dir(resolve_host_path(self.logs_directory)) except ValueError as ex: raise ValueError(f"logs_directory is invalid: {ex}") def validate_tmp_dir(self): try: - validate_path_could_be_dir(self.tmp_directory, True) + validate_path_could_be_dir(resolve_host_path(self.tmp_directory)) except ValueError as ex: raise ValueError(f"tmp_directory is invalid: {ex}") diff --git a/components/clp-py-utils/clp_py_utils/core.py b/components/clp-py-utils/clp_py_utils/core.py index aed5b125df..578f91f478 100644 --- a/components/clp-py-utils/clp_py_utils/core.py +++ b/components/clp-py-utils/clp_py_utils/core.py @@ -85,8 +85,8 @@ def resolve_host_path(path: pathlib.Path) -> pathlib.Path: return resolved -def validate_path_could_be_dir(path: pathlib.Path, use_host_mount: bool): - part = resolve_host_path(path) if use_host_mount else path +def validate_path_could_be_dir(path: pathlib.Path): + part = path while True: if part.exists(): if not part.is_dir(): From 885e9b55c7804924c78751de209a75cd57a728bf Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 28 Oct 2025 21:08:03 -0400 Subject: [PATCH 09/50] revert unrelated change --- .../clp-package-utils/clp_package_utils/controller.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 91ff02ab38..7264737f01 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -719,10 +719,8 @@ def set_up_env(self) -> None: env_vars["CLP_LOGS_INPUT_DIR_HOST"] = str(self._clp_config.logs_input.directory) # Output config - archive_output_dir = self._clp_config.archive_output.get_directory() - stream_output_dir = self._clp_config.stream_output.get_directory() - archive_output_dir_str = str(archive_output_dir) - stream_output_dir_str = str(stream_output_dir) + archive_output_dir_str = str(self._clp_config.archive_output.get_directory()) + stream_output_dir_str = str(self._clp_config.stream_output.get_directory()) if self._clp_config.archive_output.storage.type == StorageType.FS: env_vars["CLP_ARCHIVE_OUTPUT_DIR_HOST"] = archive_output_dir_str if self._clp_config.archive_output.storage.type == StorageType.S3: From 0db2f30261c5a219c5cd7f18e8231e5a9ea5ee1b Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 29 Oct 2025 00:32:55 -0400 Subject: [PATCH 10/50] fix package docker workflow --- .github/workflows/clp-artifact-build.yaml | 2 +- taskfile.yaml | 8 ++++++++ taskfiles/docker-images.yaml | 6 +----- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/clp-artifact-build.yaml b/.github/workflows/clp-artifact-build.yaml index 9cbb0b6c4b..3fcd59cb54 100644 --- a/.github/workflows/clp-artifact-build.yaml +++ b/.github/workflows/clp-artifact-build.yaml @@ -548,7 +548,7 @@ jobs: ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} run_command: >- - CLP_CPP_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task package + CLP_CPP_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task package-build-deps - uses: "./.github/actions/clp-build-runtime-image" with: diff --git a/taskfile.yaml b/taskfile.yaml index 56525e49e9..22857e0bff 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -212,6 +212,14 @@ tasks: vars: COMPONENT: "{{.TASK}}" + package-build-deps: + deps: + - "init" + - "core" + - "deps:spider" + - "python-libs" + - "webui" + webui: vars: CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK}}.md5" diff --git a/taskfiles/docker-images.yaml b/taskfiles/docker-images.yaml index 488b943ff6..1177075d06 100644 --- a/taskfiles/docker-images.yaml +++ b/taskfiles/docker-images.yaml @@ -4,10 +4,6 @@ tasks: package: dir: "{{.ROOT_DIR}}/tools/docker-images/clp-package" deps: - - ":init" - - ":core" - - ":deps:spider" - - ":python-libs" - - ":webui" + - ":package-build-deps" cmds: - "./build.sh" From 695ed779c73222c791349c3019b1a37b7e213451 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 29 Oct 2025 00:39:26 -0400 Subject: [PATCH 11/50] lint --- .github/workflows/clp-artifact-build.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/clp-artifact-build.yaml b/.github/workflows/clp-artifact-build.yaml index 3fcd59cb54..63184289e5 100644 --- a/.github/workflows/clp-artifact-build.yaml +++ b/.github/workflows/clp-artifact-build.yaml @@ -548,7 +548,8 @@ jobs: ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} run_command: >- - CLP_CPP_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task package-build-deps + CLP_CPP_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) + task package-build-deps - uses: "./.github/actions/clp-build-runtime-image" with: From 4f21e0f8056af5f700f0294d61964f2f52e09da3 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 29 Oct 2025 00:40:59 -0400 Subject: [PATCH 12/50] docs(dev-docs): Update task command name for building packages. --- docs/src/dev-docs/building-package.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/dev-docs/building-package.md b/docs/src/dev-docs/building-package.md index 4149366a12..8275af1ef9 100755 --- a/docs/src/dev-docs/building-package.md +++ b/docs/src/dev-docs/building-package.md @@ -57,8 +57,8 @@ task The build will be in `build/clp-package` and defaults to using the storage engine for `clp-text`. :::{note} -The `task` command runs `task docker-images:package` under the hood. In addition to the build, a -Docker image named `clp-package:dev--` will also be created. +The `task` command runs `task package` under the hood. In addition to the build, a Docker image +named `clp-package:dev--` will also be created. ::: :::{note} From a7c81a3f086eae863c9c7c278816f449b1079fd3 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 29 Oct 2025 00:43:54 -0400 Subject: [PATCH 13/50] docs: Remove Python section and streamline Docker Desktop requirements --- docs/src/user-docs/quick-start/index.md | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/docs/src/user-docs/quick-start/index.md b/docs/src/user-docs/quick-start/index.md index faf814c0ba..1d1d5b6f96 100644 --- a/docs/src/user-docs/quick-start/index.md +++ b/docs/src/user-docs/quick-start/index.md @@ -17,7 +17,6 @@ To run a CLP release, you'll need: * `docker-ce` >= 27.0.3 * `docker-ce-cli` >= 27.0.3 * `docker-compose-plugin` >= 2.28.1 -* [Python](#python) ### Docker @@ -33,19 +32,7 @@ NOTE: * If you're not running as root, ensure Docker can be run [without superuser privileges][docker-non-root]. -* If you're using Docker Desktop, ensure version 4.34 or higher is installed, and - [host networking is enabled][docker-desktop-host-networking]. - -### Python - -To check whether Python is installed on your system, run: - -```bash -python3 --version -``` - -CLP requires Python 3.10 or higher. If Python isn't installed, or if the version isn't high enough, -install or upgrade it by following the instructions for your OS. +* If you're using Docker Desktop, ensure version 4.34 or higher is installed. --- @@ -143,5 +130,4 @@ How to compress and search unstructured text logs. [clp-releases]: https://github.com/y-scope/clp/releases [Docker]: https://docs.docker.com/engine/install/ -[docker-desktop-host-networking]: https://docs.docker.com/engine/network/drivers/host/#docker-desktop [docker-non-root]: https://docs.docker.com/engine/install/linux-postinstall/#manage-docker-as-a-non-root-user From be2d20046fd7e95f56ab6335c76e4334096f6e70 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 29 Oct 2025 02:21:03 -0400 Subject: [PATCH 14/50] refactor(clp-py-utils): Remove redundant debug print in config directory validation. --- components/clp-py-utils/clp_py_utils/clp_config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index d605830aad..99456c8745 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -752,7 +752,6 @@ def validate_aws_config_dir(self): "aws_config_directory must be set when using profile authentication" ) if not resolve_host_path(self.aws_config_directory).exists(): - print(resolve_host_path(self.aws_config_directory)) raise ValueError( f"aws_config_directory does not exist: '{self.aws_config_directory}'" ) From d910944c5c563f3226d216ef17abcc7fa51b34d2 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 29 Oct 2025 02:28:26 -0400 Subject: [PATCH 15/50] chore(scripts): Add shellcheck directive comments for sourcing common environment scripts. --- .../package-template/src/sbin/admin-tools/archive-manager.sh | 2 ++ .../package-template/src/sbin/admin-tools/dataset-manager.sh | 2 ++ components/package-template/src/sbin/compress-from-s3.sh | 2 ++ components/package-template/src/sbin/compress.sh | 2 ++ components/package-template/src/sbin/decompress.sh | 2 ++ components/package-template/src/sbin/search.sh | 2 ++ components/package-template/src/sbin/start-clp.sh | 2 ++ components/package-template/src/sbin/stop-clp.sh | 2 ++ 8 files changed, 16 insertions(+) diff --git a/components/package-template/src/sbin/admin-tools/archive-manager.sh b/components/package-template/src/sbin/admin-tools/archive-manager.sh index 3949330e33..afa2acdd25 100755 --- a/components/package-template/src/sbin/admin-tools/archive-manager.sh +++ b/components/package-template/src/sbin/admin-tools/archive-manager.sh @@ -2,6 +2,8 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" common_env_path="$script_dir/../.common-env.sh" + +# shellcheck source=.common-env.sh source "$common_env_path" docker compose -f "$CLP_HOME/docker-compose.runtime.yaml" run --rm clp-runtime \ diff --git a/components/package-template/src/sbin/admin-tools/dataset-manager.sh b/components/package-template/src/sbin/admin-tools/dataset-manager.sh index 281b44f72b..668e4e5373 100755 --- a/components/package-template/src/sbin/admin-tools/dataset-manager.sh +++ b/components/package-template/src/sbin/admin-tools/dataset-manager.sh @@ -2,6 +2,8 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" common_env_path="$script_dir/../.common-env.sh" + +# shellcheck source=.common-env.sh source "$common_env_path" docker compose -f "$CLP_HOME/docker-compose.runtime.yaml" run --rm clp-runtime \ diff --git a/components/package-template/src/sbin/compress-from-s3.sh b/components/package-template/src/sbin/compress-from-s3.sh index 8aede3a885..8cb10e594b 100755 --- a/components/package-template/src/sbin/compress-from-s3.sh +++ b/components/package-template/src/sbin/compress-from-s3.sh @@ -2,6 +2,8 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" common_env_path="$script_dir/.common-env.sh" + +# shellcheck source=.common-env.sh source "$common_env_path" docker compose -f "$CLP_HOME/docker-compose.runtime.yaml" run --rm clp-runtime \ diff --git a/components/package-template/src/sbin/compress.sh b/components/package-template/src/sbin/compress.sh index 1cc1b61cf5..797255e2e6 100755 --- a/components/package-template/src/sbin/compress.sh +++ b/components/package-template/src/sbin/compress.sh @@ -2,6 +2,8 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" common_env_path="$script_dir/.common-env.sh" + +# shellcheck source=.common-env.sh source "$common_env_path" docker compose -f "$CLP_HOME/docker-compose.runtime.yaml" run --rm clp-runtime \ diff --git a/components/package-template/src/sbin/decompress.sh b/components/package-template/src/sbin/decompress.sh index 046398e818..10cf529d29 100755 --- a/components/package-template/src/sbin/decompress.sh +++ b/components/package-template/src/sbin/decompress.sh @@ -2,6 +2,8 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" common_env_path="$script_dir/.common-env.sh" + +# shellcheck source=.common-env.sh source "$common_env_path" docker compose -f "$CLP_HOME/docker-compose.runtime.yaml" run --rm clp-runtime \ diff --git a/components/package-template/src/sbin/search.sh b/components/package-template/src/sbin/search.sh index 87af94ffd5..1ad1d7efa6 100755 --- a/components/package-template/src/sbin/search.sh +++ b/components/package-template/src/sbin/search.sh @@ -2,6 +2,8 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" common_env_path="$script_dir/.common-env.sh" + +# shellcheck source=.common-env.sh source "$common_env_path" docker compose -f "$CLP_HOME/docker-compose.runtime.yaml" run --rm clp-runtime \ diff --git a/components/package-template/src/sbin/start-clp.sh b/components/package-template/src/sbin/start-clp.sh index 71c3121419..bc2e360508 100755 --- a/components/package-template/src/sbin/start-clp.sh +++ b/components/package-template/src/sbin/start-clp.sh @@ -2,6 +2,8 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" common_env_path="$script_dir/.common-env.sh" + +# shellcheck source=.common-env.sh source "$common_env_path" docker compose -f "$CLP_HOME/docker-compose.runtime.yaml" run --rm clp-runtime \ diff --git a/components/package-template/src/sbin/stop-clp.sh b/components/package-template/src/sbin/stop-clp.sh index fcc3d68553..aa582f84c3 100755 --- a/components/package-template/src/sbin/stop-clp.sh +++ b/components/package-template/src/sbin/stop-clp.sh @@ -2,6 +2,8 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" common_env_path="$script_dir/.common-env.sh" + +# shellcheck source=.common-env.sh source "$common_env_path" docker compose -f "$CLP_HOME/docker-compose.runtime.yaml" run --rm clp-runtime \ From 163921b113efb84893c70a220f90395962a6f9db Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 29 Oct 2025 02:32:37 -0400 Subject: [PATCH 16/50] fix SC2155 --- components/package-template/src/sbin/.common-env.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/package-template/src/sbin/.common-env.sh b/components/package-template/src/sbin/.common-env.sh index 9a997383be..83c58f0ed1 100755 --- a/components/package-template/src/sbin/.common-env.sh +++ b/components/package-template/src/sbin/.common-env.sh @@ -27,7 +27,8 @@ uid="$(id --user 2>/dev/null || echo "1000")" gid="$(getent group docker | cut -d: -f3 2>/dev/null || echo "999")" export CLP_FIRST_PARTY_SERVICE_UID_GID="$uid:$gid" -export CLP_PWD_HOST="$(pwd)" +CLP_PWD_HOST="$(pwd 2>/dev/null || echo "")" +export CLP_PWD_HOST if [[ -z "${CLP_DOCKER_PLUGIN_DIR:-}" ]]; then for dir in \ From 3f5c30251e1f7ca285199fec0fc396d1767c2490 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 29 Oct 2025 02:34:29 -0400 Subject: [PATCH 17/50] fix(scripts): Ensure CLP_DOCKER_SOCK_PATH is set only when undefined. --- components/package-template/src/sbin/.common-env.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/components/package-template/src/sbin/.common-env.sh b/components/package-template/src/sbin/.common-env.sh index 83c58f0ed1..4052429dd7 100755 --- a/components/package-template/src/sbin/.common-env.sh +++ b/components/package-template/src/sbin/.common-env.sh @@ -48,7 +48,9 @@ if [[ -z "${CLP_DOCKER_PLUGIN_DIR:-}" ]]; then fi fi -socket="$(docker context inspect --format '{{.Endpoints.docker.Host}}' 2>/dev/null | sed -E 's|^unix://||')" -if [[ -S "$socket" ]]; then - export CLP_DOCKER_SOCK_PATH="$socket" +if [[ -z "${CLP_DOCKER_SOCK_PATH:-}" ]]; then + socket="$(docker context inspect --format '{{.Endpoints.docker.Host}}' 2>/dev/null | sed -E 's|^unix://||')" + if [[ -S "$socket" ]]; then + export CLP_DOCKER_SOCK_PATH="$socket" + fi fi From 65d421f956905be52db99a8696a87b1a7795615c Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 29 Oct 2025 02:36:18 -0400 Subject: [PATCH 18/50] fix(init): invert CLP_HOME check to use -z for unset detection --- components/package-template/src/sbin/.common-env.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/package-template/src/sbin/.common-env.sh b/components/package-template/src/sbin/.common-env.sh index 4052429dd7..83c1ae6ff7 100755 --- a/components/package-template/src/sbin/.common-env.sh +++ b/components/package-template/src/sbin/.common-env.sh @@ -3,10 +3,10 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" package_root=$(readlink -f "$script_dir/..") -if [[ -n "${CLP_HOME:-}" ]]; then - export CLP_HOME="$CLP_HOME" -else +if [[ -z "${CLP_HOME:-}" ]]; then export CLP_HOME="$package_root" +else + export CLP_HOME="$CLP_HOME" fi image_id_file="$CLP_HOME/clp-package-image.id" From 83c581c50214dfa64c3a76d14f41ca5e08284fb6 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 29 Oct 2025 02:39:57 -0400 Subject: [PATCH 19/50] fix(scripts): Remove exit 1; Remove redundant stderr redirection in error handling --- components/package-template/src/sbin/.common-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/package-template/src/sbin/.common-env.sh b/components/package-template/src/sbin/.common-env.sh index 83c1ae6ff7..5bfd517c7a 100755 --- a/components/package-template/src/sbin/.common-env.sh +++ b/components/package-template/src/sbin/.common-env.sh @@ -20,7 +20,7 @@ elif [[ -f "$version_file" ]]; then export CLP_PACKAGE_CONTAINER_IMAGE_REF="ghcr.io/y-scope/clp/clp-package:v$version" else echo "Error: Neither clp-package-image.id nor VERSION file exists." >&2 - return 1 2>/dev/null || exit 1 + return 1 fi uid="$(id --user 2>/dev/null || echo "1000")" From e03a36162cd2960458dc5433f53c794e35ebaba6 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 29 Oct 2025 02:40:48 -0400 Subject: [PATCH 20/50] chore(clp-py-utils): Remove unused os import from core module. --- components/clp-py-utils/clp_py_utils/core.py | 1 - 1 file changed, 1 deletion(-) diff --git a/components/clp-py-utils/clp_py_utils/core.py b/components/clp-py-utils/clp_py_utils/core.py index 578f91f478..13e49e2385 100644 --- a/components/clp-py-utils/clp_py_utils/core.py +++ b/components/clp-py-utils/clp_py_utils/core.py @@ -1,4 +1,3 @@ -import os import pathlib import yaml From 299795f955cab69d0f7e742b83ad4197c6a3a052 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 29 Oct 2025 02:43:20 -0400 Subject: [PATCH 21/50] Update volume definitions in docker-compose.runtime.yaml to use the extended syntax --- .../package/docker-compose.runtime.yaml | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/tools/deployment/package/docker-compose.runtime.yaml b/tools/deployment/package/docker-compose.runtime.yaml index 6895f85ba7..3fc3576f42 100644 --- a/tools/deployment/package/docker-compose.runtime.yaml +++ b/tools/deployment/package/docker-compose.runtime.yaml @@ -12,8 +12,20 @@ services: CLP_HOME: "${CLP_HOME}" CLP_PWD_HOST: "${CLP_PWD_HOST:-${PWD}}" volumes: - - "${CLP_DOCKER_PLUGIN_DIR}:/usr/local/lib/docker/cli-plugins:ro" - - "${CLP_DOCKER_SOCK_PATH}:/var/run/docker.sock" - - "${CLP_HOME}:${CLP_HOME}" - - "/:/mnt/host" - - "/usr/bin/docker:/usr/bin/docker:ro" + - type: "bind" + source: "${CLP_DOCKER_PLUGIN_DIR}" + target: "/usr/local/lib/docker/cli-plugins" + read_only: true + - type: "bind" + source: "${CLP_DOCKER_SOCK_PATH}" + target: "/var/run/docker.sock" + - type: "bind" + source: "${CLP_HOME}" + target: "${CLP_HOME}" + - type: "bind" + source: "/" + target: "/mnt/host" + - type: "bind" + source: "/usr/bin/docker" + target: "/usr/bin/docker" + read_only: true From 0ccd1553949dde39dd935c8d13ee918d44ff79a9 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 29 Oct 2025 02:44:33 -0400 Subject: [PATCH 22/50] fix(deployment): Add default values and validations in docker-compose.runtime.yaml. --- tools/deployment/package/docker-compose.runtime.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/deployment/package/docker-compose.runtime.yaml b/tools/deployment/package/docker-compose.runtime.yaml index 3fc3576f42..c2e478db29 100644 --- a/tools/deployment/package/docker-compose.runtime.yaml +++ b/tools/deployment/package/docker-compose.runtime.yaml @@ -8,20 +8,20 @@ services: tty: true user: "${CLP_FIRST_PARTY_SERVICE_UID_GID:-1000:999}" environment: - HOME: "${HOME}" - CLP_HOME: "${CLP_HOME}" + HOME: "${HOME:?Please set a value.}" + CLP_HOME: "${CLP_HOME:?Please set a value.}" CLP_PWD_HOST: "${CLP_PWD_HOST:-${PWD}}" volumes: - type: "bind" - source: "${CLP_DOCKER_PLUGIN_DIR}" + source: "${CLP_DOCKER_PLUGIN_DIR:-/usr/local/lib/docker/cli-plugins}" target: "/usr/local/lib/docker/cli-plugins" read_only: true - type: "bind" - source: "${CLP_DOCKER_SOCK_PATH}" + source: "${CLP_DOCKER_SOCK_PATH:-/var/run/docker.sock}" target: "/var/run/docker.sock" - type: "bind" - source: "${CLP_HOME}" - target: "${CLP_HOME}" + source: "${CLP_HOME:?Please set a value.}" + target: "${CLP_HOME:?Please set a value.}" - type: "bind" source: "/" target: "/mnt/host" From beacdcf05ca108854d1a2d3b7ba1c9c4596761b0 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 29 Oct 2025 02:47:55 -0400 Subject: [PATCH 23/50] fix(scripts): Adjust formatting for better readability in .common-env.sh. --- .../package-template/src/sbin/.common-env.sh | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/components/package-template/src/sbin/.common-env.sh b/components/package-template/src/sbin/.common-env.sh index 5bfd517c7a..82c9a38c43 100755 --- a/components/package-template/src/sbin/.common-env.sh +++ b/components/package-template/src/sbin/.common-env.sh @@ -44,13 +44,17 @@ if [[ -z "${CLP_DOCKER_PLUGIN_DIR:-}" ]]; then fi done if [[ -z "${CLP_DOCKER_PLUGIN_DIR:-}" ]]; then - echo "Warning: Docker plugin directory not found; Docker Compose may not work inside container." >&2 + echo "Warning: Docker plugin directory not found;" \ + "Docker Compose may not work inside container." >&2 fi fi if [[ -z "${CLP_DOCKER_SOCK_PATH:-}" ]]; then - socket="$(docker context inspect --format '{{.Endpoints.docker.Host}}' 2>/dev/null | sed -E 's|^unix://||')" - if [[ -S "$socket" ]]; then - export CLP_DOCKER_SOCK_PATH="$socket" - fi + socket="$(docker context inspect \ + --format '{{.Endpoints.docker.Host}}' 2>/dev/null \ + | sed -E 's|^unix://||')" + + if [[ -S "$socket" ]]; then + export CLP_DOCKER_SOCK_PATH="$socket" + fi fi From c7141e73cc8cffad6bbb2240d7d638192fc67d0d Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 29 Oct 2025 02:48:45 -0400 Subject: [PATCH 24/50] move `>&2` before echo --- components/package-template/src/sbin/.common-env.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/package-template/src/sbin/.common-env.sh b/components/package-template/src/sbin/.common-env.sh index 82c9a38c43..3d858e49c3 100755 --- a/components/package-template/src/sbin/.common-env.sh +++ b/components/package-template/src/sbin/.common-env.sh @@ -19,7 +19,7 @@ elif [[ -f "$version_file" ]]; then version="$(tr -d '[:space:]' < "$version_file")" export CLP_PACKAGE_CONTAINER_IMAGE_REF="ghcr.io/y-scope/clp/clp-package:v$version" else - echo "Error: Neither clp-package-image.id nor VERSION file exists." >&2 + >&2 echo "Error: Neither clp-package-image.id nor VERSION file exists." return 1 fi @@ -44,8 +44,8 @@ if [[ -z "${CLP_DOCKER_PLUGIN_DIR:-}" ]]; then fi done if [[ -z "${CLP_DOCKER_PLUGIN_DIR:-}" ]]; then - echo "Warning: Docker plugin directory not found;" \ - "Docker Compose may not work inside container." >&2 + >&2 echo "Warning: Docker plugin directory not found;" \ + "Docker Compose may not work inside container." fi fi From 25c24508b973f59643229e6ceba6d3a288180a20 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 29 Oct 2025 03:12:30 -0400 Subject: [PATCH 25/50] fix(scripts): Resolve host paths for config files during load operations. --- components/clp-package-utils/clp_package_utils/general.py | 5 ++--- .../clp_package_utils/scripts/archive_manager.py | 4 +++- .../clp_package_utils/scripts/compress.py | 6 +++++- .../clp_package_utils/scripts/compress_from_s3.py | 6 +++++- .../clp_package_utils/scripts/dataset_manager.py | 6 +++++- .../clp_package_utils/scripts/decompress.py | 6 +++++- .../clp-package-utils/clp_package_utils/scripts/search.py | 6 +++++- .../clp_package_utils/scripts/start_clp.py | 6 +++++- .../clp_package_utils/scripts/stop_clp.py | 7 ++++++- 9 files changed, 41 insertions(+), 11 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 6662092259..21bcb6dbb4 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -429,9 +429,8 @@ def validate_config_key_existence(config, key): def load_config_file( config_file_path: pathlib.Path, default_config_file_path: pathlib.Path, clp_home: pathlib.Path ): - resolved_config_file_path = resolve_host_path(config_file_path) - if resolved_config_file_path.exists(): - raw_clp_config = read_yaml_config_file(resolved_config_file_path) + if config_file_path.exists(): + raw_clp_config = read_yaml_config_file(config_file_path) if raw_clp_config is None: clp_config = CLPConfig() else: diff --git a/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py b/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py index af64c4aeda..14a8aad5ed 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py +++ b/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py @@ -172,7 +172,9 @@ def main(argv: List[str]) -> int: try: config_file_path: Path = Path(parsed_args.config) clp_config: CLPConfig = load_config_file( - config_file_path, default_config_file_path, clp_home + resolve_host_path(config_file_path), + resolve_host_path(default_config_file_path), + clp_home, ) clp_config.validate_logs_dir() diff --git a/components/clp-package-utils/clp_package_utils/scripts/compress.py b/components/clp-package-utils/clp_package_utils/scripts/compress.py index 5ba7a73f92..bfa8c80483 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/compress.py +++ b/components/clp-package-utils/clp_package_utils/scripts/compress.py @@ -177,7 +177,11 @@ def main(argv): # Validate and load config file try: config_file_path = pathlib.Path(parsed_args.config) - clp_config = load_config_file(config_file_path, default_config_file_path, clp_home) + clp_config = load_config_file( + resolve_host_path(config_file_path), + resolve_host_path(default_config_file_path), + clp_home, + ) clp_config.validate_logs_dir() # Validate and load necessary credentials diff --git a/components/clp-package-utils/clp_package_utils/scripts/compress_from_s3.py b/components/clp-package-utils/clp_package_utils/scripts/compress_from_s3.py index 1abf2a060c..90205afb0f 100644 --- a/components/clp-package-utils/clp_package_utils/scripts/compress_from_s3.py +++ b/components/clp-package-utils/clp_package_utils/scripts/compress_from_s3.py @@ -224,7 +224,11 @@ def main(argv): try: config_file_path = pathlib.Path(parsed_args.config) - clp_config = load_config_file(config_file_path, default_config_file_path, clp_home) + clp_config = load_config_file( + resolve_host_path(config_file_path), + resolve_host_path(default_config_file_path), + clp_home, + ) clp_config.validate_logs_dir() validate_and_load_db_credentials_file(clp_config, clp_home, False) diff --git a/components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py b/components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py index f13d77698f..64771906ff 100644 --- a/components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py +++ b/components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py @@ -93,7 +93,11 @@ def main(argv: List[str]) -> int: # Validate and load config file try: config_file_path = Path(parsed_args.config) - clp_config = load_config_file(config_file_path, default_config_file_path, clp_home) + clp_config = load_config_file( + resolve_host_path(config_file_path), + resolve_host_path(default_config_file_path), + clp_home, + ) clp_config.validate_logs_dir() # Validate and load necessary credentials diff --git a/components/clp-package-utils/clp_package_utils/scripts/decompress.py b/components/clp-package-utils/clp_package_utils/scripts/decompress.py index ffd8516af8..f151423931 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/decompress.py +++ b/components/clp-package-utils/clp_package_utils/scripts/decompress.py @@ -53,7 +53,11 @@ def validate_and_load_config( :return: The config object on success, None otherwise. """ try: - clp_config = load_config_file(config_file_path, default_config_file_path, clp_home) + clp_config = load_config_file( + resolve_host_path(config_file_path), + resolve_host_path(default_config_file_path), + clp_home, + ) clp_config.validate_logs_dir() # Validate and load necessary credentials diff --git a/components/clp-package-utils/clp_package_utils/scripts/search.py b/components/clp-package-utils/clp_package_utils/scripts/search.py index bba5364db2..d83a43bda5 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/search.py +++ b/components/clp-package-utils/clp_package_utils/scripts/search.py @@ -92,7 +92,11 @@ def main(argv): # Validate and load config file try: config_file_path = pathlib.Path(parsed_args.config) - clp_config = load_config_file(config_file_path, default_config_file_path, clp_home) + clp_config = load_config_file( + resolve_host_path(config_file_path), + resolve_host_path(default_config_file_path), + clp_home, + ) clp_config.validate_logs_dir() # Validate and load necessary credentials diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 5e76bdca74..7d79bc8a5e 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -54,7 +54,11 @@ def main(argv): try: # Validate and load config file. config_file_path = pathlib.Path(parsed_args.config) - clp_config = load_config_file(config_file_path, default_config_file_path, clp_home) + clp_config = load_config_file( + resolve_host_path(config_file_path), + resolve_host_path(default_config_file_path), + clp_home, + ) validate_and_load_db_credentials_file(clp_config, clp_home, True) validate_and_load_queue_credentials_file(clp_config, clp_home, True) diff --git a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py index 105e929df7..d95c4e9aa0 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py @@ -4,6 +4,7 @@ import sys from clp_py_utils.clp_config import CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH +from clp_py_utils.core import resolve_host_path from clp_package_utils.controller import DockerComposeController, get_or_create_instance_id from clp_package_utils.general import ( @@ -30,7 +31,11 @@ def main(argv): try: config_file_path = pathlib.Path(parsed_args.config) - clp_config = load_config_file(config_file_path, default_config_file_path, clp_home) + clp_config = load_config_file( + resolve_host_path(config_file_path), + resolve_host_path(default_config_file_path), + clp_home, + ) except: logger.exception("Failed to load config.") return -1 From 0ac4501c28f0428f105804728370bee3f43be6eb Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 29 Oct 2025 03:43:17 -0400 Subject: [PATCH 26/50] fix(clp-config): Add `use_host_mount` parameter to validation methods and enable it only in the boostraping scripts --- .../clp_package_utils/general.py | 9 +---- .../scripts/archive_manager.py | 2 +- .../clp_package_utils/scripts/compress.py | 2 +- .../scripts/compress_from_s3.py | 2 +- .../scripts/dataset_manager.py | 2 +- .../clp_package_utils/scripts/decompress.py | 2 +- .../clp_package_utils/scripts/search.py | 2 +- .../clp_package_utils/scripts/start_clp.py | 11 +++--- .../clp-py-utils/clp_py_utils/clp_config.py | 39 ++++++++++++------- 9 files changed, 38 insertions(+), 33 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 21bcb6dbb4..759a76d5c8 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -570,14 +570,9 @@ def validate_results_cache_config( clp_config.results_cache.port, ) - -def validate_logs_input_config(clp_config: CLPConfig) -> None: - clp_config.validate_logs_input_config() - - def validate_output_storage_config(clp_config: CLPConfig) -> None: - clp_config.validate_archive_output_config() - clp_config.validate_stream_output_config() + clp_config.validate_archive_output_config(True) + clp_config.validate_stream_output_config(True) validate_path_for_container_mount(clp_config.archive_output.get_directory()) validate_path_for_container_mount(clp_config.stream_output.get_directory()) diff --git a/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py b/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py index 14a8aad5ed..45e0341f42 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py +++ b/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py @@ -176,7 +176,7 @@ def main(argv: List[str]) -> int: resolve_host_path(default_config_file_path), clp_home, ) - clp_config.validate_logs_dir() + clp_config.validate_logs_dir(True) # Validate and load necessary credentials validate_and_load_db_credentials_file(clp_config, clp_home, False) diff --git a/components/clp-package-utils/clp_package_utils/scripts/compress.py b/components/clp-package-utils/clp_package_utils/scripts/compress.py index bfa8c80483..1e98ab3b4b 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/compress.py +++ b/components/clp-package-utils/clp_package_utils/scripts/compress.py @@ -182,7 +182,7 @@ def main(argv): resolve_host_path(default_config_file_path), clp_home, ) - clp_config.validate_logs_dir() + clp_config.validate_logs_dir(True) # Validate and load necessary credentials validate_and_load_db_credentials_file(clp_config, clp_home, False) diff --git a/components/clp-package-utils/clp_package_utils/scripts/compress_from_s3.py b/components/clp-package-utils/clp_package_utils/scripts/compress_from_s3.py index 90205afb0f..f5faf78159 100644 --- a/components/clp-package-utils/clp_package_utils/scripts/compress_from_s3.py +++ b/components/clp-package-utils/clp_package_utils/scripts/compress_from_s3.py @@ -229,7 +229,7 @@ def main(argv): resolve_host_path(default_config_file_path), clp_home, ) - clp_config.validate_logs_dir() + clp_config.validate_logs_dir(True) validate_and_load_db_credentials_file(clp_config, clp_home, False) except Exception: diff --git a/components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py b/components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py index 64771906ff..f1c3111877 100644 --- a/components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py +++ b/components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py @@ -98,7 +98,7 @@ def main(argv: List[str]) -> int: resolve_host_path(default_config_file_path), clp_home, ) - clp_config.validate_logs_dir() + clp_config.validate_logs_dir(True) # Validate and load necessary credentials validate_and_load_db_credentials_file(clp_config, clp_home, False) diff --git a/components/clp-package-utils/clp_package_utils/scripts/decompress.py b/components/clp-package-utils/clp_package_utils/scripts/decompress.py index f151423931..f4204f0924 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/decompress.py +++ b/components/clp-package-utils/clp_package_utils/scripts/decompress.py @@ -58,7 +58,7 @@ def validate_and_load_config( resolve_host_path(default_config_file_path), clp_home, ) - clp_config.validate_logs_dir() + clp_config.validate_logs_dir(True) # Validate and load necessary credentials validate_and_load_db_credentials_file(clp_config, clp_home, False) diff --git a/components/clp-package-utils/clp_package_utils/scripts/search.py b/components/clp-package-utils/clp_package_utils/scripts/search.py index d83a43bda5..f8b299c791 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/search.py +++ b/components/clp-package-utils/clp_package_utils/scripts/search.py @@ -97,7 +97,7 @@ def main(argv): resolve_host_path(default_config_file_path), clp_home, ) - clp_config.validate_logs_dir() + clp_config.validate_logs_dir(True) # Validate and load necessary credentials validate_and_load_db_credentials_file(clp_config, clp_home, False) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 7d79bc8a5e..0b2a29fbf5 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -13,7 +13,6 @@ validate_and_load_db_credentials_file, validate_and_load_queue_credentials_file, validate_and_load_redis_credentials_file, - validate_logs_input_config, validate_output_storage_config, validate_retention_config, ) @@ -63,14 +62,14 @@ def main(argv): validate_and_load_db_credentials_file(clp_config, clp_home, True) validate_and_load_queue_credentials_file(clp_config, clp_home, True) validate_and_load_redis_credentials_file(clp_config, clp_home, True) - validate_logs_input_config(clp_config) + clp_config.validate_logs_input_config(True) validate_output_storage_config(clp_config) validate_retention_config(clp_config) - clp_config.validate_aws_config_dir() - clp_config.validate_data_dir() - clp_config.validate_logs_dir() - clp_config.validate_tmp_dir() + clp_config.validate_aws_config_dir(True) + clp_config.validate_data_dir(True) + clp_config.validate_logs_dir(True) + clp_config.validate_tmp_dir(True) except: logger.exception("Failed to load config.") return -1 diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 99456c8745..e54d7478d6 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -667,13 +667,13 @@ def make_config_paths_absolute(self, clp_home: pathlib.Path): ) self._version_file_path = make_config_path_absolute(clp_home, self._version_file_path) - def validate_logs_input_config(self): + def validate_logs_input_config(self, use_host_mount: bool = False): logs_input_type = self.logs_input.type if StorageType.FS == logs_input_type: # NOTE: This can't be a pydantic validator since input_logs_dir might be a # package-relative path that will only be resolved after pydantic validation input_logs_dir = self.logs_input.directory - resolved_input_logs_dir = resolve_host_path(input_logs_dir) + resolved_input_logs_dir = resolve_host_path(input_logs_dir) if use_host_mount else input_logs_dir if not resolved_input_logs_dir.exists(): raise ValueError(f"logs_input.directory '{input_logs_dir}' doesn't exist.") if not resolved_input_logs_dir.is_dir(): @@ -684,7 +684,7 @@ def validate_logs_input_config(self): f" = '{StorageEngine.CLP_S}'" ) - def validate_archive_output_config(self): + def validate_archive_output_config(self, use_host_mount: bool = False): if ( StorageType.S3 == self.archive_output.storage.type and StorageEngine.CLP_S != self.package.storage_engine @@ -693,12 +693,14 @@ def validate_archive_output_config(self): f"archive_output.storage.type = 's3' is only supported with package.storage_engine" f" = '{StorageEngine.CLP_S}'" ) + archive_output_dir = self.archive_output.get_directory() + resolved_archive_output_dir = resolve_host_path(archive_output_dir) if use_host_mount else archive_output_dir try: - validate_path_could_be_dir(resolve_host_path(self.archive_output.get_directory())) + validate_path_could_be_dir(resolved_archive_output_dir) except ValueError as ex: raise ValueError(f"archive_output.storage's directory is invalid: {ex}") - def validate_stream_output_config(self): + def validate_stream_output_config(self, use_host_mount: bool = False): if ( StorageType.S3 == self.stream_output.storage.type and StorageEngine.CLP_S != self.package.storage_engine @@ -707,30 +709,38 @@ def validate_stream_output_config(self): f"stream_output.storage.type = 's3' is only supported with package.storage_engine" f" = '{StorageEngine.CLP_S}'" ) + stream_output_dir = self.stream_output.get_directory() + resolved_stream_output_dir = resolve_host_path(stream_output_dir) if use_host_mount else stream_output_dir try: - validate_path_could_be_dir(resolve_host_path(self.stream_output.get_directory())) + validate_path_could_be_dir( resolved_stream_output_dir) except ValueError as ex: raise ValueError(f"stream_output.storage's directory is invalid: {ex}") - def validate_data_dir(self): + def validate_data_dir(self, use_host_mount: bool = False): + data_dir = self.data_directory + resolved_data_dir = resolve_host_path(data_dir) if use_host_mount else data_dir try: - validate_path_could_be_dir(resolve_host_path(self.data_directory)) + validate_path_could_be_dir( resolved_data_dir) except ValueError as ex: raise ValueError(f"data_directory is invalid: {ex}") - def validate_logs_dir(self): + def validate_logs_dir(self, use_host_mount: bool = False): + logs_dir = self.logs_directory + resolved_logs_dir = resolve_host_path(logs_dir) if use_host_mount else logs_dir try: - validate_path_could_be_dir(resolve_host_path(self.logs_directory)) + validate_path_could_be_dir( resolved_logs_dir) except ValueError as ex: raise ValueError(f"logs_directory is invalid: {ex}") - def validate_tmp_dir(self): + def validate_tmp_dir(self, use_host_mount: bool = False): + tmp_dir = self.tmp_directory + resolved_tmp_dir = resolve_host_path(tmp_dir) if use_host_mount else tmp_dir try: - validate_path_could_be_dir(resolve_host_path(self.tmp_directory)) + validate_path_could_be_dir( resolved_tmp_dir) except ValueError as ex: raise ValueError(f"tmp_directory is invalid: {ex}") - def validate_aws_config_dir(self): + def validate_aws_config_dir(self, use_host_mount: bool = False): profile_auth_used = False auth_configs = [] @@ -751,7 +761,8 @@ def validate_aws_config_dir(self): raise ValueError( "aws_config_directory must be set when using profile authentication" ) - if not resolve_host_path(self.aws_config_directory).exists(): + resolved_aws_config_dir = resolve_host_path(self.aws_config_directory) if use_host_mount else self.aws_config_directory + if not resolved_aws_config_dir.exists(): raise ValueError( f"aws_config_directory does not exist: '{self.aws_config_directory}'" ) From 479fc81d6a7ddd5eaae932b24a94c13124df10ae Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 29 Oct 2025 14:30:27 -0400 Subject: [PATCH 27/50] lint --- .../clp_package_utils/general.py | 1 + .../clp-py-utils/clp_py_utils/clp_config.py | 26 +++++++++++++------ 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 759a76d5c8..756578e834 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -570,6 +570,7 @@ def validate_results_cache_config( clp_config.results_cache.port, ) + def validate_output_storage_config(clp_config: CLPConfig) -> None: clp_config.validate_archive_output_config(True) clp_config.validate_stream_output_config(True) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index e54d7478d6..159f3144fd 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -673,7 +673,9 @@ def validate_logs_input_config(self, use_host_mount: bool = False): # NOTE: This can't be a pydantic validator since input_logs_dir might be a # package-relative path that will only be resolved after pydantic validation input_logs_dir = self.logs_input.directory - resolved_input_logs_dir = resolve_host_path(input_logs_dir) if use_host_mount else input_logs_dir + resolved_input_logs_dir = ( + resolve_host_path(input_logs_dir) if use_host_mount else input_logs_dir + ) if not resolved_input_logs_dir.exists(): raise ValueError(f"logs_input.directory '{input_logs_dir}' doesn't exist.") if not resolved_input_logs_dir.is_dir(): @@ -694,7 +696,9 @@ def validate_archive_output_config(self, use_host_mount: bool = False): f" = '{StorageEngine.CLP_S}'" ) archive_output_dir = self.archive_output.get_directory() - resolved_archive_output_dir = resolve_host_path(archive_output_dir) if use_host_mount else archive_output_dir + resolved_archive_output_dir = ( + resolve_host_path(archive_output_dir) if use_host_mount else archive_output_dir + ) try: validate_path_could_be_dir(resolved_archive_output_dir) except ValueError as ex: @@ -710,9 +714,11 @@ def validate_stream_output_config(self, use_host_mount: bool = False): f" = '{StorageEngine.CLP_S}'" ) stream_output_dir = self.stream_output.get_directory() - resolved_stream_output_dir = resolve_host_path(stream_output_dir) if use_host_mount else stream_output_dir + resolved_stream_output_dir = ( + resolve_host_path(stream_output_dir) if use_host_mount else stream_output_dir + ) try: - validate_path_could_be_dir( resolved_stream_output_dir) + validate_path_could_be_dir(resolved_stream_output_dir) except ValueError as ex: raise ValueError(f"stream_output.storage's directory is invalid: {ex}") @@ -720,7 +726,7 @@ def validate_data_dir(self, use_host_mount: bool = False): data_dir = self.data_directory resolved_data_dir = resolve_host_path(data_dir) if use_host_mount else data_dir try: - validate_path_could_be_dir( resolved_data_dir) + validate_path_could_be_dir(resolved_data_dir) except ValueError as ex: raise ValueError(f"data_directory is invalid: {ex}") @@ -728,7 +734,7 @@ def validate_logs_dir(self, use_host_mount: bool = False): logs_dir = self.logs_directory resolved_logs_dir = resolve_host_path(logs_dir) if use_host_mount else logs_dir try: - validate_path_could_be_dir( resolved_logs_dir) + validate_path_could_be_dir(resolved_logs_dir) except ValueError as ex: raise ValueError(f"logs_directory is invalid: {ex}") @@ -736,7 +742,7 @@ def validate_tmp_dir(self, use_host_mount: bool = False): tmp_dir = self.tmp_directory resolved_tmp_dir = resolve_host_path(tmp_dir) if use_host_mount else tmp_dir try: - validate_path_could_be_dir( resolved_tmp_dir) + validate_path_could_be_dir(resolved_tmp_dir) except ValueError as ex: raise ValueError(f"tmp_directory is invalid: {ex}") @@ -761,7 +767,11 @@ def validate_aws_config_dir(self, use_host_mount: bool = False): raise ValueError( "aws_config_directory must be set when using profile authentication" ) - resolved_aws_config_dir = resolve_host_path(self.aws_config_directory) if use_host_mount else self.aws_config_directory + resolved_aws_config_dir = ( + resolve_host_path(self.aws_config_directory) + if use_host_mount + else self.aws_config_directory + ) if not resolved_aws_config_dir.exists(): raise ValueError( f"aws_config_directory does not exist: '{self.aws_config_directory}'" From 9921d48c2070531cb071ba043769244301013fc0 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 30 Oct 2025 03:27:09 -0400 Subject: [PATCH 28/50] fix(ci): Update package build step name to indicate the package image is excluded --- .github/workflows/clp-artifact-build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/clp-artifact-build.yaml b/.github/workflows/clp-artifact-build.yaml index 63184289e5..b0106ec45e 100644 --- a/.github/workflows/clp-artifact-build.yaml +++ b/.github/workflows/clp-artifact-build.yaml @@ -538,7 +538,7 @@ jobs: shell: "bash" run: "chown $(id -u):$(id -g) -R ." - - name: "Build the package" + - name: "Build the package without the package image" uses: "./.github/actions/run-on-image" env: OS_NAME: "ubuntu-jammy" From a2dd5cf3ee1fc38e90a046b6c2deb912101355d0 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 30 Oct 2025 03:28:34 -0400 Subject: [PATCH 29/50] refactor(clp-py-utils): Rename CONTAINER_HOST_ROOT_DIR -> CONTAINER_DIR_FOR_HOST_ROOT. --- components/clp-py-utils/clp_py_utils/core.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/core.py b/components/clp-py-utils/clp_py_utils/core.py index 13e49e2385..51f1809822 100644 --- a/components/clp-py-utils/clp_py_utils/core.py +++ b/components/clp-py-utils/clp_py_utils/core.py @@ -3,7 +3,7 @@ import yaml from yaml.parser import ParserError -CONTAINER_HOST_ROOT_DIR = pathlib.Path("/") / "mnt" / "host" +CONTAINER_DIR_FOR_HOST_ROOT = pathlib.Path("/") / "mnt" / "host" class FileMetadata: @@ -72,12 +72,12 @@ def resolve_host_path(path: pathlib.Path) -> pathlib.Path: :return: The translated container path. """ path = path.absolute() - resolved = CONTAINER_HOST_ROOT_DIR / path.relative_to("/") + resolved = CONTAINER_DIR_FOR_HOST_ROOT / path.relative_to("/") try: if resolved.is_symlink(): target_path = (resolved.parent / resolved.readlink()).resolve() - resolved = CONTAINER_HOST_ROOT_DIR / target_path.relative_to("/") + resolved = CONTAINER_DIR_FOR_HOST_ROOT / target_path.relative_to("/") except OSError: pass From 8d6deb6f127fac8f85a11c8dcff53e0764f6f189 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 30 Oct 2025 03:29:19 -0400 Subject: [PATCH 30/50] refactor(clp-py-utils): Rename resolve_host_path -> resolve_host_path_in_container. --- .../clp_package_utils/controller.py | 40 ++++++++++--------- .../clp_package_utils/general.py | 18 ++++----- .../scripts/archive_manager.py | 8 ++-- .../clp_package_utils/scripts/compress.py | 16 +++++--- .../scripts/compress_from_s3.py | 16 +++++--- .../scripts/dataset_manager.py | 10 +++-- .../clp_package_utils/scripts/decompress.py | 16 +++++--- .../clp_package_utils/scripts/search.py | 10 +++-- .../clp_package_utils/scripts/start_clp.py | 16 ++++---- .../clp_package_utils/scripts/stop_clp.py | 6 +-- .../clp-py-utils/clp_py_utils/clp_config.py | 20 ++++++---- components/clp-py-utils/clp_py_utils/core.py | 2 +- 12 files changed, 101 insertions(+), 77 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 7264737f01..868ad990bf 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -37,7 +37,7 @@ get_datasets_table_name, get_files_table_name, ) -from clp_py_utils.core import resolve_host_path +from clp_py_utils.core import resolve_host_path_in_container from clp_package_utils.general import ( check_docker_dependencies, @@ -120,8 +120,8 @@ def _set_up_env_for_database(self) -> EnvVarsDict: logs_dir = self._clp_config.logs_directory / component_name validate_db_config(self._clp_config, conf_logging_file, data_dir, logs_dir) - resolved_data_dir = resolve_host_path(data_dir) - resolved_logs_dir = resolve_host_path(logs_dir) + resolved_data_dir = resolve_host_path_in_container(data_dir) + resolved_logs_dir = resolve_host_path_in_container(logs_dir) resolved_data_dir.mkdir(exist_ok=True, parents=True) resolved_logs_dir.mkdir(exist_ok=True, parents=True) @@ -170,7 +170,7 @@ def _set_up_env_for_queue(self) -> EnvVarsDict: logs_dir = self._clp_config.logs_directory / component_name validate_queue_config(self._clp_config, logs_dir) - resolved_logs_dir = resolve_host_path(logs_dir) + resolved_logs_dir = resolve_host_path_in_container(logs_dir) resolved_logs_dir.mkdir(exist_ok=True, parents=True) _chown_paths_if_root(resolved_logs_dir) @@ -210,8 +210,8 @@ def _set_up_env_for_redis(self) -> EnvVarsDict: logs_dir = self._clp_config.logs_directory / component_name validate_redis_config(self._clp_config, conf_file, data_dir, logs_dir) - resolved_data_dir = resolve_host_path(data_dir) - resolved_logs_dir = resolve_host_path(logs_dir) + resolved_data_dir = resolve_host_path_in_container(data_dir) + resolved_logs_dir = resolve_host_path_in_container(logs_dir) resolved_data_dir.mkdir(exist_ok=True, parents=True) resolved_logs_dir.mkdir(exist_ok=True, parents=True) @@ -261,8 +261,8 @@ def _set_up_env_for_results_cache(self) -> EnvVarsDict: logs_dir = self._clp_config.logs_directory / component_name validate_results_cache_config(self._clp_config, conf_file, data_dir, logs_dir) - resolved_data_dir = resolve_host_path(data_dir) - resolved_logs_dir = resolve_host_path(logs_dir) + resolved_data_dir = resolve_host_path_in_container(data_dir) + resolved_logs_dir = resolve_host_path_in_container(logs_dir) resolved_data_dir.mkdir(exist_ok=True, parents=True) resolved_logs_dir.mkdir(exist_ok=True, parents=True) @@ -303,7 +303,7 @@ def _set_up_env_for_compression_scheduler(self) -> EnvVarsDict: logger.info(f"Setting up environment for {component_name}...") logs_dir = self._clp_config.logs_directory / component_name - resolved_logs_dir = resolve_host_path(logs_dir) + resolved_logs_dir = resolve_host_path_in_container(logs_dir) resolved_logs_dir.mkdir(parents=True, exist_ok=True) env_vars = EnvVarsDict() @@ -327,7 +327,7 @@ def _set_up_env_for_query_scheduler(self) -> EnvVarsDict: logger.info(f"Setting up environment for {component_name}...") logs_dir = self._clp_config.logs_directory / component_name - resolved_logs_dir = resolve_host_path(logs_dir) + resolved_logs_dir = resolve_host_path_in_container(logs_dir) resolved_logs_dir.mkdir(parents=True, exist_ok=True) env_vars = EnvVarsDict() @@ -350,7 +350,7 @@ def _set_up_env_for_compression_worker(self, num_workers: int) -> EnvVarsDict: logger.info(f"Setting up environment for {component_name}...") logs_dir = self._clp_config.logs_directory / component_name - resolved_logs_dir = resolve_host_path(logs_dir) + resolved_logs_dir = resolve_host_path_in_container(logs_dir) resolved_logs_dir.mkdir(parents=True, exist_ok=True) env_vars = EnvVarsDict() @@ -380,7 +380,7 @@ def _set_up_env_for_query_worker(self, num_workers: int) -> EnvVarsDict: logger.info(f"Setting up environment for {component_name}...") logs_dir = self._clp_config.logs_directory / component_name - resolved_logs_dir = resolve_host_path(logs_dir) + resolved_logs_dir = resolve_host_path_in_container(logs_dir) resolved_logs_dir.mkdir(parents=True, exist_ok=True) env_vars = EnvVarsDict() @@ -408,7 +408,7 @@ def _set_up_env_for_reducer(self, num_workers: int) -> EnvVarsDict: logger.info(f"Setting up environment for {component_name}...") logs_dir = self._clp_config.logs_directory / component_name - resolved_logs_dir = resolve_host_path(logs_dir) + resolved_logs_dir = resolve_host_path_in_container(logs_dir) resolved_logs_dir.mkdir(parents=True, exist_ok=True) env_vars = EnvVarsDict() @@ -443,8 +443,12 @@ def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict: server_settings_json_path = ( self._clp_home / "var" / "www" / "webui" / "server" / "dist" / "settings.json" ) - resolved_client_settings_json_path = resolve_host_path(client_settings_json_path) - resolved_server_settings_json_path = resolve_host_path(server_settings_json_path) + resolved_client_settings_json_path = resolve_host_path_in_container( + client_settings_json_path + ) + resolved_server_settings_json_path = resolve_host_path_in_container( + server_settings_json_path + ) validate_webui_config( self._clp_config, client_settings_json_path, @@ -565,7 +569,7 @@ def _set_up_env_for_mcp_server(self) -> EnvVarsDict: logs_dir = self._clp_config.logs_directory / component_name validate_mcp_server_config(self._clp_config, logs_dir) - resolved_logs_dir = resolve_host_path(logs_dir) + resolved_logs_dir = resolve_host_path_in_container(logs_dir) resolved_logs_dir.mkdir(parents=True, exist_ok=True) env_vars = EnvVarsDict() @@ -603,7 +607,7 @@ def _set_up_env_for_garbage_collector(self) -> EnvVarsDict: logger.info(f"Setting up environment for {component_name}...") logs_dir = self._clp_config.logs_directory / component_name - resolved_logs_dir = resolve_host_path(logs_dir) + resolved_logs_dir = resolve_host_path_in_container(logs_dir) resolved_logs_dir.mkdir(parents=True, exist_ok=True) env_vars = EnvVarsDict() @@ -827,7 +831,7 @@ def get_or_create_instance_id(clp_config: CLPConfig) -> str: :return: The instance ID. """ instance_id_file_path = clp_config.logs_directory / "instance-id" - resolved_instance_id_file_path = resolve_host_path(instance_id_file_path) + resolved_instance_id_file_path = resolve_host_path_in_container(instance_id_file_path) if resolved_instance_id_file_path.exists(): with open(resolved_instance_id_file_path, "r") as f: diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 756578e834..10a36003e2 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -38,7 +38,7 @@ get_config_value, make_config_path_absolute, read_yaml_config_file, - resolve_host_path, + resolve_host_path_in_container, validate_path_could_be_dir, ) from strenum import KebabCaseStrEnum @@ -356,7 +356,7 @@ def dump_container_config( """ config_file_path_on_host = clp_config.logs_directory / config_filename config_file_path_on_container = container_clp_config.logs_directory / config_filename - resolved_config_file_path_on_host = resolve_host_path(config_file_path_on_host) + resolved_config_file_path_on_host = resolve_host_path_in_container(config_file_path_on_host) with open(resolved_config_file_path_on_host, "w") as f: yaml.safe_dump(container_clp_config.dump_to_primitive_dict(), f) @@ -466,7 +466,7 @@ def validate_credentials_file_path( clp_config: CLPConfig, clp_home: pathlib.Path, generate_default_file: bool ): credentials_file_path = clp_config.credentials_file_path - resolved_credentials_file_path = resolve_host_path(credentials_file_path) + resolved_credentials_file_path = resolve_host_path_in_container(credentials_file_path) if not resolved_credentials_file_path.exists(): if ( make_config_path_absolute(clp_home, CLP_DEFAULT_CREDENTIALS_FILE_PATH) @@ -507,7 +507,7 @@ def validate_db_config( data_dir: pathlib.Path, logs_dir: pathlib.Path, ): - resolved_component_config = resolve_host_path(component_config) + resolved_component_config = resolve_host_path_in_container(component_config) if not resolved_component_config.exists(): raise ValueError(f"{DB_COMPONENT_NAME} configuration file missing: '{component_config}'.") _validate_data_directory(data_dir, DB_COMPONENT_NAME) @@ -528,7 +528,7 @@ def validate_redis_config( data_dir: pathlib.Path, logs_dir: pathlib.Path, ): - resolved_component_config = resolve_host_path(component_config) + resolved_component_config = resolve_host_path_in_container(component_config) if not resolved_component_config.exists(): raise ValueError( f"{REDIS_COMPONENT_NAME} configuration file missing: '{component_config}'." @@ -556,7 +556,7 @@ def validate_results_cache_config( data_dir: pathlib.Path, logs_dir: pathlib.Path, ): - resolved_component_config = resolve_host_path(component_config) + resolved_component_config = resolve_host_path_in_container(component_config) if not resolved_component_config.exists(): raise ValueError( f"{RESULTS_CACHE_COMPONENT_NAME} configuration file missing: '{component_config}'." @@ -585,7 +585,7 @@ def validate_webui_config( server_settings_json_path: pathlib.Path, ): for path in [client_settings_json_path, server_settings_json_path]: - resolved_path = resolve_host_path(path) + resolved_path = resolve_host_path_in_container(path) if not resolved_path.exists(): raise ValueError(f"{WEBUI_COMPONENT_NAME} {path} is not a valid path to settings.json") @@ -768,7 +768,7 @@ def _is_docker_compose_project_running(project_name: str) -> bool: def _validate_data_directory(data_dir: pathlib.Path, component_name: str) -> None: try: - validate_path_could_be_dir(resolve_host_path(data_dir)) + validate_path_could_be_dir(resolve_host_path_in_container(data_dir)) except ValueError as ex: raise ValueError(f"{component_name} data directory is invalid: {ex}") @@ -782,6 +782,6 @@ def _validate_log_directory(logs_dir: pathlib.Path, component_name: str): :raise ValueError: If the path is invalid or can't be a directory. """ try: - validate_path_could_be_dir(resolve_host_path(logs_dir)) + validate_path_could_be_dir(resolve_host_path_in_container(logs_dir)) except ValueError as ex: raise ValueError(f"{component_name} logs directory is invalid: {ex}") diff --git a/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py b/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py index 45e0341f42..570ca7f7f4 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py +++ b/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py @@ -14,7 +14,7 @@ StorageEngine, StorageType, ) -from clp_py_utils.core import resolve_host_path +from clp_py_utils.core import resolve_host_path_in_container from clp_package_utils.general import ( CLPConfig, @@ -172,8 +172,8 @@ def main(argv: List[str]) -> int: try: config_file_path: Path = Path(parsed_args.config) clp_config: CLPConfig = load_config_file( - resolve_host_path(config_file_path), - resolve_host_path(default_config_file_path), + resolve_host_path_in_container(config_file_path), + resolve_host_path_in_container(default_config_file_path), clp_home, ) clp_config.validate_logs_dir(True) @@ -283,7 +283,7 @@ def main(argv: List[str]) -> int: logger.debug(f"Docker command failed: {shlex.join(cmd)}") # Remove generated files - resolved_generated_config_path_on_host = resolve_host_path(generated_config_path_on_host) + resolved_generated_config_path_on_host = resolve_host_path_in_container(generated_config_path_on_host) resolved_generated_config_path_on_host.unlink() return ret_code diff --git a/components/clp-package-utils/clp_package_utils/scripts/compress.py b/components/clp-package-utils/clp_package_utils/scripts/compress.py index 1e98ab3b4b..0678753a56 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/compress.py +++ b/components/clp-package-utils/clp_package_utils/scripts/compress.py @@ -15,7 +15,7 @@ StorageEngine, StorageType, ) -from clp_py_utils.core import resolve_host_path +from clp_py_utils.core import resolve_host_path_in_container from clp_package_utils.general import ( CONTAINER_INPUT_LOGS_ROOT_DIR, @@ -57,7 +57,9 @@ def _generate_logs_list( return len(parsed_args.paths) != 0 no_path_found = True - resolved_host_logs_list_path = resolve_host_path(pathlib.Path(host_logs_list_path)) + resolved_host_logs_list_path = resolve_host_path_in_container( + pathlib.Path(host_logs_list_path) + ) with open(resolved_host_logs_list_path, "r") as host_logs_list_file: for line in host_logs_list_file: stripped_path_str = line.rstrip() @@ -178,8 +180,8 @@ def main(argv): try: config_file_path = pathlib.Path(parsed_args.config) clp_config = load_config_file( - resolve_host_path(config_file_path), - resolve_host_path(default_config_file_path), + resolve_host_path_in_container(config_file_path), + resolve_host_path_in_container(default_config_file_path), clp_home, ) clp_config.validate_logs_dir(True) @@ -243,7 +245,7 @@ def main(argv): # Get unused output path logs_list_filename = f"{uuid.uuid4()}.txt" logs_list_path_on_host = clp_config.logs_directory / logs_list_filename - resolved_logs_list_path_on_host = resolve_host_path(logs_list_path_on_host) + resolved_logs_list_path_on_host = resolve_host_path_in_container(logs_list_path_on_host) logs_list_path_on_container = container_clp_config.logs_directory / logs_list_filename if not resolved_logs_list_path_on_host.exists(): break @@ -273,7 +275,9 @@ def main(argv): else: resolved_logs_list_path_on_host.unlink() - resolved_generated_config_path_on_host = resolve_host_path(generated_config_path_on_host) + resolved_generated_config_path_on_host = resolve_host_path_in_container( + generated_config_path_on_host + ) resolved_generated_config_path_on_host.unlink() return ret_code diff --git a/components/clp-package-utils/clp_package_utils/scripts/compress_from_s3.py b/components/clp-package-utils/clp_package_utils/scripts/compress_from_s3.py index f5faf78159..de70983031 100644 --- a/components/clp-package-utils/clp_package_utils/scripts/compress_from_s3.py +++ b/components/clp-package-utils/clp_package_utils/scripts/compress_from_s3.py @@ -14,7 +14,7 @@ StorageEngine, StorageType, ) -from clp_py_utils.core import resolve_host_path +from clp_py_utils.core import resolve_host_path_in_container from clp_package_utils.general import ( dump_container_config, @@ -57,7 +57,9 @@ def _generate_url_list( return len(parsed_args.inputs) != 0 no_url_found = True - resolved_inputs_from_path = resolve_host_path(pathlib.Path(parsed_args.inputs_from)) + resolved_inputs_from_path = resolve_host_path_in_container( + pathlib.Path(parsed_args.inputs_from) + ) with open(resolved_inputs_from_path, "r") as input_file: for line in input_file: stripped_url = line.strip() @@ -225,8 +227,8 @@ def main(argv): try: config_file_path = pathlib.Path(parsed_args.config) clp_config = load_config_file( - resolve_host_path(config_file_path), - resolve_host_path(default_config_file_path), + resolve_host_path_in_container(config_file_path), + resolve_host_path_in_container(default_config_file_path), clp_home, ) clp_config.validate_logs_dir(True) @@ -297,7 +299,7 @@ def main(argv): while True: url_list_filename = f"{uuid.uuid4()}.txt" url_list_path_on_host = clp_config.logs_directory / url_list_filename - resolved_url_list_path_on_host = resolve_host_path(url_list_path_on_host) + resolved_url_list_path_on_host = resolve_host_path_in_container(url_list_path_on_host) url_list_path_on_container = container_clp_config.logs_directory / url_list_filename if not resolved_url_list_path_on_host.exists(): break @@ -327,7 +329,9 @@ def main(argv): else: resolved_url_list_path_on_host.unlink() - resolved_generated_config_path_on_host = resolve_host_path(generated_config_path_on_host) + resolved_generated_config_path_on_host = resolve_host_path_in_container( + generated_config_path_on_host + ) resolved_generated_config_path_on_host.unlink() return ret_code diff --git a/components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py b/components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py index f1c3111877..d4b1218d1e 100644 --- a/components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py +++ b/components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py @@ -14,7 +14,7 @@ StorageEngine, StorageType, ) -from clp_py_utils.core import resolve_host_path +from clp_py_utils.core import resolve_host_path_in_container from clp_py_utils.s3_utils import generate_container_auth_options from clp_package_utils.general import ( @@ -94,8 +94,8 @@ def main(argv: List[str]) -> int: try: config_file_path = Path(parsed_args.config) clp_config = load_config_file( - resolve_host_path(config_file_path), - resolve_host_path(default_config_file_path), + resolve_host_path_in_container(config_file_path), + resolve_host_path_in_container(default_config_file_path), clp_home, ) clp_config.validate_logs_dir(True) @@ -189,7 +189,9 @@ def main(argv: List[str]) -> int: logger.debug(f"Docker command failed: {shlex.join(cmd)}") # Remove generated files - resolved_generated_config_path_on_host = resolve_host_path(generated_config_path_on_host) + resolved_generated_config_path_on_host = resolve_host_path_in_container( + generated_config_path_on_host + ) resolved_generated_config_path_on_host.unlink() return ret_code diff --git a/components/clp-package-utils/clp_package_utils/scripts/decompress.py b/components/clp-package-utils/clp_package_utils/scripts/decompress.py index f4204f0924..747874fa5b 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/decompress.py +++ b/components/clp-package-utils/clp_package_utils/scripts/decompress.py @@ -16,7 +16,7 @@ StorageEngine, StorageType, ) -from clp_py_utils.core import resolve_host_path +from clp_py_utils.core import resolve_host_path_in_container from clp_package_utils.general import ( DockerMount, @@ -54,8 +54,8 @@ def validate_and_load_config( """ try: clp_config = load_config_file( - resolve_host_path(config_file_path), - resolve_host_path(default_config_file_path), + resolve_host_path_in_container(config_file_path), + resolve_host_path_in_container(default_config_file_path), clp_home, ) clp_config.validate_logs_dir(True) @@ -84,7 +84,7 @@ def handle_extract_file_cmd( # Validate extraction directory extraction_dir = pathlib.Path(parsed_args.extraction_dir).resolve() - resolved_extraction_dir = resolve_host_path(extraction_dir) + resolved_extraction_dir = resolve_host_path_in_container(extraction_dir) try: validate_path_could_be_dir(resolved_extraction_dir) except ValueError as ex: @@ -167,7 +167,9 @@ def handle_extract_file_cmd( logger.debug(f"Docker command failed: {shlex.join(cmd)}") # Remove generated files - resolved_generated_config_path_on_host = resolve_host_path(generated_config_path_on_host) + resolved_generated_config_path_on_host = resolve_host_path_in_container( + generated_config_path_on_host + ) resolved_generated_config_path_on_host.unlink() return ret_code @@ -273,7 +275,9 @@ def handle_extract_stream_cmd( logger.debug(f"Docker command failed: {shlex.join(cmd)}") # Remove generated files - resolved_generated_config_path_on_host = resolve_host_path(generated_config_path_on_host) + resolved_generated_config_path_on_host = resolve_host_path_in_container( + generated_config_path_on_host + ) resolved_generated_config_path_on_host.unlink() return ret_code diff --git a/components/clp-package-utils/clp_package_utils/scripts/search.py b/components/clp-package-utils/clp_package_utils/scripts/search.py index f8b299c791..0c4eda4ec0 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/search.py +++ b/components/clp-package-utils/clp_package_utils/scripts/search.py @@ -13,7 +13,7 @@ StorageEngine, StorageType, ) -from clp_py_utils.core import resolve_host_path +from clp_py_utils.core import resolve_host_path_in_container from clp_package_utils.general import ( dump_container_config, @@ -93,8 +93,8 @@ def main(argv): try: config_file_path = pathlib.Path(parsed_args.config) clp_config = load_config_file( - resolve_host_path(config_file_path), - resolve_host_path(default_config_file_path), + resolve_host_path_in_container(config_file_path), + resolve_host_path_in_container(default_config_file_path), clp_home, ) clp_config.validate_logs_dir(True) @@ -185,7 +185,9 @@ def main(argv): logger.debug(f"Docker command failed: {shlex.join(cmd)}") # Remove generated files - resolved_generated_config_path_on_host = resolve_host_path(generated_config_path_on_host) + resolved_generated_config_path_on_host = resolve_host_path_in_container( + generated_config_path_on_host + ) resolved_generated_config_path_on_host.unlink() return ret_code diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 0b2a29fbf5..01b28bf38f 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -4,7 +4,7 @@ import sys from clp_py_utils.clp_config import CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH -from clp_py_utils.core import resolve_host_path +from clp_py_utils.core import resolve_host_path_in_container from clp_package_utils.controller import DockerComposeController, get_or_create_instance_id from clp_package_utils.general import ( @@ -54,8 +54,8 @@ def main(argv): # Validate and load config file. config_file_path = pathlib.Path(parsed_args.config) clp_config = load_config_file( - resolve_host_path(config_file_path), - resolve_host_path(default_config_file_path), + resolve_host_path_in_container(config_file_path), + resolve_host_path_in_container(default_config_file_path), clp_home, ) @@ -76,13 +76,13 @@ def main(argv): try: # Create necessary directories. - resolve_host_path(clp_config.data_directory).mkdir(parents=True, exist_ok=True) - resolve_host_path(clp_config.logs_directory).mkdir(parents=True, exist_ok=True) - resolve_host_path(clp_config.tmp_directory).mkdir(parents=True, exist_ok=True) - resolve_host_path(clp_config.archive_output.get_directory()).mkdir( + resolve_host_path_in_container(clp_config.data_directory).mkdir(parents=True, exist_ok=True) + resolve_host_path_in_container(clp_config.logs_directory).mkdir(parents=True, exist_ok=True) + resolve_host_path_in_container(clp_config.tmp_directory).mkdir(parents=True, exist_ok=True) + resolve_host_path_in_container(clp_config.archive_output.get_directory()).mkdir( parents=True, exist_ok=True ) - resolve_host_path(clp_config.stream_output.get_directory()).mkdir( + resolve_host_path_in_container(clp_config.stream_output.get_directory()).mkdir( parents=True, exist_ok=True ) except: diff --git a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py index d95c4e9aa0..b450a73027 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py @@ -4,7 +4,7 @@ import sys from clp_py_utils.clp_config import CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH -from clp_py_utils.core import resolve_host_path +from clp_py_utils.core import resolve_host_path_in_container from clp_package_utils.controller import DockerComposeController, get_or_create_instance_id from clp_package_utils.general import ( @@ -32,8 +32,8 @@ def main(argv): try: config_file_path = pathlib.Path(parsed_args.config) clp_config = load_config_file( - resolve_host_path(config_file_path), - resolve_host_path(default_config_file_path), + resolve_host_path_in_container(config_file_path), + resolve_host_path_in_container(default_config_file_path), clp_home, ) except: diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 159f3144fd..94a6c063da 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -19,7 +19,7 @@ get_config_value, make_config_path_absolute, read_yaml_config_file, - resolve_host_path, + resolve_host_path_in_container, validate_path_could_be_dir, ) from .serialization_utils import serialize_path, serialize_str_enum @@ -674,7 +674,7 @@ def validate_logs_input_config(self, use_host_mount: bool = False): # package-relative path that will only be resolved after pydantic validation input_logs_dir = self.logs_input.directory resolved_input_logs_dir = ( - resolve_host_path(input_logs_dir) if use_host_mount else input_logs_dir + resolve_host_path_in_container(input_logs_dir) if use_host_mount else input_logs_dir ) if not resolved_input_logs_dir.exists(): raise ValueError(f"logs_input.directory '{input_logs_dir}' doesn't exist.") @@ -697,7 +697,9 @@ def validate_archive_output_config(self, use_host_mount: bool = False): ) archive_output_dir = self.archive_output.get_directory() resolved_archive_output_dir = ( - resolve_host_path(archive_output_dir) if use_host_mount else archive_output_dir + resolve_host_path_in_container(archive_output_dir) + if use_host_mount + else archive_output_dir ) try: validate_path_could_be_dir(resolved_archive_output_dir) @@ -715,7 +717,9 @@ def validate_stream_output_config(self, use_host_mount: bool = False): ) stream_output_dir = self.stream_output.get_directory() resolved_stream_output_dir = ( - resolve_host_path(stream_output_dir) if use_host_mount else stream_output_dir + resolve_host_path_in_container(stream_output_dir) + if use_host_mount + else stream_output_dir ) try: validate_path_could_be_dir(resolved_stream_output_dir) @@ -724,7 +728,7 @@ def validate_stream_output_config(self, use_host_mount: bool = False): def validate_data_dir(self, use_host_mount: bool = False): data_dir = self.data_directory - resolved_data_dir = resolve_host_path(data_dir) if use_host_mount else data_dir + resolved_data_dir = resolve_host_path_in_container(data_dir) if use_host_mount else data_dir try: validate_path_could_be_dir(resolved_data_dir) except ValueError as ex: @@ -732,7 +736,7 @@ def validate_data_dir(self, use_host_mount: bool = False): def validate_logs_dir(self, use_host_mount: bool = False): logs_dir = self.logs_directory - resolved_logs_dir = resolve_host_path(logs_dir) if use_host_mount else logs_dir + resolved_logs_dir = resolve_host_path_in_container(logs_dir) if use_host_mount else logs_dir try: validate_path_could_be_dir(resolved_logs_dir) except ValueError as ex: @@ -740,7 +744,7 @@ def validate_logs_dir(self, use_host_mount: bool = False): def validate_tmp_dir(self, use_host_mount: bool = False): tmp_dir = self.tmp_directory - resolved_tmp_dir = resolve_host_path(tmp_dir) if use_host_mount else tmp_dir + resolved_tmp_dir = resolve_host_path_in_container(tmp_dir) if use_host_mount else tmp_dir try: validate_path_could_be_dir(resolved_tmp_dir) except ValueError as ex: @@ -768,7 +772,7 @@ def validate_aws_config_dir(self, use_host_mount: bool = False): "aws_config_directory must be set when using profile authentication" ) resolved_aws_config_dir = ( - resolve_host_path(self.aws_config_directory) + resolve_host_path_in_container(self.aws_config_directory) if use_host_mount else self.aws_config_directory ) diff --git a/components/clp-py-utils/clp_py_utils/core.py b/components/clp-py-utils/clp_py_utils/core.py index 51f1809822..b93539c759 100644 --- a/components/clp-py-utils/clp_py_utils/core.py +++ b/components/clp-py-utils/clp_py_utils/core.py @@ -64,7 +64,7 @@ def read_yaml_config_file(yaml_config_file_path: pathlib.Path): return config -def resolve_host_path(path: pathlib.Path) -> pathlib.Path: +def resolve_host_path_in_container(path: pathlib.Path) -> pathlib.Path: """ Translates a host path to its container-mount equivalent. From c6c516434d65b463158cdd59fd709311997d41c0 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 30 Oct 2025 03:29:54 -0400 Subject: [PATCH 31/50] Update docstring to clarify `path` translation return value --- components/clp-py-utils/clp_py_utils/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/clp-py-utils/clp_py_utils/core.py b/components/clp-py-utils/clp_py_utils/core.py index b93539c759..6185b0d44f 100644 --- a/components/clp-py-utils/clp_py_utils/core.py +++ b/components/clp-py-utils/clp_py_utils/core.py @@ -69,7 +69,7 @@ def resolve_host_path_in_container(path: pathlib.Path) -> pathlib.Path: Translates a host path to its container-mount equivalent. :param path: The host path. - :return: The translated container path. + :return: The translated path. """ path = path.absolute() resolved = CONTAINER_DIR_FOR_HOST_ROOT / path.relative_to("/") From 3b4494520d9557f4cf11d1237d61cbbf6684ac12 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 30 Oct 2025 03:30:52 -0400 Subject: [PATCH 32/50] refactor(clp-py-utils): Rename path -> host_path and resolved -> translated_path. --- components/clp-py-utils/clp_py_utils/core.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/core.py b/components/clp-py-utils/clp_py_utils/core.py index 6185b0d44f..a8accffa68 100644 --- a/components/clp-py-utils/clp_py_utils/core.py +++ b/components/clp-py-utils/clp_py_utils/core.py @@ -64,24 +64,24 @@ def read_yaml_config_file(yaml_config_file_path: pathlib.Path): return config -def resolve_host_path_in_container(path: pathlib.Path) -> pathlib.Path: +def resolve_host_path_in_container(host_path: pathlib.Path) -> pathlib.Path: """ Translates a host path to its container-mount equivalent. - :param path: The host path. + :param host_path: The host path. :return: The translated path. """ - path = path.absolute() - resolved = CONTAINER_DIR_FOR_HOST_ROOT / path.relative_to("/") + host_path = host_path.absolute() + translated_path = CONTAINER_DIR_FOR_HOST_ROOT / host_path.relative_to("/") try: - if resolved.is_symlink(): - target_path = (resolved.parent / resolved.readlink()).resolve() - resolved = CONTAINER_DIR_FOR_HOST_ROOT / target_path.relative_to("/") + if translated_path.is_symlink(): + target_path = (translated_path.parent / translated_path.readlink()).resolve() + translated_path = CONTAINER_DIR_FOR_HOST_ROOT / target_path.relative_to("/") except OSError: pass - return resolved + return translated_path def validate_path_could_be_dir(path: pathlib.Path): From 4b6f3371a9a4a098760f308ae063da4d8d9b647d Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 30 Oct 2025 03:37:40 -0400 Subject: [PATCH 33/50] refactor(taskfile): Move NODE_ENV definition to webui task. --- taskfile.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/taskfile.yaml b/taskfile.yaml index 22857e0bff..4f72e56a09 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -104,8 +104,6 @@ tasks: - "rm -rf '{{.G_WEBUI_SRC_DIR}}/yscope-log-viewer/node_modules'" package: - env: - NODE_ENV: "production" vars: CHECKSUM_FILE: "{{.G_PACKAGE_CHECKSUM_FILE}}" OUTPUT_DIR: "{{.G_PACKAGE_BUILD_DIR}}" @@ -221,6 +219,8 @@ tasks: - "webui" webui: + env: + NODE_ENV: "production" vars: CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK}}.md5" OUTPUT_DIR: "{{.G_WEBUI_BUILD_DIR}}" From bb90b69336679069a3ad001edcb4920fb11cd316 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 30 Oct 2025 03:38:35 -0400 Subject: [PATCH 34/50] refactor(taskfile): Reorder package-build-deps dependencies for clarity. --- taskfile.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskfile.yaml b/taskfile.yaml index 4f72e56a09..da34155a1c 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -212,9 +212,9 @@ tasks: package-build-deps: deps: - - "init" - "core" - "deps:spider" + - "init" - "python-libs" - "webui" From ae825c0085f5a0f2ffa80da47fcc9be8a668e17b Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 30 Oct 2025 03:40:01 -0400 Subject: [PATCH 35/50] refactor(taskfile): Change "{{.G_BUILD_DIR}}/{{.TASK}}.md5" -> G_WEBUI_CHECKSUM_FILE in CHECKSUM_FILE var. --- taskfile.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskfile.yaml b/taskfile.yaml index da34155a1c..569c9c2741 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -222,7 +222,7 @@ tasks: env: NODE_ENV: "production" vars: - CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK}}.md5" + CHECKSUM_FILE: "{{.G_WEBUI_CHECKSUM_FILE}}" OUTPUT_DIR: "{{.G_WEBUI_BUILD_DIR}}" sources: - "{{.G_WEBUI_CLIENT_NODE_MODULES_CHECKSUM_FILE}}" From 9a66669555aabb36f639885dcb77d5fe1748d84c Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 30 Oct 2025 03:41:46 -0400 Subject: [PATCH 36/50] Add back package-template to package dependencies. --- taskfile.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/taskfile.yaml b/taskfile.yaml index 569c9c2741..c215cb8d87 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -112,6 +112,7 @@ tasks: - "{{.G_WEBUI_BUILD_DIR}}/client/settings.json" - "{{.G_WEBUI_BUILD_DIR}}/server/dist/settings.json" - "{{.TASKFILE}}" + - "components/package-template/src/**/*" - "tools/deployment/package/**/*" generates: ["{{.CHECKSUM_FILE}}"] deps: From 6983943e67abcf4565966aeab68a1f2d5d133a47 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 30 Oct 2025 03:56:42 -0400 Subject: [PATCH 37/50] fix(clp-py-utils): Explain in docs that only single-level symlink are resolved in host path translation; rewrite to handle absolute symlink targets as well --- components/clp-py-utils/clp_py_utils/core.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/core.py b/components/clp-py-utils/clp_py_utils/core.py index a8accffa68..99a649200d 100644 --- a/components/clp-py-utils/clp_py_utils/core.py +++ b/components/clp-py-utils/clp_py_utils/core.py @@ -66,7 +66,8 @@ def read_yaml_config_file(yaml_config_file_path: pathlib.Path): def resolve_host_path_in_container(host_path: pathlib.Path) -> pathlib.Path: """ - Translates a host path to its container-mount equivalent. + Translates a host path to its container-mount equivalent. It also resolves a single level of + symbolic link if the host path itself is a symlink. :param host_path: The host path. :return: The translated path. @@ -75,9 +76,13 @@ def resolve_host_path_in_container(host_path: pathlib.Path) -> pathlib.Path: translated_path = CONTAINER_DIR_FOR_HOST_ROOT / host_path.relative_to("/") try: - if translated_path.is_symlink(): - target_path = (translated_path.parent / translated_path.readlink()).resolve() - translated_path = CONTAINER_DIR_FOR_HOST_ROOT / target_path.relative_to("/") + if not translated_path.is_symlink(): + return translated_path + link_target = translated_path.readlink() + if link_target.is_absolute(): + return CONTAINER_DIR_FOR_HOST_ROOT / link_target.relative_to("/") + else: + return (translated_path.parent / link_target).resolve() except OSError: pass From bee1b79886b7c0c6687d548ab26e73534abe0a50 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 30 Oct 2025 10:37:47 -0400 Subject: [PATCH 38/50] Reorder dependencies --- taskfile.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskfile.yaml b/taskfile.yaml index c215cb8d87..293a6b1298 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -492,10 +492,10 @@ tasks: - "components/job-orchestration/dist/*.whl" generates: ["{{.CHECKSUM_FILE}}"] deps: - - "init" - "clp-mcp-server" - "clp-package-utils" - "clp-py-utils" + - "init" - "job-orchestration" - "package-venv" - task: "utils:checksum:validate" From b0e8abfae40d2218f82799fd9e3831fdf9dbf7d2 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 31 Oct 2025 18:22:49 -0400 Subject: [PATCH 39/50] shfmt - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- components/package-template/src/sbin/.common-env.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/components/package-template/src/sbin/.common-env.sh b/components/package-template/src/sbin/.common-env.sh index 3d858e49c3..623ff9cc3f 100755 --- a/components/package-template/src/sbin/.common-env.sh +++ b/components/package-template/src/sbin/.common-env.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" package_root=$(readlink -f "$script_dir/..") if [[ -z "${CLP_HOME:-}" ]]; then @@ -13,10 +13,10 @@ image_id_file="$CLP_HOME/clp-package-image.id" version_file="$CLP_HOME/VERSION" if [[ -f "$image_id_file" ]]; then - image_id="$(tr -d '[:space:]' < "$image_id_file")" + image_id="$(tr -d '[:space:]' <"$image_id_file")" export CLP_PACKAGE_CONTAINER_IMAGE_REF="$image_id" elif [[ -f "$version_file" ]]; then - version="$(tr -d '[:space:]' < "$version_file")" + version="$(tr -d '[:space:]' <"$version_file")" export CLP_PACKAGE_CONTAINER_IMAGE_REF="ghcr.io/y-scope/clp/clp-package:v$version" else >&2 echo "Error: Neither clp-package-image.id nor VERSION file exists." @@ -45,7 +45,7 @@ if [[ -z "${CLP_DOCKER_PLUGIN_DIR:-}" ]]; then done if [[ -z "${CLP_DOCKER_PLUGIN_DIR:-}" ]]; then >&2 echo "Warning: Docker plugin directory not found;" \ - "Docker Compose may not work inside container." + "Docker Compose may not work inside container." fi fi From 5172963babfe283906ccf0b116c023fed6fa5f86 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 31 Oct 2025 18:23:36 -0400 Subject: [PATCH 40/50] avoid hardcoding names - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- components/package-template/src/sbin/.common-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/package-template/src/sbin/.common-env.sh b/components/package-template/src/sbin/.common-env.sh index 623ff9cc3f..c265c2b8ef 100755 --- a/components/package-template/src/sbin/.common-env.sh +++ b/components/package-template/src/sbin/.common-env.sh @@ -19,7 +19,7 @@ elif [[ -f "$version_file" ]]; then version="$(tr -d '[:space:]' <"$version_file")" export CLP_PACKAGE_CONTAINER_IMAGE_REF="ghcr.io/y-scope/clp/clp-package:v$version" else - >&2 echo "Error: Neither clp-package-image.id nor VERSION file exists." + >&2 echo "Error: Neither '${image_id_file}' nor '${version_file}' exist." return 1 fi From bb5d8d2bedd5bc66c3eea1ec3a9882c7a691ef6f Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 3 Nov 2025 03:01:05 -0500 Subject: [PATCH 41/50] shfmt --- components/package-template/src/sbin/.common-env.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/package-template/src/sbin/.common-env.sh b/components/package-template/src/sbin/.common-env.sh index c265c2b8ef..7f6672c9c9 100755 --- a/components/package-template/src/sbin/.common-env.sh +++ b/components/package-template/src/sbin/.common-env.sh @@ -51,8 +51,8 @@ fi if [[ -z "${CLP_DOCKER_SOCK_PATH:-}" ]]; then socket="$(docker context inspect \ - --format '{{.Endpoints.docker.Host}}' 2>/dev/null \ - | sed -E 's|^unix://||')" + --format '{{.Endpoints.docker.Host}}' 2>/dev/null | + sed -E 's|^unix://||')" if [[ -S "$socket" ]]; then export CLP_DOCKER_SOCK_PATH="$socket" From 95a8e1fe9767272432e3a089aabdda379c78ed2d Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 3 Nov 2025 03:02:17 -0500 Subject: [PATCH 42/50] Use standard form for stderr redirection --- components/package-template/src/sbin/.common-env.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/package-template/src/sbin/.common-env.sh b/components/package-template/src/sbin/.common-env.sh index 7f6672c9c9..0ba9d3c19c 100755 --- a/components/package-template/src/sbin/.common-env.sh +++ b/components/package-template/src/sbin/.common-env.sh @@ -19,7 +19,7 @@ elif [[ -f "$version_file" ]]; then version="$(tr -d '[:space:]' <"$version_file")" export CLP_PACKAGE_CONTAINER_IMAGE_REF="ghcr.io/y-scope/clp/clp-package:v$version" else - >&2 echo "Error: Neither '${image_id_file}' nor '${version_file}' exist." + echo >&2 "Error: Neither '${image_id_file}' nor '${version_file}' exist." return 1 fi @@ -44,7 +44,7 @@ if [[ -z "${CLP_DOCKER_PLUGIN_DIR:-}" ]]; then fi done if [[ -z "${CLP_DOCKER_PLUGIN_DIR:-}" ]]; then - >&2 echo "Warning: Docker plugin directory not found;" \ + echo >&2 "Warning: Docker plugin directory not found;" \ "Docker Compose may not work inside container." fi fi From 395e42f60a7b9151c5d066569d1d411e55647310 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 3 Nov 2025 03:03:07 -0500 Subject: [PATCH 43/50] shfmt --- .../package-template/src/sbin/admin-tools/archive-manager.sh | 2 +- .../package-template/src/sbin/admin-tools/dataset-manager.sh | 2 +- components/package-template/src/sbin/compress-from-s3.sh | 2 +- components/package-template/src/sbin/compress.sh | 2 +- components/package-template/src/sbin/decompress.sh | 2 +- components/package-template/src/sbin/search.sh | 2 +- components/package-template/src/sbin/start-clp.sh | 2 +- components/package-template/src/sbin/stop-clp.sh | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/components/package-template/src/sbin/admin-tools/archive-manager.sh b/components/package-template/src/sbin/admin-tools/archive-manager.sh index afa2acdd25..9c69fce68a 100755 --- a/components/package-template/src/sbin/admin-tools/archive-manager.sh +++ b/components/package-template/src/sbin/admin-tools/archive-manager.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" common_env_path="$script_dir/../.common-env.sh" # shellcheck source=.common-env.sh diff --git a/components/package-template/src/sbin/admin-tools/dataset-manager.sh b/components/package-template/src/sbin/admin-tools/dataset-manager.sh index 668e4e5373..7b3e002788 100755 --- a/components/package-template/src/sbin/admin-tools/dataset-manager.sh +++ b/components/package-template/src/sbin/admin-tools/dataset-manager.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" common_env_path="$script_dir/../.common-env.sh" # shellcheck source=.common-env.sh diff --git a/components/package-template/src/sbin/compress-from-s3.sh b/components/package-template/src/sbin/compress-from-s3.sh index 8cb10e594b..49b9a5e1b6 100755 --- a/components/package-template/src/sbin/compress-from-s3.sh +++ b/components/package-template/src/sbin/compress-from-s3.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" common_env_path="$script_dir/.common-env.sh" # shellcheck source=.common-env.sh diff --git a/components/package-template/src/sbin/compress.sh b/components/package-template/src/sbin/compress.sh index 797255e2e6..be24b32ea4 100755 --- a/components/package-template/src/sbin/compress.sh +++ b/components/package-template/src/sbin/compress.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" common_env_path="$script_dir/.common-env.sh" # shellcheck source=.common-env.sh diff --git a/components/package-template/src/sbin/decompress.sh b/components/package-template/src/sbin/decompress.sh index 10cf529d29..cb89685964 100755 --- a/components/package-template/src/sbin/decompress.sh +++ b/components/package-template/src/sbin/decompress.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" common_env_path="$script_dir/.common-env.sh" # shellcheck source=.common-env.sh diff --git a/components/package-template/src/sbin/search.sh b/components/package-template/src/sbin/search.sh index 1ad1d7efa6..183d9a87af 100755 --- a/components/package-template/src/sbin/search.sh +++ b/components/package-template/src/sbin/search.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" common_env_path="$script_dir/.common-env.sh" # shellcheck source=.common-env.sh diff --git a/components/package-template/src/sbin/start-clp.sh b/components/package-template/src/sbin/start-clp.sh index bc2e360508..11b09d3f6c 100755 --- a/components/package-template/src/sbin/start-clp.sh +++ b/components/package-template/src/sbin/start-clp.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" common_env_path="$script_dir/.common-env.sh" # shellcheck source=.common-env.sh diff --git a/components/package-template/src/sbin/stop-clp.sh b/components/package-template/src/sbin/stop-clp.sh index aa582f84c3..c94c3ac947 100755 --- a/components/package-template/src/sbin/stop-clp.sh +++ b/components/package-template/src/sbin/stop-clp.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" common_env_path="$script_dir/.common-env.sh" # shellcheck source=.common-env.sh From 3af2e4d8df82d4fefdd669b9c0c4a8daf193989b Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 3 Nov 2025 03:04:06 -0500 Subject: [PATCH 44/50] Rename dir -> compose_plugin_dir to avoid overlap with the command dir --- components/package-template/src/sbin/.common-env.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/package-template/src/sbin/.common-env.sh b/components/package-template/src/sbin/.common-env.sh index 0ba9d3c19c..2651568a32 100755 --- a/components/package-template/src/sbin/.common-env.sh +++ b/components/package-template/src/sbin/.common-env.sh @@ -31,15 +31,15 @@ CLP_PWD_HOST="$(pwd 2>/dev/null || echo "")" export CLP_PWD_HOST if [[ -z "${CLP_DOCKER_PLUGIN_DIR:-}" ]]; then - for dir in \ + for compose_plugin_dir in \ "$HOME/.docker/cli-plugins" \ "/mnt/wsl/docker-desktop/cli-tools/usr/local/lib/docker/cli-plugins" \ "/usr/local/lib/docker/cli-plugins" \ "/usr/libexec/docker/cli-plugins"; do - compose_plugin_path="$dir/docker-compose" + compose_plugin_path="$compose_plugin_dir/docker-compose" if [[ -f "$compose_plugin_path" ]]; then - export CLP_DOCKER_PLUGIN_DIR="$dir" + export CLP_DOCKER_PLUGIN_DIR="$compose_plugin_dir" break fi done From 198f398196849ae22627be9b9cb6128d9a9b1bdd Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 3 Nov 2025 03:08:19 -0500 Subject: [PATCH 45/50] add docs for resolve_host_path_in_container --- components/clp-py-utils/clp_py_utils/core.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/clp-py-utils/clp_py_utils/core.py b/components/clp-py-utils/clp_py_utils/core.py index 99a649200d..8ea063cdbe 100644 --- a/components/clp-py-utils/clp_py_utils/core.py +++ b/components/clp-py-utils/clp_py_utils/core.py @@ -82,8 +82,10 @@ def resolve_host_path_in_container(host_path: pathlib.Path) -> pathlib.Path: if link_target.is_absolute(): return CONTAINER_DIR_FOR_HOST_ROOT / link_target.relative_to("/") else: + # If the symlink points to a relative path, resolve it relative to the symlink's parent. return (translated_path.parent / link_target).resolve() except OSError: + # Ignore if reading the symlink fails (e.g., broken link or permission error). pass return translated_path From 3063e85f45ade08a90387453aba17eb9b81f421e Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 4 Nov 2025 10:42:37 -0500 Subject: [PATCH 46/50] format - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- components/clp-package-utils/clp_package_utils/controller.py | 2 +- .../clp_package_utils/scripts/archive_manager.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index ba5cac6aba..522439f588 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -171,7 +171,6 @@ def _set_up_env_for_queue(self) -> EnvVarsDict: validate_queue_config(self._clp_config, logs_dir) resolved_logs_dir = resolve_host_path_in_container(logs_dir) - resolved_logs_dir.mkdir(exist_ok=True, parents=True) _chown_paths_if_root(resolved_logs_dir) @@ -569,6 +568,7 @@ def _set_up_env_for_mcp_server(self) -> EnvVarsDict: logs_dir = self._clp_config.logs_directory / component_name validate_mcp_server_config(self._clp_config, logs_dir) + resolved_logs_dir = resolve_host_path_in_container(logs_dir) resolved_logs_dir.mkdir(parents=True, exist_ok=True) diff --git a/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py b/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py index 570ca7f7f4..81bcdf7e1c 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py +++ b/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py @@ -283,7 +283,9 @@ def main(argv: List[str]) -> int: logger.debug(f"Docker command failed: {shlex.join(cmd)}") # Remove generated files - resolved_generated_config_path_on_host = resolve_host_path_in_container(generated_config_path_on_host) + resolved_generated_config_path_on_host = resolve_host_path_in_container( + generated_config_path_on_host + ) resolved_generated_config_path_on_host.unlink() return ret_code From 306997a89ea044e1f5323e89297364584273c755 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 4 Nov 2025 11:04:30 -0500 Subject: [PATCH 47/50] split volumes into groups and add docs --- tools/deployment/package/docker-compose.runtime.yaml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tools/deployment/package/docker-compose.runtime.yaml b/tools/deployment/package/docker-compose.runtime.yaml index c2e478db29..0c7be272e9 100644 --- a/tools/deployment/package/docker-compose.runtime.yaml +++ b/tools/deployment/package/docker-compose.runtime.yaml @@ -12,6 +12,7 @@ services: CLP_HOME: "${CLP_HOME:?Please set a value.}" CLP_PWD_HOST: "${CLP_PWD_HOST:-${PWD}}" volumes: + # Docker daemon bridge - type: "bind" source: "${CLP_DOCKER_PLUGIN_DIR:-/usr/local/lib/docker/cli-plugins}" target: "/usr/local/lib/docker/cli-plugins" @@ -19,13 +20,15 @@ services: - type: "bind" source: "${CLP_DOCKER_SOCK_PATH:-/var/run/docker.sock}" target: "/var/run/docker.sock" + - type: "bind" + source: "/usr/bin/docker" + target: "/usr/bin/docker" + read_only: true + + # Host filesystem - type: "bind" source: "${CLP_HOME:?Please set a value.}" target: "${CLP_HOME:?Please set a value.}" - type: "bind" source: "/" target: "/mnt/host" - - type: "bind" - source: "/usr/bin/docker" - target: "/usr/bin/docker" - read_only: true From 4f6c5b4286d12e129099fc083b504df3d9af28d9 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 4 Nov 2025 11:08:30 -0500 Subject: [PATCH 48/50] Move resolved webui settings paths declarations closer to their usages --- .../clp_package_utils/controller.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index 522439f588..a1088efefa 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -442,12 +442,6 @@ def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict: server_settings_json_path = ( self._clp_home / "var" / "www" / "webui" / "server" / "dist" / "settings.json" ) - resolved_client_settings_json_path = resolve_host_path_in_container( - client_settings_json_path - ) - resolved_server_settings_json_path = resolve_host_path_in_container( - server_settings_json_path - ) validate_webui_config( self._clp_config, client_settings_json_path, @@ -478,6 +472,9 @@ def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict: "SqlDbClpTablePrefix": table_prefix, "SqlDbCompressionJobsTableName": COMPRESSION_JOBS_TABLE_NAME, } + resolved_client_settings_json_path = resolve_host_path_in_container( + client_settings_json_path + ) client_settings_json = self._read_and_update_settings_json( resolved_client_settings_json_path, client_settings_json_updates ) @@ -533,6 +530,9 @@ def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict: server_settings_json_updates["PrestoHost"] = None server_settings_json_updates["PrestoPort"] = None + resolved_server_settings_json_path = resolve_host_path_in_container( + server_settings_json_path + ) server_settings_json = self._read_and_update_settings_json( resolved_server_settings_json_path, server_settings_json_updates ) From ef627c602b6a072de8529de985c4972f7bf9e158 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 4 Nov 2025 11:21:14 -0500 Subject: [PATCH 49/50] format - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- components/clp-py-utils/clp_py_utils/core.py | 1 + 1 file changed, 1 insertion(+) diff --git a/components/clp-py-utils/clp_py_utils/core.py b/components/clp-py-utils/clp_py_utils/core.py index 8ea063cdbe..b04a4f7aa8 100644 --- a/components/clp-py-utils/clp_py_utils/core.py +++ b/components/clp-py-utils/clp_py_utils/core.py @@ -78,6 +78,7 @@ def resolve_host_path_in_container(host_path: pathlib.Path) -> pathlib.Path: try: if not translated_path.is_symlink(): return translated_path + link_target = translated_path.readlink() if link_target.is_absolute(): return CONTAINER_DIR_FOR_HOST_ROOT / link_target.relative_to("/") From 969fde06b422df0f39e2de87aa99c9d33d8e920e Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 4 Nov 2025 15:50:10 -0500 Subject: [PATCH 50/50] Clarify `HOME` forwarding into the container - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- tools/deployment/package/docker-compose.runtime.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/deployment/package/docker-compose.runtime.yaml b/tools/deployment/package/docker-compose.runtime.yaml index 0c7be272e9..daf72f2623 100644 --- a/tools/deployment/package/docker-compose.runtime.yaml +++ b/tools/deployment/package/docker-compose.runtime.yaml @@ -8,7 +8,11 @@ services: tty: true user: "${CLP_FIRST_PARTY_SERVICE_UID_GID:-1000:999}" environment: + # NOTE: We forward "$HOME" into the container so that if the user specified "~" anywhere in + # their config, the container can resolve it to the relevant path on the host. As a result, + # the container should not rely on "$HOME" or "~" pointing to a path inside the container. HOME: "${HOME:?Please set a value.}" + CLP_HOME: "${CLP_HOME:?Please set a value.}" CLP_PWD_HOST: "${CLP_PWD_HOST:-${PWD}}" volumes: