diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 65aaf4e..cb516b9 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -21,21 +21,13 @@ updates: patterns: - "*" - - package-ecosystem: "pip" - directory: "/src" - schedule: - interval: "weekly" - day: "sunday" - groups: - all-pip: - patterns: - - "*" - - package-ecosystem: "npm" - directory: "/admin" + directory: "/src/labnow-open-web" schedule: interval: "weekly" day: "sunday" + cooldown: + default-days: 7 groups: all-npm: patterns: diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 469a287..e14c021 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -17,6 +17,7 @@ concurrency: cancel-in-progress: true env: + TZ: Asia/Shanghai BUILDKIT_PROGRESS: "plain" # Full logs for CI build. REGISTRY_SRC: ${{ vars.REGISTRY_SRC || 'docker.io' }} # For BASE_NAMESPACE of images: where to pull base images from, docker.io or other source registry URL. REGISTRY_DST: ${{ vars.REGISTRY_DST || 'quay.io' }} # For tags of built images: where to push images to, docker.io or other destination registry URL. @@ -31,7 +32,7 @@ env: COMMIT_SHA: ${{ github.sha }} # COMMIT_SHA="$(git rev-parse --short HEAD)" jobs: - semgrep: + job-semgrep: runs-on: ubuntu-latest permissions: {"contents": "read", "pull-requests": "write"} container: @@ -59,7 +60,7 @@ jobs: - run: | source ./tool/tool.sh build_image labnow-open-dev latest ./src/labnow-open.Dockerfile \ - --build-arg "PROFILE_LOCALIZE=default" --build-arg "BASE_IMG=labnow/developer:latest" + --build-arg "PROFILE_LOCALIZE=default" --build-arg "BASE_IMG=developer:latest" push_image labnow-open-dev job-data-science: @@ -71,7 +72,7 @@ jobs: - run: | source ./tool/tool.sh build_image labnow-open-data-science latest ./src/labnow-open.Dockerfile \ - --build-arg "PROFILE_LOCALIZE=default" --build-arg "BASE_IMG=labnow/data-science-dev:latest" + --build-arg "PROFILE_LOCALIZE=default" --build-arg "BASE_IMG=data-science-dev:latest" push_image labnow-open-data-science job-openclaw: @@ -83,12 +84,12 @@ jobs: - run: | source ./tool/tool.sh build_image labnow-open-openclaw latest ./src/labnow-open.Dockerfile \ - --build-arg "PROFILE_LOCALIZE=default" --build-arg "BASE_IMG=labnow/openclaw:latest" + --build-arg "PROFILE_LOCALIZE=default" --build-arg "BASE_IMG=openclaw:latest" push_image labnow-open-openclaw ## Sync all images in this build (listed by "names") to mirror registry. - sync_images: + job-sync-images: needs: ["job-developer", "job-data-science", "job-openclaw"] runs-on: ubuntu-latest permissions: {"contents": "read", "pull-requests": "write"} diff --git a/src/labnow-open-etc/Caddyfile b/src/labnow-open-etc/Caddyfile index 417a925..3b36d95 100644 --- a/src/labnow-open-etc/Caddyfile +++ b/src/labnow-open-etc/Caddyfile @@ -1,5 +1,5 @@ :80 { - log { # Enable access logging for all requests + log { output file /dev/null { roll_size 10mb roll_keep 5 @@ -8,60 +8,97 @@ format json } - # Add a global header to all responses + # Global response headers header { - -Server - Location-Base {$URL_PREFIX:} + -Server + Location-Base {$URL_PREFIX:} } - # Redirect the root path to the user home page - @root path_regexp root ^{$URL_PREFIX:}/?$ + # Redirect the application root to the home page + @root path {$URL_PREFIX:} {$URL_PREFIX:}/ redir @root {$URL_PREFIX:}home 307 - # Handle user home paths - @path_user_home path_regexp ^{$URL_PREFIX}home/.*$ + # Home + @path_user_home path {$URL_PREFIX:}home/* handle @path_user_home { uri strip_prefix {$URL_PREFIX:}home reverse_proxy localhost:9001 - # root * /opt/labnow-open/web - # file_server } - @path_user_home_no_slash path_regexp ^{$URL_PREFIX:}home$ - redir @path_user_home_no_slash {path}/ 308 + @path_user_home_no_slash path {$URL_PREFIX:}home + redir @path_user_home_no_slash {path}/ 308 - @path_user_vscode path_regexp ^{$URL_PREFIX:}vscode/.*$ - handle @path_user_vscode { # Proxy requests to vscode, strip prefix + # VS Code + @path_user_vscode path {$URL_PREFIX:}vscode/* + handle @path_user_vscode { uri strip_prefix {$URL_PREFIX:}vscode reverse_proxy localhost:9999 } - @path_user_vscode_no_slash path_regexp ^{$URL_PREFIX:}vscode$ - redir @path_user_vscode_no_slash {path}/ 308 # Redirect paths ending in "vscode" to "vscode/" + @path_user_vscode_no_slash path {$URL_PREFIX:}vscode + redir @path_user_vscode_no_slash {path}/ 308 - - @path_user_rserver path_regexp ^{$URL_PREFIX:}rserver/.*$ - handle @path_user_rserver { # Proxy requests to rserver, strip prefix + # RStudio Server + @path_user_rserver path {$URL_PREFIX:}rserver/* + handle @path_user_rserver { uri strip_prefix {$URL_PREFIX:}rserver reverse_proxy localhost:8787 { - header_up -X-Forwarded-Port + header_up -X-Forwarded-Port } } - @path_user_rserver_no_slash path_regexp ^{$URL_PREFIX:}rserver$ - redir @path_user_rserver_no_slash {path}/ 308 # Redirect paths ending in "rserver" to "rserver/" - + @path_user_rserver_no_slash path {$URL_PREFIX:}rserver + redir @path_user_rserver_no_slash {path}/ 308 - @path_user_rshiny path_regexp ^{$URL_PREFIX:}rshiny/.*$ - handle @path_user_rshiny { # Proxy requests to rshiny, strip prefix + # Shiny Server + @path_user_rshiny path {$URL_PREFIX:}rshiny/* + handle @path_user_rshiny { uri strip_prefix {$URL_PREFIX:}rshiny reverse_proxy localhost:3838 { - header_up -X-Forwarded-Port + header_up -X-Forwarded-Port } } - @path_user_rshiny_no_slash path_regexp ^{$URL_PREFIX:}rshiny$ - redir @path_user_rshiny_no_slash {path}/ 308 # Redirect paths ending in "rshiny" to "rshiny/" + @path_user_rshiny_no_slash path {$URL_PREFIX:}rshiny + redir @path_user_rshiny_no_slash {path}/ 308 + + # OpenClaw + # Redirect non-WebSocket requests to the canonical trailing-slash URL + @path_user_openclaw_no_slash { + path {$URL_PREFIX:}openclaw + not header Connection *Upgrade* + } + redir @path_user_openclaw_no_slash {path}/ 308 + + @path_user_openclaw path {$URL_PREFIX:}openclaw/* + handle @path_user_openclaw { + uri strip_prefix {$URL_PREFIX:}openclaw + reverse_proxy localhost:18789 + } + + # Hermes + @path_user_hermes_no_slash path {$URL_PREFIX:}hermes + redir @path_user_hermes_no_slash {path}/ 308 + + @path_user_hermes path {$URL_PREFIX:}hermes/* + handle @path_user_hermes { + uri strip_prefix {$URL_PREFIX:}hermes + reverse_proxy localhost:9119 + } + + # Selkies + @path_user_selkies_no_slash path {$URL_PREFIX:}selkies + redir @path_user_selkies_no_slash {path}/ 308 + + @path_user_selkies path {$URL_PREFIX:}selkies/* + handle @path_user_selkies { + uri strip_prefix {$URL_PREFIX:}selkies + reverse_proxy localhost:8080 + } + # Health check + @path_user_api_healthcheck path {$URL_PREFIX:}api + respond @path_user_api_healthcheck "OK" 200 - handle /* { # Proxy all other requests to a default backend server on port 8888 for jupyter + # Default backend + handle { reverse_proxy localhost:8888 } } diff --git a/src/labnow-open-web/src/App.jsx b/src/labnow-open-web/src/App.jsx index dea475c..deebe26 100644 --- a/src/labnow-open-web/src/App.jsx +++ b/src/labnow-open-web/src/App.jsx @@ -62,6 +62,13 @@ const PROGRAM_DEFINITIONS = { description: "Run Shiny applications for interactive dashboards. Useful for sharing data apps with your team.", }, + openclaw: { + displayName: "OpenClaw", + link: "/openclaw/", + logo: "logo-openclaw.svg", + description: + "OpenClaw AI Agent platform. Build and deploy intelligent agents with natural language.", + }, }; function NotificationBar({ notice, onClose }) { diff --git a/src/labnow-open.Dockerfile b/src/labnow-open.Dockerfile index 010eea6..b240fe6 100644 --- a/src/labnow-open.Dockerfile +++ b/src/labnow-open.Dockerfile @@ -1,6 +1,6 @@ # Use the existing image as the base ARG BASE_NAMESPACE="quay.io" -ARG BASE_IMG="labnow/developer:latest" +ARG BASE_IMG="developer:latest" # this ENV will be used in /opt/utils/script-localize.sh ARG PROFILE_LOCALIZE="aliyun-pub" @@ -10,8 +10,8 @@ ARG PROFILE_LOCALIZE="aliyun-pub" ENV PROFILE_LOCALIZE=${PROFILE_LOCALIZE} -COPY ./src/labnow-open-web /tmp/labnow-open-web -COPY ./src/labnow-open-etc /opt/labnow-open/etc +COPY ./labnow-open-web /tmp/labnow-open-web +COPY ./labnow-open-etc /opt/labnow-open/etc RUN set -eux \ && source /opt/utils/script-localize.sh ${PROFILE_LOCALIZE} \ && source /opt/utils/script-setup-core.sh && setup_node_pnpm 11 \ @@ -29,20 +29,26 @@ ENV PROFILE_LOCALIZE=${PROFILE_LOCALIZE} COPY --from=builder /opt/labnow-open/ /opt/labnow-open/ RUN set -eux && source /opt/utils/script-localize.sh ${PROFILE_LOCALIZE} \ - # handle control scripts and extensions + ## handle control scripts and extensions && (type supervisord || (source /opt/utils/script-setup-sys.sh && setup_supervisord && echo "Supervisord installed")) \ && (type caddy || (source /opt/utils/script-setup-net.sh && setup_caddy && echo "Caddy installed")) \ && mkdir -pv /etc/supervisord && ln -sf /opt/labnow-open/etc/supervisord.conf /etc/supervisord/ \ && mkdir -pv /etc/caddy && ln -sf /opt/labnow-open/etc/Caddyfile /etc/caddy/ \ + ## create start-supervisord.sh and start-caddy.sh scripts if not existing, and make them executable + && ([ ! -f /usr/local/bin/start-supervisord.sh ] && printf '#!/bin/bash\nLOG_FORMAT=json exec supervisord -c /etc/supervisord/supervisord.conf\n' > /usr/local/bin/start-supervisord.sh || true ) \ + && ([ ! -f /usr/local/bin/start-caddy.sh ] && printf '#!/bin/bash\ncaddy run --config /etc/caddy/Caddyfile\n' > /usr/local/bin/start-caddy.sh || true ) \ + && chmod +x /usr/local/bin/start-caddy.sh \ + && chmod +x /usr/local/bin/start-supervisord.sh \ + ## components that are optional, only add them to supervisord if they exist && (type jupyter && echo '{"ServerApp":{"ip":"0.0.0.0","port":8888,"root_dir":"/root","default_url":"/home","token":"","password":"","allow_root":true,"allow_origin":"*","open_browser":false}}' > /opt/conda/etc/jupyter/jupyter_server_config.json || true) \ && (type jupyter && printf "[program:jupyter]\ncommand=/usr/local/bin/start-jupyterlab.sh\n" >> /etc/supervisord/supervisord.conf || true) \ && (type code-server && printf "[program:vscode]\ncommand=/usr/local/bin/start-code-server.sh\n" >> /etc/supervisord/supervisord.conf || true) \ && (type rserver && printf "[program:rserver]\ncommand=/usr/local/bin/start-rserver.sh\n" >> /etc/supervisord/supervisord.conf || true) \ && (type shiny-server && printf "[program:rshiny]\ncommand=/usr/local/bin/start-shiny-server.sh\n" >> /etc/supervisord/supervisord.conf || true) \ - && (type openclaw && printf "[program:openclaw]\ncommand=/usr/local/bin/start-openclaw.sh\n" >> /etc/supervisord/supervisord.conf || true) \ - && (type hermes && printf "[program:hermes]\ncommand=/usr/local/bin/start-hermes.sh\n" >> /etc/supervisord/supervisord.conf || true) \ + ## optional agent components: openclaw and hermes + && (type openclaw && printf "[program:openclaw]\ncommand=/usr/local/bin/start-openclaw.sh\nautostart=true\n" >> /etc/supervisord/supervisord.conf || true) \ + && (type hermes && printf "[program:hermes]\ncommand=/usr/local/bin/start-hermes.sh\nautostart=true\n" >> /etc/supervisord/supervisord.conf || true) \ # cleanup of any temporary or cache files to keep the image size down - && rm -rf /opt/conda/share/jupyter/lab/staging \ && source /opt/utils/script-utils.sh && install__clean WORKDIR $HOME_DIR diff --git a/tool/tool.sh b/tool/tool.sh index dfc686e..550a9ff 100644 --- a/tool/tool.sh +++ b/tool/tool.sh @@ -1,43 +1,58 @@ #!/bin/bash -set -xu +set -eux + +# If not executed in GitHub Action, run script in project root, and export the following 3 variables manually: +# export REGISTRY_SRC='quay.io' # For BASE_NAMESPACE of images: where to pull base images from, docker.io or other source registry URL. +# export REGISTRY_DST='quay.io' # For tags of built images: where to push images to, docker.io or other destination registry URL. +# export CI_PROJECT_NAME='LabNow/lab-dev' CI_PROJECT_NAME=${CI_PROJECT_NAME:-$GITHUB_REPOSITORY} CI_PROJECT_BRANCH=${GITHUB_HEAD_REF:-"main"} CI_PROJECT_SPACE=$(echo "${CI_PROJECT_BRANCH}" | cut -f1 -d'/') -if [ "${CI_PROJECT_BRANCH}" = "main" ] ; then - # If on the main branch, docker images namespace will be same as CI_PROJECT_NAME's name space - export CI_PROJECT_NAMESPACE="$(dirname ${CI_PROJECT_NAME})" ; -else - # not main branch, docker namespace = {CI_PROJECT_NAME's name space} + "-" + {1st substr before / in CI_PROJECT_SPACE} - export CI_PROJECT_NAMESPACE="$(dirname ${CI_PROJECT_NAME})0${CI_PROJECT_SPACE}" ; -fi +# If on the main branch, image namespace will be same as CI_PROJECT_NAME's name space; +# else (not main branch), image namespace = {CI_PROJECT_NAME's name space} + "0" + {1st substr before / in CI_PROJECT_SPACE}. +[ "${CI_PROJECT_BRANCH}" = "main" ] && NAMESPACE_SUFFIX="" || NAMESPACE_SUFFIX="0${CI_PROJECT_SPACE}" ; +export CI_PROJECT_NAMESPACE="$(dirname ${CI_PROJECT_NAME})${NAMESPACE_SUFFIX}" ; export IMG_NAMESPACE=$(echo "${CI_PROJECT_NAMESPACE}" | awk '{print tolower($0)}') -export IMG_PREFIX_SRC=$(echo "${REGISTRY_SRC:-"docker.io"}" | awk '{print tolower($0)}') -export IMG_PREFIX_DST=$(echo "${REGISTRY_DST:-"docker.io"}/${CI_PROJECT_NAMESPACE}" | awk '{print tolower($0)}') +export IMG_PREFIX_SRC=$(echo "${REGISTRY_SRC:-"docker.io"}/${IMG_NAMESPACE}" | awk '{print tolower($0)}') +export IMG_PREFIX_DST=$(echo "${REGISTRY_DST:-"docker.io"}/${IMG_NAMESPACE}" | awk '{print tolower($0)}') export TAG_SUFFIX="-$(git rev-parse --short HEAD)" -echo "--------> CI_PROJECT_NAMESPACE=${CI_PROJECT_NAMESPACE}" # use different namespace for dev/prd +echo "--------> CI_PROJECT_NAMESPACE=${CI_PROJECT_NAMESPACE}" +echo "--------> DOCKER_IMG_NAMESPACE=${IMG_NAMESPACE}" echo "--------> DOCKER_IMG_PREFIX_SRC=${IMG_PREFIX_SRC}" echo "--------> DOCKER_IMG_PREFIX_DST=${IMG_PREFIX_DST}" echo "--------> DOCKER_TAG_SUFFIX=${TAG_SUFFIX}" -[ ! -f /etc/docker/daemon.json ] && sudo tee /etc/docker/daemon.json > /dev/null <<< '{}' -jq '.experimental=true | ."data-root"="/mnt/docker"' /etc/docker/daemon.json > /tmp/daemon.json && sudo mv /tmp/daemon.json /etc/docker/ -( sudo service docker restart || true ) && cat /etc/docker/daemon.json && docker info build_image() { echo "$@" ; - IMG=$1; TAG=$2; FILE=$3; shift 3; VER=$(date +%Y.%m%d.%H%M)${TAG_SUFFIX}; WORKDIR="$(pwd)"; - docker build --compress --force-rm=true -t "${IMG_PREFIX_DST}/${IMG}:${TAG}" -f "$FILE" --build-arg "BASE_NAMESPACE=${IMG_PREFIX_SRC}" "$@" "${WORKDIR}" ; - docker tag "${IMG_PREFIX_DST}/${IMG}:${TAG}" "${IMG_PREFIX_DST}/${IMG}:${VER}" ; + IMG=$1; TAG=$2; FILE=$3; shift 3; VER=$(date +%Y.%m%d.%H%M)${TAG_SUFFIX}; WORKDIR="$(dirname $FILE)"; + docker build --compress --force-rm=true -t "${IMG_PREFIX_DST}/${IMG}:${TAG}" -f "$FILE" --build-arg "BASE_NAMESPACE=${IMG_PREFIX_SRC}" "$@" "${WORKDIR}" + docker tag "${IMG_PREFIX_DST}/${IMG}:${TAG}" "${IMG_PREFIX_DST}/${IMG}:${VER}" + echo "${IMG_PREFIX_DST}/${IMG}:${TAG}" +} + +build_image_no_tag() { + echo "$@" ; + IMG=$1; TAG=$2; FILE=$3; shift 3; WORKDIR="$(dirname $FILE)"; + docker build --compress --force-rm=true -t "${IMG_PREFIX_DST}/${IMG}:${TAG}" -f "$FILE" --build-arg "BASE_NAMESPACE=${IMG_PREFIX_SRC}" "$@" "${WORKDIR}" + echo "${IMG_PREFIX_DST}/${IMG}:${TAG}" +} + +alias_image() { + IMG_1=$1; TAG_1=$2; IMG_2=$3; TAG_2=$4; shift 4; VER=$(date +%Y.%m%d.%H%M)${TAG_SUFFIX}; + docker tag "${IMG_PREFIX_DST}/${IMG_1}:${TAG_1}" "${IMG_PREFIX_DST}/${IMG_2}:${TAG_2}" + docker tag "${IMG_PREFIX_DST}/${IMG_2}:${TAG_2}" "${IMG_PREFIX_DST}/${IMG_2}:${VER}" } push_image() { KEYWORD="${1:-second}"; docker image prune --force && docker images | sort; - IMAGES=$(docker images --format "table {{.Repository}}\t{{.Tag}}\t{{.CreatedSince}}" | grep "${KEYWORD}" | awk '{print $1 ":" $2}') ; + IMAGES=$(docker images --format "{{.Repository}}\t{{.Tag}}\t{{.CreatedSince}}" | grep "${KEYWORD}" | awk '{print $1 ":" $2}') ; + [ -n "${IMAGES}" ] || { echo "!! No images matched keyword: ${KEYWORD}"; return 1; } echo "$DOCKER_REGISTRY_PASSWORD" | docker login "${REGISTRY_DST}" -u "$DOCKER_REGISTRY_USERNAME" --password-stdin ; for IMG in $(echo "${IMAGES}" | tr " " "\n") ; do @@ -48,24 +63,37 @@ push_image() { } clear_images() { - KEYWORD=${1:-'days ago\|weeks ago\|months ago\|years ago'}; # if no keyword is provided, clear all images build days ago + KEYWORD=${1:-'days ago\|weeks ago\|months ago\|years ago'}; # if no keyword is provided, clear all images built days ago IMGS_1=$(docker images | grep "${KEYWORD}" | awk '{print $1 ":" $2}') ; IMGS_2=$(docker images | grep "${KEYWORD}" | awk '{print $3}') ; for IMG in $(echo "$IMGS_1 $IMGS_2" | tr " " "\n") ; do - docker rmi "${IMG}" || true; status=$?; echo "[${status}] image removed > ${IMG}"; + docker rmi "${IMG}" ; status=$?; echo "[${status}] image removed > ${IMG}"; done docker image prune --force && docker images ; } remove_folder() { - sudo du -h -d1 "$1" || true ; - sudo rm -rf "$1" || true ; + for dir in "$@"; do + if [ -d "$dir" ]; then + echo "Removing folder: $dir" ; + sudo du -h -d1 "$dir" || true ; + sudo rm -rf "$dir" || true ; + else + echo "Warn: directory not found: $dir" ; + fi + done } free_diskspace() { - remove_folder /usr/share/dotnet - remove_folder /usr/local/lib/android - df -h + remove_folder /usr/share/dotnet ; # /usr/local/lib/android /var/lib/docker + df -h ; +} + +setup_github_actions() { + [ ! -f /etc/docker/daemon.json ] && sudo tee /etc/docker/daemon.json > /dev/null <<< '{}' ; + jq '.experimental=true | ."data-root"="/mnt/docker"' /etc/docker/daemon.json > /tmp/daemon.json && sudo mv /tmp/daemon.json /etc/docker/ ; + ( sudo service docker restart || true ) && cat /etc/docker/daemon.json && docker info ; } +[ ${GITHUB_ACTIONS:-"false"} = "true" ] && echo "Running in GitHub Actions and Setup Env: $(setup_github_actions)" || echo "Not running in GitHub Action." ;