From 61591de98792a7d4f8c74980ea5409ed9fb9dfb8 Mon Sep 17 00:00:00 2001 From: Jeff Garzik Date: Tue, 10 Mar 2026 02:47:50 +0000 Subject: [PATCH 1/2] feat: migrate bot containers to Ubuntu 24.04 Noble with recreate API Convert Dockerfile.botenv from single-stage Debian Bookworm (inherited from OpenClaw) to a multi-stage build using Ubuntu Noble as the runtime base. This gives significantly better HDL/FPGA package coverage (nextpnr-ecp5, yosys-abc, yosys-plugin-ghdl, ghdl-tools) and adds libboost-all-dev. OpenClaw app and Node.js runtime are copied from the upstream image in the first stage. Add DockerService.recreateContainer() which inspects an existing container, preserves its full config (env, binds, ports, network, labels, healthcheck), removes it, and creates a new one with a different image. Exposed via POST /api/bots/:hostname/recreate and POST /api/admin/recreate-all for bulk image upgrades. Co-Authored-By: Claude Opus 4.6 --- Dockerfile.botenv | 50 ++++++++++-- README.md | 2 +- src/server.ts | 55 +++++++++++++ src/services/DockerService.test.ts | 127 +++++++++++++++++++++++++++++ src/services/DockerService.ts | 63 ++++++++++++++ 5 files changed, 288 insertions(+), 9 deletions(-) diff --git a/Dockerfile.botenv b/Dockerfile.botenv index a6151e2..2743007 100644 --- a/Dockerfile.botenv +++ b/Dockerfile.botenv @@ -1,12 +1,35 @@ -# Extended base image for bot containers with build tools and utilities -# Build: docker compose build botenv +# Extended bot container: Ubuntu Noble base with OpenClaw + dev tools +# Build: docker compose --profile build build botenv # Usage: Automatically used by botmaker when spawning bot containers +# Stage 1: Source OpenClaw app files and Node.js runtime from upstream ARG BASE_IMAGE=ghcr.io/openclaw/openclaw:latest -FROM ${BASE_IMAGE} +FROM ${BASE_IMAGE} AS openclaw-source -# Switch to root for package installation -USER root +# Stage 2: Ubuntu Noble base with Node.js + OpenClaw + dev tools +FROM ubuntu:noble + +# Create node user (UID 1000) to match OpenClaw expectations +# Ubuntu Noble ships with ubuntu:1000 — remove it first, then create node:1000 +RUN userdel --remove ubuntu 2>/dev/null; groupdel ubuntu 2>/dev/null; \ + groupadd --gid 1000 node \ + && useradd --uid 1000 --gid node --shell /bin/bash --create-home node + +# Copy Node.js runtime from OpenClaw image +COPY --from=openclaw-source /usr/local/bin/node /usr/local/bin/ +COPY --from=openclaw-source /usr/local/bin/docker-entrypoint.sh /usr/local/bin/ +COPY --from=openclaw-source /usr/local/include/node /usr/local/include/node +COPY --from=openclaw-source /usr/local/lib/node_modules /usr/local/lib/node_modules + +# Symlink npm/npx/corepack (they're wrappers into lib/node_modules) +RUN ln -s ../lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm \ + && ln -s ../lib/node_modules/npm/bin/npx-cli.js /usr/local/bin/npx \ + && ln -s ../lib/node_modules/corepack/dist/corepack.js /usr/local/bin/corepack \ + && corepack enable + +# Copy OpenClaw application +COPY --from=openclaw-source --chown=node:node /app /app +WORKDIR /app # Install build tools, languages, and utilities in single layer for cache efficiency RUN apt-get update && apt-get install -y --no-install-recommends \ @@ -27,13 +50,19 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libncurses-dev \ libpcre2-dev \ libprotobuf-dev protobuf-compiler \ + # C++ Boost libraries + libboost-all-dev \ # CLI utilities jq ripgrep fd-find less \ unzip zip xz-utils zstd \ wget tree file patch \ openssh-client rsync \ - # HDL/FPGA tools - yosys ghdl \ + # HDL/FPGA tools (expanded for Ubuntu Noble — no Gowin) + yosys yosys-abc yosys-dev yosys-plugin-ghdl \ + ghdl ghdl-common ghdl-tools \ + nextpnr-ice40 nextpnr-ice40-chipdb \ + nextpnr-ecp5 nextpnr-ecp5-chipdb \ + nextpnr-generic \ # Process/network tools procps lsof \ iproute2 netcat-openbsd dnsutils \ @@ -49,5 +78,10 @@ RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \ && rustc --version \ && chmod -R a+w $RUSTUP_HOME $CARGO_HOME -# Switch back to non-root user +# Replicate OpenClaw container settings +ENV NODE_ENV=production +ENTRYPOINT ["docker-entrypoint.sh"] +CMD ["node", "openclaw.mjs", "gateway", "--allow-unconfigured"] + +# Switch to non-root user USER node diff --git a/README.md b/README.md index 805b544..3115d1f 100644 --- a/README.md +++ b/README.md @@ -203,7 +203,7 @@ curl -X POST -H "Authorization: Bearer $TOKEN" http://localhost:7100/api/logout | `DATA_DIR` | ./data | Database and bot workspaces | | `SECRETS_DIR` | ./secrets | Per-bot secret storage | | `BOTENV_IMAGE` | botmaker-env:latest | Bot container image (built from botenv) | -| `OPENCLAW_BASE_IMAGE` | ghcr.io/openclaw/openclaw:latest | Base image for botenv | +| `OPENCLAW_BASE_IMAGE` | ghcr.io/openclaw/openclaw:latest | OpenClaw source image (multi-stage copy into Ubuntu Noble) | | `BOT_PORT_START` | 19000 | Starting port for bot containers | | `SESSION_EXPIRY_MS` | 86400000 | Session expiry in milliseconds (default 24h) | diff --git a/src/server.ts b/src/server.ts index abdde8a..891f903 100644 --- a/src/server.ts +++ b/src/server.ts @@ -763,6 +763,61 @@ export async function buildServer(): Promise { } }); + // Recreate bot container with current image (e.g., after botenv rebuild) + server.post<{ Params: { hostname: string } }>('/api/bots/:hostname/recreate', async (request, reply) => { + const bot = getBotByHostname(request.params.hostname); + + if (!bot) { + reply.code(404); + return { error: 'Bot not found' }; + } + + try { + const newContainerId = await docker.recreateContainer(bot.hostname, config.openclawImage); + updateBot(bot.id, { container_id: newContainerId, image_version: config.openclawImage }); + await docker.startContainer(bot.hostname); + updateBot(bot.id, { status: 'running' }); + + return { success: true, status: 'running', containerId: newContainerId, image: config.openclawImage }; + } catch (err) { + if (err instanceof ContainerError) { + reply.code(500); + return { error: `Failed to recreate container: ${err.message}` }; + } + throw err; + } + }); + + // Recreate all bot containers with current image + server.post('/api/admin/recreate-all', async () => { + const bots = listBots(); + const managedContainers = await docker.listManagedContainers(); + const containerHostnames = new Set(managedContainers.map(c => c.hostname)); + + const results: { hostname: string; success: boolean; error?: string }[] = []; + + for (const bot of bots) { + // Only recreate bots that have an existing container + if (!containerHostnames.has(bot.hostname)) { + results.push({ hostname: bot.hostname, success: false, error: 'No container found' }); + continue; + } + + try { + const newContainerId = await docker.recreateContainer(bot.hostname, config.openclawImage); + updateBot(bot.id, { container_id: newContainerId, image_version: config.openclawImage }); + await docker.startContainer(bot.hostname); + updateBot(bot.id, { status: 'running' }); + results.push({ hostname: bot.hostname, success: true }); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + results.push({ hostname: bot.hostname, success: false, error: msg }); + } + } + + return { image: config.openclawImage, results }; + }); + // Approve a Telegram pairing code server.post<{ Params: { hostname: string }; Body: { code?: string } }>( '/api/bots/:hostname/pair', diff --git a/src/services/DockerService.test.ts b/src/services/DockerService.test.ts index 8087548..dc9e663 100644 --- a/src/services/DockerService.test.ts +++ b/src/services/DockerService.test.ts @@ -7,12 +7,16 @@ const mockExec = vi.fn(); const mockExecStart = vi.fn(); const mockExecInspect = vi.fn(); const mockGetContainer = vi.fn(); +const mockCreateContainer = vi.fn(); +const mockListContainers = vi.fn(); const mockDemuxStream = vi.fn(); vi.mock('dockerode', () => { return { default: vi.fn().mockImplementation(() => ({ getContainer: mockGetContainer, + createContainer: mockCreateContainer, + listContainers: mockListContainers, modem: { demuxStream: mockDemuxStream, }, @@ -149,3 +153,126 @@ describe('DockerService.execCommand', () => { expect(result.exitCode).toBe(-1); }); }); + +describe('DockerService.recreateContainer', () => { + let docker: DockerService; + const mockStop = vi.fn(); + const mockRemove = vi.fn(); + const mockInspect = vi.fn(); + + const fakeInspectResult = { + Config: { + Cmd: ['node', 'openclaw.mjs', 'gateway'], + Env: ['BOT_ID=123', 'PORT=19000', 'OPENCLAW_STATE_DIR=/app/botdata'], + ExposedPorts: { '8080/tcp': {} }, + Labels: { + 'botmaker.managed': 'true', + 'botmaker.bot-id': 'uuid-123', + 'botmaker.bot-hostname': 'bob', + }, + Healthcheck: { + Test: ['CMD', 'curl', '-sf', 'http://localhost:8080/'], + Interval: 2_000_000_000, + Timeout: 3_000_000_000, + Retries: 30, + StartPeriod: 5_000_000_000, + }, + }, + HostConfig: { + Binds: [ + '/data/secrets/bob:/run/secrets:ro', + '/data/bots/bob:/app/botdata:rw', + '/data/bots/bob/sandbox:/app/workspace:rw', + ], + PortBindings: { '8080/tcp': [{ HostIp: '127.0.0.1', HostPort: '19000' }] }, + RestartPolicy: { Name: 'unless-stopped' }, + NetworkMode: 'bm-internal', + ExtraHosts: null, + }, + }; + + beforeEach(() => { + vi.clearAllMocks(); + docker = new DockerService(); + + mockGetContainer.mockReturnValue({ + inspect: mockInspect, + stop: mockStop, + remove: mockRemove, + }); + mockInspect.mockResolvedValue(fakeInspectResult); + mockStop.mockResolvedValue(undefined); + mockRemove.mockResolvedValue(undefined); + mockCreateContainer.mockResolvedValue({ id: 'new-container-id-456' }); + }); + + it('should inspect old container, remove it, and create new one with new image', async () => { + const newId = await docker.recreateContainer('bob', 'botmaker-env:v2'); + + expect(newId).toBe('new-container-id-456'); + + // Should have inspected and stopped the old container + expect(mockGetContainer).toHaveBeenCalledWith('botmaker-bob'); + expect(mockStop).toHaveBeenCalledWith({ t: 10 }); + expect(mockRemove).toHaveBeenCalled(); + + // Should create new container with new image but same config + expect(mockCreateContainer).toHaveBeenCalledWith( + expect.objectContaining({ + name: 'botmaker-bob', + Image: 'botmaker-env:v2', + Cmd: fakeInspectResult.Config.Cmd, + Env: fakeInspectResult.Config.Env, + ExposedPorts: fakeInspectResult.Config.ExposedPorts, + Labels: fakeInspectResult.Config.Labels, + Healthcheck: fakeInspectResult.Config.Healthcheck, + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + HostConfig: expect.objectContaining({ + Binds: fakeInspectResult.HostConfig.Binds, + PortBindings: fakeInspectResult.HostConfig.PortBindings, + RestartPolicy: fakeInspectResult.HostConfig.RestartPolicy, + NetworkMode: 'bm-internal', + }), + }), + ); + }); + + it('should handle container already stopped (304)', async () => { + mockStop.mockRejectedValue({ statusCode: 304 }); + + const newId = await docker.recreateContainer('bob', 'botmaker-env:v2'); + + expect(newId).toBe('new-container-id-456'); + expect(mockRemove).toHaveBeenCalled(); + expect(mockCreateContainer).toHaveBeenCalled(); + }); + + it('should preserve ExtraHosts when present', async () => { + mockInspect.mockResolvedValue({ + ...fakeInspectResult, + HostConfig: { + ...fakeInspectResult.HostConfig, + ExtraHosts: ['host.docker.internal:host-gateway'], + }, + }); + + await docker.recreateContainer('bob', 'botmaker-env:v2'); + + expect(mockCreateContainer).toHaveBeenCalledWith( + expect.objectContaining({ + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + HostConfig: expect.objectContaining({ + ExtraHosts: ['host.docker.internal:host-gateway'], + }), + }), + ); + }); + + it('should throw ContainerError when container not found', async () => { + mockGetContainer.mockReturnValue({ + inspect: vi.fn().mockRejectedValue({ statusCode: 404, message: 'no such container' }), + }); + + await expect(docker.recreateContainer('nonexistent', 'img')).rejects.toThrow(ContainerError); + }); +}); diff --git a/src/services/DockerService.ts b/src/services/DockerService.ts index 2973663..43b84a7 100644 --- a/src/services/DockerService.ts +++ b/src/services/DockerService.ts @@ -207,6 +207,69 @@ export class DockerService { } } + /** + * Recreates a container with a new image, preserving all config. + * Inspects the existing container, removes it, creates a new one with the same + * env/binds/ports/network/labels/healthcheck but the specified image. + * + * @param hostname - Hostname of the bot + * @param newImage - Docker image to use for the new container + * @returns New container ID + */ + async recreateContainer(hostname: string, newImage: string): Promise { + const containerName = `botmaker-${hostname}`; + + try { + const container = this.docker.getContainer(containerName); + const info = await container.inspect(); + + // Extract config from existing container (cast via unknown for dockerode types) + const oldConfig = info.Config as unknown as { + Cmd: string[]; Env: string[]; + ExposedPorts: Record>; Labels: Record; + Healthcheck: Docker.HealthConfig; + }; + const oldHostConfig = info.HostConfig as unknown as { + Binds: string[]; PortBindings: Record; + RestartPolicy: { Name: string }; NetworkMode: string; + ExtraHosts: string[] | null; + }; + + // Stop and remove the old container + try { + await container.stop({ t: 10 }); + } catch (stopErr) { + const dockerErr = stopErr as { statusCode?: number }; + if (dockerErr.statusCode !== 304 && dockerErr.statusCode !== 404) { + throw stopErr; + } + } + await container.remove(); + + // Create new container with same config but new image + const newContainer = await this.docker.createContainer({ + name: containerName, + Image: newImage, + Cmd: oldConfig.Cmd, + Env: oldConfig.Env, + ExposedPorts: oldConfig.ExposedPorts, + Labels: oldConfig.Labels, + Healthcheck: oldConfig.Healthcheck, + HostConfig: { + Binds: oldHostConfig.Binds, + PortBindings: oldHostConfig.PortBindings, + RestartPolicy: oldHostConfig.RestartPolicy, + NetworkMode: oldHostConfig.NetworkMode, + ...(oldHostConfig.ExtraHosts?.length ? { ExtraHosts: oldHostConfig.ExtraHosts } : {}), + } + }); + + return newContainer.id; + } catch (err) { + throw wrapDockerError(err, hostname); + } + } + /** * Gets the status of a container for a bot. * From 3a9f2bcb744f74dc3c59518bce3e76f35c4a0c26 Mon Sep 17 00:00:00 2001 From: Jeff Garzik Date: Thu, 12 Mar 2026 02:57:11 +0000 Subject: [PATCH 2/2] feat: add Playwright + Chromium to bot environment with 1GB ShmSize Install Playwright and Chromium system dependencies in botenv container for browser automation. Set ShmSize to 1GB in container HostConfig to prevent Chromium OOM crashes. Harden recreateContainer() to tolerate 404 on remove and update bot status on failure. Document recreate API endpoints in README. Co-Authored-By: Claude Opus 4.6 --- Dockerfile.botenv | 13 +++++++++++++ README.md | 2 ++ src/server.ts | 2 ++ src/services/DockerService.ts | 20 +++++++++++++++----- 4 files changed, 32 insertions(+), 5 deletions(-) diff --git a/Dockerfile.botenv b/Dockerfile.botenv index 2743007..5219b4e 100644 --- a/Dockerfile.botenv +++ b/Dockerfile.botenv @@ -66,8 +66,21 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ # Process/network tools procps lsof \ iproute2 netcat-openbsd dnsutils \ + # Browser automation (Playwright/Chromium dependencies) + libnss3 libnspr4 libatk-bridge2.0-0 libdrm2 libxkbcommon0 \ + libxcomposite1 libxdamage1 libxrandr2 libgbm1 libxss1 \ + libasound2t64 libatk1.0-0t64 libcups2t64 libpango-1.0-0 \ + libcairo2 libatspi2.0-0 libx11-xcb1 libxfixes3 \ + fonts-liberation fonts-noto-color-emoji \ && rm -rf /var/lib/apt/lists/* +# Install Playwright globally and download Chromium browser +ENV PLAYWRIGHT_BROWSERS_PATH=/opt/playwright-browsers +RUN mkdir -p /opt/playwright-browsers \ + && npm install -g playwright \ + && npx playwright install chromium \ + && chmod -R a+rX /opt/playwright-browsers + # Install Rust toolchain via rustup (APT packages are severely outdated) ENV RUSTUP_HOME=/usr/local/rustup \ CARGO_HOME=/usr/local/cargo \ diff --git a/README.md b/README.md index 3115d1f..6fb4502 100644 --- a/README.md +++ b/README.md @@ -228,6 +228,7 @@ All `/api/*` endpoints require authentication via Bearer token (see Authenticati | DELETE | `/api/bots/:hostname` | Delete bot and cleanup resources | | POST | `/api/bots/:hostname/start` | Start bot container | | POST | `/api/bots/:hostname/stop` | Stop bot container | +| POST | `/api/bots/:hostname/recreate` | Recreate container with current botenv image | ### Monitoring & Admin @@ -237,6 +238,7 @@ All `/api/*` endpoints require authentication via Bearer token (see Authenticati | GET | `/api/stats` | Container resource stats (CPU, memory) | | GET | `/api/admin/orphans` | Preview orphaned resources | | POST | `/api/admin/cleanup` | Clean orphaned containers/workspaces/secrets | +| POST | `/api/admin/recreate-all` | Recreate all bot containers with current botenv image | ## Project Structure diff --git a/src/server.ts b/src/server.ts index 891f903..cff176d 100644 --- a/src/server.ts +++ b/src/server.ts @@ -780,6 +780,7 @@ export async function buildServer(): Promise { return { success: true, status: 'running', containerId: newContainerId, image: config.openclawImage }; } catch (err) { + updateBot(bot.id, { status: 'stopped' }); if (err instanceof ContainerError) { reply.code(500); return { error: `Failed to recreate container: ${err.message}` }; @@ -810,6 +811,7 @@ export async function buildServer(): Promise { updateBot(bot.id, { status: 'running' }); results.push({ hostname: bot.hostname, success: true }); } catch (err) { + updateBot(bot.id, { status: 'stopped' }); const msg = err instanceof Error ? err.message : String(err); results.push({ hostname: bot.hostname, success: false, error: msg }); } diff --git a/src/services/DockerService.ts b/src/services/DockerService.ts index 43b84a7..ad0783a 100644 --- a/src/services/DockerService.ts +++ b/src/services/DockerService.ts @@ -95,6 +95,7 @@ export class DockerService { StartPeriod: 5_000_000_000, // 5s in nanoseconds }, HostConfig: { + ShmSize: 1024 * 1024 * 1024, // 1GB — Chromium needs larger /dev/shm Binds: [ `${config.hostSecretsPath}:/run/secrets:ro`, `${config.hostWorkspacePath}:/app/botdata:rw`, @@ -208,9 +209,10 @@ export class DockerService { } /** - * Recreates a container with a new image, preserving all config. - * Inspects the existing container, removes it, creates a new one with the same - * env/binds/ports/network/labels/healthcheck but the specified image. + * Recreates a container with a new image, preserving key runtime configuration. + * Inspects the existing container, removes it, and creates a new one that keeps + * the same command, environment, exposed ports, labels, healthcheck, and relevant + * host configuration (such as binds and networking), but uses the specified image. * * @param hostname - Hostname of the bot * @param newImage - Docker image to use for the new container @@ -232,7 +234,7 @@ export class DockerService { const oldHostConfig = info.HostConfig as unknown as { Binds: string[]; PortBindings: Record; RestartPolicy: { Name: string }; NetworkMode: string; - ExtraHosts: string[] | null; + ExtraHosts: string[] | null; ShmSize: number; }; // Stop and remove the old container @@ -244,7 +246,14 @@ export class DockerService { throw stopErr; } } - await container.remove(); + try { + await container.remove(); + } catch (removeErr) { + const dockerErr = removeErr as { statusCode?: number }; + if (dockerErr.statusCode !== 404) { + throw removeErr; + } + } // Create new container with same config but new image const newContainer = await this.docker.createContainer({ @@ -256,6 +265,7 @@ export class DockerService { Labels: oldConfig.Labels, Healthcheck: oldConfig.Healthcheck, HostConfig: { + ShmSize: oldHostConfig.ShmSize || 1024 * 1024 * 1024, Binds: oldHostConfig.Binds, PortBindings: oldHostConfig.PortBindings, RestartPolicy: oldHostConfig.RestartPolicy,