From 4b016a3e17ac9cab6d8d2a3661fcbef7ae05a309 Mon Sep 17 00:00:00 2001 From: Will Zakielarz Date: Thu, 9 Apr 2026 16:54:10 -0400 Subject: [PATCH] Fully local development --- .env.local.example | 85 ++++++------------------- .env.production.example | 24 ++++--- README.md | 130 +++++++++++++++++-------------------- api/prisma/schema.prisma | 2 +- api/src/index.ts | 30 ++++++--- api/src/lib/devProfile.ts | 36 +++++++++++ api/src/lib/s3.ts | 26 ++++++++ infra/BOOTSTRAP.md | 2 +- package.json | 6 +- pnpm-lock.yaml | 131 ++++++++++++++++++++++++++++++++++++++ worker/README.md | 2 +- 11 files changed, 308 insertions(+), 166 deletions(-) create mode 100644 api/src/lib/devProfile.ts diff --git a/.env.local.example b/.env.local.example index ddb411d..8bff4ed 100644 --- a/.env.local.example +++ b/.env.local.example @@ -1,82 +1,33 @@ # Copy to `.env.local` at the repository root. Never commit `.env.local`. -# Prisma scripts load this file via `dotenv -e ../.env.local` from `api/`. -# The API loads `.env.local` when NODE_ENV is not production. - -# ----------------------------------------------------------------------------- -# Recommended: **localhost frontend only** — API + data on AWS (ECS, RDS, S3, SQS) -# ----------------------------------------------------------------------------- -# Do not run `pnpm dev:api` for this workflow. -# -# 1) Copy `frontend/.env.local.example` → `frontend/.env.local` -# 2) Set `VITE_API_URL` to `http://:3001` (see example file for where to find the IP). -# 3) Ensure the ECS `cardboardforge-prod-api` service has desired count ≥ 1 and tasks are running. -# 4) Run: `pnpm dev:frontend` and open http://localhost:5173 -# -# No AWS keys in the browser — only the API URL. The API task uses IAM roles for S3/SQS/RDS. - -# ----------------------------------------------------------------------------- -# Local UI + API against **production** AWS (S3, SQS, RDS) -# ----------------------------------------------------------------------------- -# 1) IAM: use an access key for a user/role that can reach S3 buckets, SQS queue, -# and (for Prisma) RDS. Same credentials you use with AWS CLI are fine if scoped. -# -# 2) RDS from your laptop: the managed DB is in a VPC. Allow your public IP: -# - Get IP: curl -s https://checkip.amazonaws.com -# - Set rds_dev_access_cidrs without committing IPs, e.g. before apply: -# export TF_VAR_rds_dev_access_cidrs='["YOUR.HOME.IP/32","YOUR.SCHOOL.IP/32"]' -# or copy infra/envs/prod/rds.auto.tfvars.example → rds.auto.tfvars (gitignored). -# - Run: cd infra/envs/prod && terraform apply -# That opens Postgres to your IP and sets the instance to publicly addressable. +# Fully local dev: Docker Postgres + LocalStack (no real AWS). # -# 3) DATABASE_URL: postgresql://forge:PASSWORD@HOST:5432/cardboardforge -# Password: cd infra/envs/prod && terraform output -raw rds_master_password -# Host: terraform output -raw rds_endpoint (hostname only, no :5432 in host) +# Used by: `pnpm dev:local`, `pnpm dev:api`, `pnpm migrate:deploy`, and the worker (default). # -# 4) Buckets + queue: terraform output assets_bucket_name exports_bucket_name pdf_queue_url -# -# 5) **Do not set AWS_ENDPOINT_URL** — leave unset so the SDK uses real AWS. -# -# 6) ECS worker vs local worker: if the prod ECS worker service is running, it shares -# the same SQS queue. For predictable local debugging, scale ECS worker desired count -# to 0 in Terraform or the console, then run the Python worker on your machine with -# the same env (real AWS, no endpoint URL). +# 1) cp .env.local.example .env.local +# 2) pnpm docker:up +# 3) pnpm migrate:deploy +# 4) pnpm dev:local NODE_ENV=development PORT=3001 LOG_LEVEL=debug -AWS_REGION=us-east-1 -# AWS_ACCESS_KEY_ID= -# AWS_SECRET_ACCESS_KEY= -# AWS_SESSION_TOKEN= # if using temporary credentials / SSO export +# Optional: fail fast if values look like prod AWS/RDS (see api/src/lib/devProfile.ts) +CARDGOOSE_DEV_PROFILE=fully-local -# Leave commented for production AWS: -# AWS_ENDPOINT_URL=http://localhost:4566 +AWS_REGION=us-east-1 +AWS_ACCESS_KEY_ID=test +AWS_SECRET_ACCESS_KEY=test +AWS_ENDPOINT_URL=http://localhost:4566 -DATABASE_URL=postgresql://forge:REPLACE_WITH_RDS_PASSWORD@REPLACE_WITH_RDS_HOST:5432/cardboardforge +DATABASE_URL=postgresql://forge:forge@localhost:5433/cardgoose -S3_BUCKET_ASSETS=cardboardforge-prod-assets-xxxxxxxx -S3_BUCKET_EXPORTS=cardboardforge-prod-exports-xxxxxxxx -SQS_QUEUE_URL=https://sqs.us-east-1.amazonaws.com/123456789012/cardboardforge-prod-pdf-generation +S3_BUCKET_ASSETS=cardgoose-assets +S3_BUCKET_EXPORTS=cardgoose-exports +SQS_QUEUE_URL=http://localhost:4566/000000000000/cardgoose-pdf-generation -# Local dev only — tokens issued by your local API. Not the same as ECS JWT unless you copy it. JWT_SECRET=change-me-local-dev-only -# PDF worker (Playwright) loads this origin + /render. MUST match Vite’s port (check terminal: -# "Local: http://localhost:5173/" — often 5174 if 5173 is already taken). -# RENDER_URL=http://localhost:5173 - -# Frontend (Vite): leave empty to use the dev proxy to the local API -# VITE_API_URL= +RENDER_URL=http://localhost:5173 -# ----------------------------------------------------------------------------- -# Alternative: full local stack (Docker Postgres + LocalStack) -# ----------------------------------------------------------------------------- -# DATABASE_URL=postgresql://forge:forge@localhost:5433/cardgoose -# AWS_ACCESS_KEY_ID=test -# AWS_SECRET_ACCESS_KEY=test -# AWS_ENDPOINT_URL=http://localhost:4566 -# S3_BUCKET_ASSETS=cardgoose-assets -# S3_BUCKET_EXPORTS=cardgoose-exports -# SQS_QUEUE_URL=http://localhost:4566/000000000000/cardgoose-pdf-generation -# Then: pnpm docker:up +# VITE_API_URL= # leave unset; Vite proxies to local API diff --git a/.env.production.example b/.env.production.example index 1c2cae8..8a01881 100644 --- a/.env.production.example +++ b/.env.production.example @@ -1,24 +1,22 @@ -# Production (AWS) — copy to `.env.production` at the repository root. Never commit `.env.production`. -# Use for: running Prisma against RDS from your laptop (`pnpm migrate:deploy:prod`), -# or local smoke tests against the live API. ECS tasks get env from Terraform, not this file. +# Copy to `.env.production` at the repository root. Never commit `.env.production`. +# +# Used only on your laptop for: +# - `pnpm migrate:deploy:prod` (Prisma migrations against RDS) +# +# Real ECS tasks get env from Terraform, not this file. + +NODE_ENV=production DATABASE_URL=postgresql://forge:YOUR_RDS_PASSWORD@YOUR_RDS_HOST:5432/cardboardforge AWS_REGION=us-east-1 -# IAM user or SSO profile credentials; leave endpoint empty for real AWS -AWS_ACCESS_KEY_ID= -AWS_SECRET_ACCESS_KEY= -AWS_ENDPOINT_URL= +# AWS_ACCESS_KEY_ID= +# AWS_SECRET_ACCESS_KEY= -# From `terraform output` in infra/envs/prod S3_BUCKET_ASSETS=cardboardforge-prod-assets-xxxxxxxx S3_BUCKET_EXPORTS=cardboardforge-prod-exports-xxxxxxxx SQS_QUEUE_URL=https://sqs.us-east-1.amazonaws.com/123456789012/cardboardforge-prod-pdf-generation -JWT_SECRET=use-a-long-random-string +JWT_SECRET=use-a-long-random-string-matching-prod-if-needed PORT=3001 -NODE_ENV=production - -# Frontend pointed at ECS task public IP or future ALB -# VITE_API_URL=https://api.example.com diff --git a/README.md b/README.md index bf55048..14c5269 100644 --- a/README.md +++ b/README.md @@ -27,117 +27,104 @@ Designing board games is hard. It requires **rapid iteration** and the ability t - Node.js 20+ - [pnpm](https://pnpm.io/) 9+ -- Python 3.x + `boto3` for the worker (`pip install boto3` if needed) -- [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html) when using **hybrid** or **production** against real AWS -- Docker — required for **fully local** (Postgres + LocalStack) and useful for optional compose services -- Terraform 1.9+ — for **hybrid** (RDS access from your IP) and **production** (full AWS stack) +- Python 3.12+ recommended — for the PDF worker: `cd worker && python3 -m venv .venv && source .venv/bin/activate && pip install -r requirements.txt && playwright install chromium` +- [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html) — for **production** operations (e.g. Prisma against RDS); not needed for fully local dev +- Docker — **fully local** dev (Postgres + LocalStack) +- Terraform 1.9+ — **production** infrastructure ([`infra/envs/prod`](infra/envs/prod)) --- -## Development modes +## Local development (recommended) -Pick one path. They differ by **where Postgres and object/queue services run**, not by repo layout. - - -| Mode | Frontend | API & worker | Database | Buckets & queue | -| --------------- | -------- | ------------ | -------- | --------------------- | -| **Fully local** | Your Mac | Your Mac | Docker | LocalStack (emulated) | -| **Hybrid** | Your Mac | Your Mac | AWS RDS | AWS S3 & SQS | -| **Production** | ECS | ECS | AWS RDS | AWS S3 & SQS | +Develop against **Docker Postgres** and **LocalStack** (S3/SQS). No real AWS calls. When you are ready, **push to `main`** and [GitHub Actions](.github/workflows/ci.yml) builds and deploys to ECS. +| Piece | Where it runs locally | +| ----- | --------------------- | +| Frontend + API + worker | Your machine | +| Database | Docker Postgres (`localhost:5433`) | +| S3 / SQS | LocalStack (`localhost:4566`) | The API container can serve the built SPA in production (`NODE_ENV=production`). Until you add a stable URL (ALB, CloudFront, etc.), you may use the task public IP for smoke tests. -### Fully local (Docker Postgres + LocalStack) - -Use this when you want **no AWS calls**: everything emulated or on your machine. +### Setup -1. **Start backing services** (Postgres on host port **5433**, LocalStack on **4566**): - ```bash - pnpm docker:up - ``` - Optional: `docker compose build` if you use the `api` / `worker` compose services. -2. **Configure** `[.env.local.example](.env.local.example)` → `**.env.local`** at the repo root using the **“full local stack”** block: `DATABASE_URL` pointing at `localhost:5433`, `AWS_ENDPOINT_URL=http://localhost:4566`, and bucket/queue names `**cardgoose-*`** (they must match `[docker-compose.yml](docker-compose.yml)` and `[docker/localstack-ready.d/init-aws.sh](docker/localstack-ready.d/init-aws.sh)`). -3. **Migrations:** - ```bash - pnpm migrate:deploy - ``` -4. **Run three processes** (three terminals from the repo root): - ```bash - pnpm dev:api - pnpm dev:frontend - ``` -5. Open the URL Vite prints (often `http://localhost:5173`). +1. **One-time:** copy [`.env.local.example`](.env.local.example) to **`.env.local`** at the repo root. It sets LocalStack (`AWS_ENDPOINT_URL`), Docker Postgres on **5433**, and `cardgoose-*` buckets/queue names that match [`docker-compose.yml`](docker-compose.yml) and [`docker/localstack-ready.d/init-aws.sh`](docker/localstack-ready.d/init-aws.sh). Optional `CARDGOOSE_DEV_PROFILE=fully-local` makes the API exit on startup if values look like real AWS/RDS by mistake. -Do **not** set `VITE_API_URL` in `frontend/.env.local` if the Vite dev server should proxy `/api` and `/health` to `http://localhost:3001` (see `[frontend/vite.config.ts](frontend/vite.config.ts)`). +2. **Start the app** (Postgres + LocalStack, then API + Vite): -**PDF exports:** set `**RENDER_URL`** in `.env.local` to the exact origin Vite prints (including port). If the worker runs in Docker and Vite on the host, use something like `http://host.docker.internal:5173`. Restart the worker after changes. +```bash +pnpm dev:local +``` ---- +Or start backing services only, then run processes yourself: -### Hybrid (local apps, real AWS) +```bash +pnpm docker:up +pnpm migrate:deploy +pnpm dev:api +pnpm dev:frontend +``` -Use this for day-to-day work: **Vite, the API, and the worker on your laptop** with **real RDS, S3, and SQS** in `us-east-1`. You get hot reload without deploying containers. +`pnpm docker:up` and `pnpm docker:up:local` are equivalent (Postgres on **5433**, LocalStack on **4566**). -**1. Terraform (occasional)** — in `[infra/envs/prod](infra/envs/prod)`: +3. **First run / after schema changes:** -- Allow your laptop to reach RDS: set `rds_dev_access_cidrs` without committing real IPs — e.g. `export TF_VAR_rds_dev_access_cidrs='["YOUR.PUBLIC.IP/32"]'` before `terraform apply`, or copy `[infra/envs/prod/rds.auto.tfvars.example](infra/envs/prod/rds.auto.tfvars.example)` to `**rds.auto.tfvars`** (gitignored). Update when your IP changes. -- Set `ecs_desired_count = 0` for the **worker** service if you run the Python worker locally (avoids two consumers on the same SQS queue and idle Fargate cost). -- Run `terraform apply`. +```bash +pnpm migrate:deploy +``` -**2. Root `.env.local`** — copy `[.env.local.example](.env.local.example)` and fill **real** values: +4. Open the URL Vite prints (often `http://localhost:5173`). -- `**DATABASE_URL`** — RDS host, user `forge`, password from `terraform output -raw rds_master_password`, database name from your RDS instance (by default Terraform uses the `project_name` value, e.g. `cardboardforge`). -- `**S3_BUCKET_ASSETS**`, `**S3_BUCKET_EXPORTS**`, `**SQS_QUEUE_URL**` — from `terraform output` (`assets_bucket_name`, `exports_bucket_name`, `pdf_queue_url`). +**PDF exports:** use `pnpm dev:local:worker` to run API + Vite + the Python worker together, or run the worker in another terminal (see [`worker/README.md`](worker/README.md)). Set `RENDER_URL` in `.env.local` to the exact origin Vite prints (including port). If the worker runs in Docker and Vite on the host, use something like `http://host.docker.internal:5173`. -Do **not** set `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY` / `AWS_ENDPOINT_URL` for real AWS: use the default credential chain (`~/.aws/credentials` or SSO). +Do **not** set `VITE_API_URL` in `frontend/.env.local` when the dev server should proxy `/api` and `/health` to `http://localhost:3001` (see [`frontend/vite.config.ts`](frontend/vite.config.ts)). -**3. Frontend:** do **not** set `VITE_API_URL` in `frontend/.env.local` (or remove it) so the dev server proxies to the local API. +**Optional — UI only against a deployed API:** copy [`frontend/.env.local.example`](frontend/.env.local.example) to `frontend/.env.local`, set `VITE_API_URL` to your ECS task URL, run `pnpm dev:frontend`. You may need CORS configured on the deployed API for `http://localhost:5173`. -**4. Migrations:** +**Troubleshooting (local):** After a **Docker / LocalStack restart**, S3 buckets may be missing; the dev API **creates missing `S3_BUCKET_*` buckets** when `AWS_ENDPOINT_URL` points at LocalStack (**4566**). If you still see **`NoSuchBucket`**, seed manually, then restart `pnpm dev:local`: ```bash -pnpm migrate:deploy +aws --endpoint-url=http://localhost:4566 s3 mb s3://cardgoose-assets 2>/dev/null || true +aws --endpoint-url=http://localhost:4566 s3 mb s3://cardgoose-exports 2>/dev/null || true +aws --endpoint-url=http://localhost:4566 sqs create-queue --queue-name cardgoose-pdf-generation 2>/dev/null || true ``` -**5. Run** `pnpm dev:api`, `pnpm dev:frontend`, and the worker as in the fully local section. +Or: `docker compose restart localstack` (from the repo root, with compose services up). -**Queue contention:** if the **ECS worker** is scaled up, it shares the same SQS URL as your laptop. For predictable local PDF debugging, keep ECS worker desired count at **0** while testing locally. +--- -**Frontend-only against a cloud API:** set `VITE_API_URL=http://:3001` in `frontend/.env.local` and run `pnpm dev:frontend` only. Find the task IP in the ECS console (see `[frontend/.env.local.example](frontend/.env.local.example)`). This is still “hybrid” from the browser’s perspective (local UI, remote API). +## Production (AWS) -**Troubleshooting RDS:** if Prisma cannot connect, confirm your IP is in `rds_dev_access_cidrs`, re-apply Terraform, and that `DATABASE_URL` matches `terraform output -raw rds_endpoint`. +**Production** means the **API and worker run on ECS Fargate**, with **RDS, S3, and SQS** from Terraform. CI deploys on push to `main` (see [`.github/workflows/ci.yml`](.github/workflows/ci.yml)). ---- +1. **Infrastructure** — follow [infra/BOOTSTRAP.md](infra/BOOTSTRAP.md) and apply [`infra/envs/prod`](infra/envs/prod). -### Production (deployed on AWS) +2. **Prisma against RDS (from your laptop)** — copy [`.env.production.example`](.env.production.example) to **`.env.production`** (never commit) with `DATABASE_URL` and AWS resource names matching Terraform outputs. Then: -**Production** means the **API and worker run on ECS Fargate**, with **RDS, S3, and SQS** provisioned by Terraform. CI can build and push images and force new deployments (see `[.github/workflows/ci.yml](.github/workflows/ci.yml)`). +```bash +pnpm migrate:deploy:prod +``` -1. **Bootstrap and apply** — follow [infra/BOOTSTRAP.md](infra/BOOTSTRAP.md) for remote state, then apply `[infra/envs/prod](infra/envs/prod)`. Resource names, buckets, queues, and the RDS database name follow your Terraform variables (defaults in this repo still use the historical `cardboardforge` prefix for AWS resources). -2. **Runtime config** — ECS tasks receive env vars from Terraform (not from `.env.local`). For **one-off Prisma operations** against production RDS (migrations, introspection), copy `[.env.production.example](.env.production.example)` to `**.env.production`** (never commit) with values that match your live stack: - ```bash - pnpm migrate:deploy:prod - ``` -3. **Deploys** — pushes to `main` run lint, tests, Terraform validate, Docker builds, and (with secrets configured) ECR push + ECS `update-service --force-new-deployment`. Adjust branch rules in the workflow if your default branch differs. -4. **Smoke checks** — hit `/health` on a running API task; confirm `service` identifies as `cardgoose-api`. Scale worker and API services in ECS per load and cost. +ECS tasks do **not** read `.env.production`; they get env from Terraform. ---- +3. **Deploy** — push to `main`: lint, tests, Docker builds, ECR push, ECS rolling update. Run the same checks locally first: `pnpm run format:check`, `pnpm -r run lint`, `pnpm test:all`. -## Environment files +4. **Smoke** — `GET /health` on a running API task; `service` should be `cardgoose-api`. Confirm whether the API container runs migrations on startup so you do not apply twice ([`api/Dockerfile`](api/Dockerfile)). +--- -| File | Purpose | -| ---------------------------------------------------- | ----------------------------------------------------------------------------------------------------------- | -| `[.env.local.example](.env.local.example)` | Template for **local** dev — copy to `**.env.local`** at the repo root. | -| `[.env.production.example](.env.production.example)` | Template for **production DB / ops** — copy to `**.env.production`** for Prisma and tooling (never commit). | +## Environment files +| File | Purpose | +| ---------------------------------------------------- | ------- | +| [`.env.local.example`](.env.local.example) | Template → **`.env.local`** — local dev (Docker Postgres + LocalStack). Used by `pnpm dev:local`, `pnpm dev:api`, `pnpm migrate:deploy`. | +| [`.env.production.example`](.env.production.example) | Template → **`.env.production`** — **only** for `pnpm migrate:deploy:prod` against RDS. | -The API loads `**.env.local**` when `NODE_ENV` is not `production`. Prisma CLI uses `dotenv -e ../.env.local` or `../.env.production` via the `prisma:deploy` scripts. The Baker worker loads the same `**.env.local**` when run locally (`python-dotenv`; existing environment variables win). +The API loads **`.env.local`** via `dotenv-cli` on `pnpm dev` scripts. The worker loads **`.env.local`** when run locally ([`worker/README.md`](worker/README.md)). -If you still have a single `**.env**` from an older setup, merge into `.env.local` and remove `.env`. +If you still have a root **`.env`** from an older setup, merge into `.env.local` and remove `.env`. -**Docker Compose API:** `docker compose up api` runs `prisma migrate deploy` before `node` (see `[docker-compose.yml](docker-compose.yml)`). +**Docker Compose API:** `docker compose up api` runs `prisma migrate deploy` before `node` (see [`docker-compose.yml`](docker-compose.yml)). --- @@ -150,4 +137,3 @@ pnpm test:all cd infra && terraform fmt -check -recursive cd infra/envs/prod && terraform init -input=false && terraform validate ``` - diff --git a/api/prisma/schema.prisma b/api/prisma/schema.prisma index 1317407..183192f 100644 --- a/api/prisma/schema.prisma +++ b/api/prisma/schema.prisma @@ -56,7 +56,7 @@ model CardGroup { @@index([projectId, sortOrder]) } -/// react-konva stage JSON + metadata (hybrid schema from design doc). +/// react-konva stage JSON + metadata. model Layout { id String @id @default(uuid()) @db.Uuid projectId String @map("project_id") @db.Uuid diff --git a/api/src/index.ts b/api/src/index.ts index a795493..bc7a0b7 100644 --- a/api/src/index.ts +++ b/api/src/index.ts @@ -10,8 +10,10 @@ import { projectsRouter } from './routes/projects.js'; import { assetsRouter } from './routes/assets.js'; import { exportsRouter } from './routes/exports.js'; import { rootLogger } from './lib/logger.js'; +import { assertDevProfileIfSet } from './lib/devProfile.js'; +import { ensureDevLocalStackBuckets } from './lib/s3.js'; -// Local DATABASE_URL etc.: use `pnpm dev` so dotenv-cli loads ../.env.local before any imports (Prisma reads env at init). +// Local env: `pnpm dev:api` / `pnpm dev:local` use dotenv-cli to load ../.env.local before imports. const app = express(); const port = Number(process.env.PORT) || 3001; @@ -59,13 +61,21 @@ if (existsSync(publicDir) && process.env.NODE_ENV === 'production') { }); } -app.listen(port, '0.0.0.0', () => { - rootLogger.info( - { - port, - nodeEnv: process.env.NODE_ENV ?? 'development', - awsEndpoint: process.env.AWS_ENDPOINT_URL ?? '(real AWS)', - }, - 'API listening' - ); +assertDevProfileIfSet(); + +void (async () => { + await ensureDevLocalStackBuckets(); + app.listen(port, '0.0.0.0', () => { + rootLogger.info( + { + port, + nodeEnv: process.env.NODE_ENV ?? 'development', + awsEndpoint: process.env.AWS_ENDPOINT_URL ?? '(real AWS)', + }, + 'API listening' + ); + }); +})().catch((err) => { + rootLogger.error(err); + process.exit(1); }); diff --git a/api/src/lib/devProfile.ts b/api/src/lib/devProfile.ts new file mode 100644 index 0000000..8a02206 --- /dev/null +++ b/api/src/lib/devProfile.ts @@ -0,0 +1,36 @@ +/** + * When CARDGOOSE_DEV_PROFILE=fully-local, fail fast if fully-local env looks like + * production-shaped URLs (wrong DB or real AWS endpoints). + */ +export function assertDevProfileIfSet(): void { + if (process.env.NODE_ENV === 'production') return; + if (process.env.CARDGOOSE_DEV_PROFILE !== 'fully-local') return; + + const dbUrl = process.env.DATABASE_URL ?? ''; + const awsEndpoint = (process.env.AWS_ENDPOINT_URL ?? '').trim(); + const sqsUrl = process.env.SQS_QUEUE_URL ?? ''; + + const localPostgres = + /@(localhost|127\.0\.0\.1):5433(\/|$|\?)/.test(dbUrl) || + /\/\/(localhost|127\.0\.0\.1):5433\//.test(dbUrl); + + const localStack = /^https?:\/\/(localhost|127\.0\.0\.1):4566\/?$/.test(awsEndpoint); + + if (!localPostgres) { + throw new Error( + 'CARDGOOSE_DEV_PROFILE=fully-local requires DATABASE_URL to use host localhost or 127.0.0.1 on port 5433 (Docker Postgres from docker-compose).' + ); + } + + if (!localStack) { + throw new Error( + 'CARDGOOSE_DEV_PROFILE=fully-local requires AWS_ENDPOINT_URL=http://localhost:4566 (or http://127.0.0.1:4566) for LocalStack.' + ); + } + + if (sqsUrl.includes('amazonaws.com')) { + throw new Error( + 'CARDGOOSE_DEV_PROFILE=fully-local requires an SQS URL pointing at LocalStack, not AWS (SQS_QUEUE_URL must not contain amazonaws.com).' + ); + } +} diff --git a/api/src/lib/s3.ts b/api/src/lib/s3.ts index f12f184..5e74b45 100644 --- a/api/src/lib/s3.ts +++ b/api/src/lib/s3.ts @@ -1,10 +1,13 @@ import { + CreateBucketCommand, GetObjectCommand, + HeadBucketCommand, ListObjectsV2Command, PutObjectCommand, S3Client, } from '@aws-sdk/client-s3'; import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; +import { rootLogger } from './logger.js'; function endpointConfig(): { endpoint?: string; forcePathStyle?: boolean } { const url = process.env.AWS_ENDPOINT_URL; @@ -38,6 +41,29 @@ export function getExportsBucket(): string { return b; } +/** After Docker/LocalStack restarts, buckets may be missing; create them in dev only. */ +export async function ensureDevLocalStackBuckets(): Promise { + const endpoint = process.env.AWS_ENDPOINT_URL ?? ''; + if (process.env.NODE_ENV === 'production') return; + if (!endpoint.includes(':4566')) return; + + const buckets = [getAssetsBucket(), getExportsBucket()]; + for (const Bucket of buckets) { + try { + await s3Client.send(new HeadBucketCommand({ Bucket })); + } catch { + try { + await s3Client.send(new CreateBucketCommand({ Bucket })); + rootLogger.info({ Bucket }, 'Created missing LocalStack S3 bucket'); + } catch (err) { + const name = err instanceof Error ? err.name : ''; + if (name === 'BucketAlreadyOwnedByYou' || name === 'BucketAlreadyExists') continue; + throw err; + } + } + } +} + export async function putObject( bucket: string, key: string, diff --git a/infra/BOOTSTRAP.md b/infra/BOOTSTRAP.md index 05e1587..277c3ae 100644 --- a/infra/BOOTSTRAP.md +++ b/infra/BOOTSTRAP.md @@ -126,7 +126,7 @@ Use Docker Compose for **PostgreSQL** and **LocalStack** (S3/SQS emulation): pnpm docker:up ``` -Copy `.env.local.example` to `.env.local` and point `AWS_ENDPOINT_URL` at LocalStack when testing the API/worker against emulated services. +Copy `.env.local.example` to `.env.local` when testing the API/worker against Docker Postgres and LocalStack (no real AWS). ## Reference diff --git a/package.json b/package.json index 8978263..c540ccf 100644 --- a/package.json +++ b/package.json @@ -11,9 +11,13 @@ "migrate:deploy": "pnpm --filter api prisma:deploy", "migrate:deploy:prod": "pnpm --filter api prisma:deploy:prod", "docker:up": "docker compose up -d postgres localstack", - "docker:down": "docker compose down" + "docker:up:local": "docker compose up -d postgres localstack", + "docker:down": "docker compose down", + "dev:local": "docker compose up -d postgres localstack && concurrently -n api,frontend -c blue,green \"pnpm dev:api\" \"pnpm dev:frontend\"", + "dev:local:worker": "docker compose up -d postgres localstack && concurrently -n api,frontend,worker -c blue,green,magenta \"pnpm dev:api\" \"pnpm dev:frontend\" \"sh -c 'cd worker && PYTHONPATH=src python3 -m baker.main'\"" }, "devDependencies": { + "concurrently": "^9.2.1", "eslint": "^9.17.0", "prettier": "^3.4.2", "typescript": "^5.7.2" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 350c1c9..5525ad6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,6 +8,9 @@ importers: .: devDependencies: + concurrently: + specifier: ^9.2.1 + version: 9.2.1 eslint: specifier: ^9.17.0 version: 9.39.4(jiti@2.6.1) @@ -1693,6 +1696,10 @@ packages: ajv@6.14.0: resolution: {integrity: sha512-IWrosm/yrn43eiKqkfkHis7QioDleaXQHdDVPKg0FSwwd/DuvyX79TZnFOnYpB7dcsFAMmtFztZuXPDvSePkFw==} + ansi-regex@5.0.1: + resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} + engines: {node: '>=8'} + ansi-styles@4.3.0: resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==} engines: {node: '>=8'} @@ -1815,6 +1822,10 @@ packages: citty@0.2.2: resolution: {integrity: sha512-+6vJA3L98yv+IdfKGZHBNiGW5KHn22e/JwID0Strsz8h4S/csAu/OuICwxrg44k5MRiZHWIo8XXuJgQTriRP4w==} + cliui@8.0.1: + resolution: {integrity: sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==} + engines: {node: '>=12'} + color-convert@2.0.1: resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} engines: {node: '>=7.0.0'} @@ -1829,6 +1840,11 @@ packages: resolution: {integrity: sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==} engines: {'0': node >= 0.8} + concurrently@9.2.1: + resolution: {integrity: sha512-fsfrO0MxV64Znoy8/l1vVIjjHa29SZyyqPgQBwhiDcaW8wJc2W3XWVOGx4M3oJBnv/zdUZIIp1gDeS98GzP8Ng==} + engines: {node: '>=18'} + hasBin: true + confbox@0.2.4: resolution: {integrity: sha512-ysOGlgTFbN2/Y6Cg3Iye8YKulHw+R2fNXHrgSmXISQdMnomY6eNDprVdW9R5xBguEqI954+S6709UyiO7B+6OQ==} @@ -1949,6 +1965,9 @@ packages: electron-to-chromium@1.5.331: resolution: {integrity: sha512-IbxXrsTlD3hRodkLnbxAPP4OuJYdWCeM3IOdT+CpcMoIwIoDfCmRpEtSPfwBXxVkg9xmBeY7Lz2Eo2TDn/HC3Q==} + emoji-regex@8.0.0: + resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==} + empathic@2.0.0: resolution: {integrity: sha512-i6UzDscO/XfAcNYD75CfICkmfLedpyPDdozrLMmQc5ORaQcdMoc21OnlEylMIqI7U8eniKrPMxxtj8k0vhmJhA==} engines: {node: '>=14'} @@ -2230,6 +2249,10 @@ packages: resolution: {integrity: sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==} engines: {node: '>=0.10.0'} + is-fullwidth-code-point@3.0.0: + resolution: {integrity: sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==} + engines: {node: '>=8'} + is-glob@4.0.3: resolution: {integrity: sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==} engines: {node: '>=0.10.0'} @@ -2711,6 +2734,10 @@ packages: resolution: {integrity: sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg==} engines: {node: '>= 12.13.0'} + require-directory@2.1.1: + resolution: {integrity: sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==} + engines: {node: '>=0.10.0'} + resolve-from@4.0.0: resolution: {integrity: sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==} engines: {node: '>=4'} @@ -2728,6 +2755,9 @@ packages: engines: {node: '>=18.0.0', npm: '>=8.0.0'} hasBin: true + rxjs@7.8.2: + resolution: {integrity: sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==} + safe-buffer@5.1.2: resolution: {integrity: sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==} @@ -2775,6 +2805,10 @@ packages: resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==} engines: {node: '>=8'} + shell-quote@1.8.3: + resolution: {integrity: sha512-ObmnIF4hXNg1BqhnHmgbDETF8dLPCggZWBjkQfhZpbszZnYur5DUljTcCHii5LC3J5E0yeO/1LIMyH+UvHQgyw==} + engines: {node: '>= 0.4'} + side-channel-list@1.0.0: resolution: {integrity: sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==} engines: {node: '>= 0.4'} @@ -2819,9 +2853,17 @@ packages: resolution: {integrity: sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==} engines: {node: '>=10.0.0'} + string-width@4.2.3: + resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==} + engines: {node: '>=8'} + string_decoder@1.1.1: resolution: {integrity: sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==} + strip-ansi@6.0.1: + resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==} + engines: {node: '>=8'} + strip-json-comments@3.1.1: resolution: {integrity: sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==} engines: {node: '>=8'} @@ -2836,6 +2878,10 @@ packages: resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==} engines: {node: '>=8'} + supports-color@8.1.1: + resolution: {integrity: sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==} + engines: {node: '>=10'} + tailwindcss@4.2.2: resolution: {integrity: sha512-KWBIxs1Xb6NoLdMVqhbhgwZf2PGBpPEiwOqgI4pFIYbNTfBXiKYyWoTsXgBQ9WFg/OlhnvHaY+AEpW7wSmFo2Q==} @@ -2877,6 +2923,10 @@ packages: resolution: {integrity: sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==} engines: {node: '>=0.6'} + tree-kill@1.2.2: + resolution: {integrity: sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==} + hasBin: true + ts-api-utils@2.5.0: resolution: {integrity: sha512-OJ/ibxhPlqrMM0UiNHJ/0CKQkoKF243/AEmplt3qpRgkW8VG7IfOS41h7V8TjITqdByHzrjcS/2si+y4lIh8NA==} engines: {node: '>=18.12'} @@ -3094,13 +3144,29 @@ packages: resolution: {integrity: sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==} engines: {node: '>=0.10.0'} + wrap-ansi@7.0.0: + resolution: {integrity: sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==} + engines: {node: '>=10'} + xtend@4.0.2: resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} engines: {node: '>=0.4'} + y18n@5.0.8: + resolution: {integrity: sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==} + engines: {node: '>=10'} + yallist@3.1.1: resolution: {integrity: sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==} + yargs-parser@21.1.1: + resolution: {integrity: sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==} + engines: {node: '>=12'} + + yargs@17.7.2: + resolution: {integrity: sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==} + engines: {node: '>=12'} + yocto-queue@0.1.0: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} @@ -4987,6 +5053,8 @@ snapshots: json-schema-traverse: 0.4.1 uri-js: 4.4.1 + ansi-regex@5.0.1: {} + ansi-styles@4.3.0: dependencies: color-convert: 2.0.1 @@ -5115,6 +5183,12 @@ snapshots: citty@0.2.2: {} + cliui@8.0.1: + dependencies: + string-width: 4.2.3 + strip-ansi: 6.0.1 + wrap-ansi: 7.0.0 + color-convert@2.0.1: dependencies: color-name: 1.1.4 @@ -5130,6 +5204,15 @@ snapshots: readable-stream: 2.3.8 typedarray: 0.0.6 + concurrently@9.2.1: + dependencies: + chalk: 4.1.2 + rxjs: 7.8.2 + shell-quote: 1.8.3 + supports-color: 8.1.1 + tree-kill: 1.2.2 + yargs: 17.7.2 + confbox@0.2.4: {} consola@3.4.2: {} @@ -5219,6 +5302,8 @@ snapshots: electron-to-chromium@1.5.331: {} + emoji-regex@8.0.0: {} + empathic@2.0.0: {} encodeurl@2.0.0: {} @@ -5557,6 +5642,8 @@ snapshots: is-extglob@2.1.1: {} + is-fullwidth-code-point@3.0.0: {} + is-glob@4.0.3: dependencies: is-extglob: 2.1.1 @@ -5987,6 +6074,8 @@ snapshots: real-require@0.2.0: {} + require-directory@2.1.1: {} + resolve-from@4.0.0: {} resolve-pkg-maps@1.0.0: {} @@ -6046,6 +6135,10 @@ snapshots: '@rollup/rollup-win32-x64-msvc': 4.60.1 fsevents: 2.3.3 + rxjs@7.8.2: + dependencies: + tslib: 2.8.1 + safe-buffer@5.1.2: {} safe-buffer@5.2.1: {} @@ -6097,6 +6190,8 @@ snapshots: shebang-regex@3.0.0: {} + shell-quote@1.8.3: {} + side-channel-list@1.0.0: dependencies: es-errors: 1.3.0 @@ -6143,10 +6238,20 @@ snapshots: streamsearch@1.1.0: {} + string-width@4.2.3: + dependencies: + emoji-regex: 8.0.0 + is-fullwidth-code-point: 3.0.0 + strip-ansi: 6.0.1 + string_decoder@1.1.1: dependencies: safe-buffer: 5.1.2 + strip-ansi@6.0.1: + dependencies: + ansi-regex: 5.0.1 + strip-json-comments@3.1.1: {} strip-literal@3.1.0: @@ -6159,6 +6264,10 @@ snapshots: dependencies: has-flag: 4.0.0 + supports-color@8.1.1: + dependencies: + has-flag: 4.0.0 + tailwindcss@4.2.2: {} tapable@2.3.2: {} @@ -6186,6 +6295,8 @@ snapshots: toidentifier@1.0.1: {} + tree-kill@1.2.2: {} + ts-api-utils@2.5.0(typescript@5.9.3): dependencies: typescript: 5.9.3 @@ -6365,10 +6476,30 @@ snapshots: word-wrap@1.2.5: {} + wrap-ansi@7.0.0: + dependencies: + ansi-styles: 4.3.0 + string-width: 4.2.3 + strip-ansi: 6.0.1 + xtend@4.0.2: {} + y18n@5.0.8: {} + yallist@3.1.1: {} + yargs-parser@21.1.1: {} + + yargs@17.7.2: + dependencies: + cliui: 8.0.1 + escalade: 3.2.0 + get-caller-file: 2.0.5 + require-directory: 2.1.1 + string-width: 4.2.3 + y18n: 5.0.8 + yargs-parser: 21.1.1 + yocto-queue@0.1.0: {} zod-validation-error@4.0.2(zod@4.3.6): diff --git a/worker/README.md b/worker/README.md index 0e160ec..647f821 100644 --- a/worker/README.md +++ b/worker/README.md @@ -15,7 +15,7 @@ pip install -r requirements.txt playwright install chromium ``` -Run (from `worker/`; walks up to the first **`.env.local`** and loads it with `python-dotenv` — same keys as the API, e.g. `S3_BUCKET_EXPORTS`, `SQS_QUEUE_URL`, `AWS_REGION`). You must have run **`pip install -r requirements.txt`** for that interpreter (system Python counts). +Run (from `worker/`; walks up to the repo root and loads **`.env.local`** with `python-dotenv` — same keys as the local API, e.g. `S3_BUCKET_EXPORTS`, `SQS_QUEUE_URL`, `AWS_REGION`). You must have run **`pip install -r requirements.txt`** for that interpreter (system Python counts). ```bash PYTHONPATH=src python3 -m baker.main