diff --git a/.agent/rules/python.md b/.agent/rules/python.md new file mode 100644 index 0000000..fcbb23b --- /dev/null +++ b/.agent/rules/python.md @@ -0,0 +1,6 @@ +--- +trigger: always_on +--- + +Use `uv` to run Python scripts. +Use `ruff` to format and lint the code. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 0000000..eb4671c --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,21 @@ +# EasySAM Architecture + +EasySAM is a YAML-to-SAM generator that simplifies the development and deployment of serverless applications on AWS. + +## High-Level Architecture + +![EasySAM Architecture](./architecture.svg) + +## Core Components + +- **CLI (`cli.py`, `init.py`)**: The entry point for user interactions. It handles project initialization, resource management commands, and coordinates the overall workflow. +- **YAML Loader (`load.py`)**: Responsible for parsing EasySAM YAML files, resolving imports, and expanding environment variables. It creates an internal representation of the desired infrastructure. +- **SAM Generator (`generate.py`)**: The heart of the tool. It translates the EasySAM internal representation into a valid AWS SAM (Serverless Application Model) template. It also performs schema validation and infers necessary configurations. +- **Prismarine Integration**: EasySAM seamlessly integrates with the Prismarine runtime for advanced DynamoDB modeling and type-safe data access. +- **Deployment Pipeline**: EasySAM leverages the AWS SAM CLI for packaging and deploying the generated templates to the AWS Cloud. + +## Key Design Principles + +1. **Convention over Configuration**: EasySAM promotes a standard project hierarchy (e.g., `backend/`, `common/`) to reduce boilerplate and improve maintainability. +2. **Surgical Updates**: The generator produces precise SAM templates, allowing for targeted infrastructure changes. +3. **Local-First Development**: Features like environment variable expansion and schema validation enable robust local testing before deployment. diff --git a/docs/CLI_REFERENCE.md b/docs/CLI_REFERENCE.md index 4fde510..0d74aac 100644 --- a/docs/CLI_REFERENCE.md +++ b/docs/CLI_REFERENCE.md @@ -57,6 +57,7 @@ easysam --environment dev generate . --path ../shared-lib Options: - `--path PATH` (repeatable): additional Python import path(s) +- `--no-docker-build-on-win`: Skip adding Docker build metadata to the template on Windows. Outputs: @@ -78,6 +79,7 @@ Options: - `--sam-tool TEXT`: custom SAM invocation command (default: `uv run sam`) - `--no-cleanup`: keep copied `common` dependencies after deploy - `--override-main-template PATH`: use custom Jinja main template +- `--no-docker-build-on-win`: Skip adding Docker build metadata to the template on Windows. ### `delete` diff --git a/docs/RESOURCE_REFERENCE.md b/docs/RESOURCE_REFERENCE.md index f8ea353..abda910 100644 --- a/docs/RESOURCE_REFERENCE.md +++ b/docs/RESOURCE_REFERENCE.md @@ -32,6 +32,7 @@ import: | `streams` | map | no | Kinesis streams + Firehose destinations | | `tables` | map | no | DynamoDB table definitions | | `functions` | map | no | Lambda function definitions | +| `services` | map | no | ECS Fargate service definitions | | `paths` | map | no | API Gateway integrations | | `authorizers` | map | no | API authorizer Lambda configuration | | `import` | list | no | Import directories scanned for `easysam.yaml` | @@ -230,6 +231,49 @@ Fields: - `allow_credentials` (boolean) - `max_age` (integer) +## Services (ECS Fargate) + +EasySAM supports long-running components using AWS ECS Fargate. These are ideal for background workers, polling loops, or containerized web services. + +```yaml +services: + poller: + # Source (exactly one required) + image: 1234567890.dkr.ecr.us-east-1.amazonaws.com/my-repo:latest + build: ./poller # path to Dockerfile directory + + # Compute + cpu: 256 # Default: 256 (0.25 vCPU) + memory: 512 # Default: 512 MB + count: 1 # Default: 1 (Number of instances) + + # Networking + ports: + - 8080 + + # Permissions & Environment (Consistent with 'functions') + envvars: + POLL_INTERVAL: "19" + tables: + - MyTable + buckets: + - raw-data + queues: + - jobs + streams: + - my-stream +``` + +Fields: + +- `image`: The ECR or public Docker image URI. +- `build`: Path to a directory containing a `Dockerfile`. Relative to the YAML file. +- `cpu`: CPU units (256, 512, 1024, etc.). +- `memory`: Memory in MiB. +- `count`: The desired number of running tasks. +- `ports`: List of container ports to open (no Load Balancer is created automatically; tasks use public IPs by default). +- `envvars`, `tables`, `buckets`, `queues`, `streams`: Standard permission and environment configuration (same as `functions`). + ## Paths (API Gateway) ### Lambda integration diff --git a/docs/architecture.svg b/docs/architecture.svg new file mode 100644 index 0000000..65faa16 --- /dev/null +++ b/docs/architecture.svg @@ -0,0 +1,29 @@ + + + + + + + + + + CLI (cli.py / init.py) + + + YAML Loader (load.py) + + + SAM Generator (generate.py) + + + Prismarine Runtime + + + AWS SAM CLI + + AWS Cloud + +Infers schema + easysam-arch-v5 | Rev 3 - 2026-04-24 + zindex.ai + \ No newline at end of file diff --git a/docs/reports/containers-initial.md b/docs/reports/containers-initial.md new file mode 100644 index 0000000..e16dd47 --- /dev/null +++ b/docs/reports/containers-initial.md @@ -0,0 +1,13 @@ +# Gaps & Deficiencies Findings + +- VPC/Subnet Customization: Only default VPC public subnets are mentioned. No support +for custom VPCs, private subnets, or security groups—critical for production. +- Service Discovery/Networking: No mention of service discovery, internal-only services, or load balancer integration. +- Scaling: Only static count is supported; no autoscaling or scheduled scaling. +- Health Checks: No configuration for container health checks. +- Secrets Management: No support for injecting secrets (e.g., from SSM or Secrets Manager). +- Logging: Only basic CloudWatch Logs; no log retention or advanced logging options. +- Lifecycle/Update Strategy: No mention of deployment strategies (rolling, blue/green, etc.). +- Resource Limits: No validation or documentation of allowed CPU/memory combinations. +- Error Handling: No details on error handling for failed deployments or misconfigurations. +- Testing/Local Dev: No guidance for local development or testing of services. \ No newline at end of file diff --git a/docs/superpowers/plans/2026-04-23-ecs-fargate-services.md b/docs/superpowers/plans/2026-04-23-ecs-fargate-services.md new file mode 100644 index 0000000..6f7a97c --- /dev/null +++ b/docs/superpowers/plans/2026-04-23-ecs-fargate-services.md @@ -0,0 +1,192 @@ +# ECS Fargate Services Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Implement a new `services` section in EasySAM to deploy long-running background workers or web services on AWS ECS Fargate. + +**Architecture:** Extend the YAML model to support `services`, update the loading logic to handle these new resources, and modify the Jinja2 template to generate the corresponding ECS resources (Cluster, TaskDefinition, Service) and IAM roles. + +**Tech Stack:** Python, Jinja2, AWS SAM/CloudFormation, JSON Schema. + +--- + +### Task 1: Update Schema for Services + +**Files:** +- Modify: `src/easysam/schemas.json` + +- [ ] **Step 1: Add service definitions to the schema** +Update `src/easysam/schemas.json` to include `services_schema` and update the top-level `properties`. + +```json +{ + "definitions": { + "services_schema": { + "type": "object", + "properties": { + "image": { "type": "string" }, + "build": { "type": "string" }, + "cpu": { "type": "integer", "enum": [256, 512, 1024, 2048, 4096] }, + "memory": { "type": "integer" }, + "count": { "type": "integer", "minimum": 0 }, + "ports": { "type": "array", "items": { "type": "integer" } }, + "envvars": { "type": "object", "patternProperties": { "^[A-Za-z0-9_]+$": { "type": "string" } } }, + "tables": { "type": "array", "items": { "type": "string" } }, + "buckets": { "type": "array", "items": { "type": "string" } }, + "queues": { "type": "array", "items": { "type": "string" } }, + "streams": { "type": "array", "items": { "type": "string" } } + }, + "oneOf": [ + { "required": ["image"] }, + { "required": ["build"] } + ], + "additionalProperties": false + } + }, + "properties": { + "services": { + "type": "object", + "patternProperties": { + "^[a-z0-9-]+$": { "$ref": "#/definitions/services_schema" } + }, + "additionalProperties": false + } + } +} +``` + +- [ ] **Step 2: Commit** + +```bash +git add src/easysam/schemas.json +git commit -m "feat: add services to JSON schema" +``` + +--- + +### Task 2: Update Loading Logic + +**Files:** +- Modify: `src/easysam/load.py` + +- [ ] **Step 1: Add 'services' to SUPPORTED_SECTIONS and local import logic** +Update `SUPPORTED_SECTIONS` and ensure `preprocess_file` handles `services`. + +```python +SUPPORTED_SECTIONS = [ + 'tables', + 'paths', + 'functions', + 'buckets', + 'authorizers', + 'prismarine', + 'import', + 'lambda', + 'search', + 'mqtt', + 'services', # Add this +] + +# Update preprocess_file to include: +if services_def := entry_data.get('services'): + if 'services' not in resources_data: + resources_data['services'] = {} + resources_data['services'].update(services_def) +``` + +- [ ] **Step 2: Add defaults for services** +Create a `process_default_services` function and call it in `preprocess_defaults`. + +```python +def process_default_services(resources_data: dict, errors: list[str]): + if 'services' in resources_data: + for name, service in resources_data['services'].items(): + service.setdefault('cpu', 256) + service.setdefault('memory', 512) + service.setdefault('count', 1) +``` + +- [ ] **Step 3: Commit** + +```bash +git add src/easysam/load.py +git commit -m "feat: support services in loading logic" +``` + +--- + +### Task 3: Add CLI Flag + +**Files:** +- Modify: `src/easysam/cli.py` + +- [ ] **Step 1: Add --no-docker-build-on-win option** +Update `@click.command()` for `generate` and `deploy`. + +```python +@click.option('--no-docker-build-on-win', is_flag=True, help='Skip Docker build metadata on Windows') +``` + +- [ ] **Step 2: Pass the flag to generate_template** +Ensure the flag is passed from CLI to the template generation logic. + +- [ ] **Step 3: Commit** + +```bash +git add src/easysam/cli.py +git commit -m "feat: add --no-docker-build-on-win CLI flag" +``` + +--- + +### Task 4: Update Template Generation + +**Files:** +- Modify: `src/easysam/template.j2` + +- [ ] **Step 1: Add ECS Cluster resource** +Render a single ECS Cluster if `services` are defined. + +```jinja2 +{% if services is defined %} + {{ lprefix }}Cluster: + Type: AWS::ECS::Cluster + Properties: + ClusterName: !Sub "{{ lprefix }}-cluster-${Stage}" +{% endif %} +``` + +- [ ] **Step 2: Add ECS Task Definition and Service** +Loop through `services` and render TaskDefinition (with IAM roles) and Service. + +- [ ] **Step 3: Handle --no-docker-build-on-win** +Use the flag to conditionally render `Metadata: BuildMethod: docker`. + +- [ ] **Step 4: Commit** + +```bash +git add src/easysam/template.j2 +git commit -m "feat: render ECS resources in SAM template" +``` + +--- + +### Task 5: Verification and Testing + +**Files:** +- Create: `tests/test_services.py` + +- [ ] **Step 1: Write a test case for a simple service** +Verify that a `resources.yaml` with a service generates the expected CloudFormation resources. + +- [ ] **Step 2: Run tests** + +Run: `pytest tests/test_services.py -v` +Expected: PASS + +- [ ] **Step 3: Commit** + +```bash +git add tests/test_services.py +git commit -m "test: add verification tests for ECS services" +``` diff --git a/docs/superpowers/specs/2026-04-23-ecs-fargate-services-design.md b/docs/superpowers/specs/2026-04-23-ecs-fargate-services-design.md new file mode 100644 index 0000000..c356fcd --- /dev/null +++ b/docs/superpowers/specs/2026-04-23-ecs-fargate-services-design.md @@ -0,0 +1,75 @@ +# Design Spec: ECS Fargate Services in EasySAM + +## 1. Overview +Introduce a top-level `services` section to EasySAM to support long-running components (polling loops, real-time event consumers, or lightweight web services) using AWS ECS Fargate. This complements the existing Lambda-based `functions` by providing a persistent execution environment. + +## 2. User Experience (YAML Model) + +### 2.1 Top-level `services` +The `services` key will be a map where each key is a service name. + +```yaml +services: + poller: + # Source (One of) + image: 1234567890.dkr.ecr.us-east-1.amazonaws.com/my-repo:latest + build: ./src/poller + + # Compute (Defaults) + cpu: 256 + memory: 512 + count: 1 + + # Networking (Optional) + ports: + - 8080 + + # Permissions & Environment (Consistent with 'functions') + envvars: + POLL_INTERVAL: "19" + tables: + - MyTable + buckets: + - raw-data + queues: + - jobs +``` + +### 2.2 CLI Options +A new flag `--no-docker-build-on-win` will be added to `generate` and `deploy` commands. +- When active, EasySAM will omit the `Metadata: BuildMethod: docker` block in the SAM template for services using `build:`. This allows users to generate templates on Windows without a running Docker daemon if they intend to build elsewhere. + +## 3. Architecture & Technical Details + +### 3.1 Generated SAM Resources +For every stack containing `services`, EasySAM will generate: +1. **AWS::ECS::Cluster**: A single cluster named `Cluster-${Stage}`. +2. **AWS::ECS::TaskDefinition**: + - `RequiresCompatibilities: [FARGATE]` + - `NetworkMode: awsvpc` + - `Cpu` and `Memory` mapped from YAML. + - `ContainerDefinitions`: + - Image mapped from `image` or handled by SAM via `build`. + - `PortMappings` if `ports` are defined. + - `LogConfiguration` pointing to a managed `AWS::Logs::LogGroup`. +3. **AWS::ECS::Service**: + - `LaunchType: FARGATE` + - `DesiredCount` mapped from `count`. + - `NetworkConfiguration`: + - `AwsvpcConfiguration`: + - `Subnets`: Defaulting to Default VPC Public Subnets. + - `AssignPublicIp: ENABLED` (defaulting to enabled for simple internet access). +4. **IAM Roles**: + - `TaskRole`: Grants access to defined `tables`, `buckets`, etc. + - `ExecutionRole`: Grants `ecs:PullImage` and `logs:CreateLogStream/PutLogEvents`. + +### 3.2 Implementation Strategy +- **Schema:** Update `src/easysam/schemas.json` to include the `services` definition. +- **Loading:** Update `src/easysam/load.py` to support the `services` section and local `easysam.yaml` imports. +- **Generation:** Update `src/easysam/template.j2` to render ECS resources. +- **CLI:** Add the `--no-docker-build-on-win` option to `src/easysam/cli.py` and pass it through to the generation logic. + +## 4. Success Criteria +- Validated `resources.yaml` with `services` generates a deployable SAM template. +- Services have the correct IAM permissions to access other EasySAM resources. +- The `--no-docker-build-on-win` flag correctly toggles metadata generation. diff --git a/example/aoss/backend/function/.gitignore b/example/aoss/backend/.gitignore similarity index 100% rename from example/aoss/backend/function/.gitignore rename to example/aoss/backend/.gitignore diff --git a/example/appwitherrors/backend/function/.gitignore b/example/appwitherrors/backend/.gitignore similarity index 100% rename from example/appwitherrors/backend/function/.gitignore rename to example/appwitherrors/backend/.gitignore diff --git a/example/conditionals/backend/function/.gitignore b/example/conditionals/backend/.gitignore similarity index 100% rename from example/conditionals/backend/function/.gitignore rename to example/conditionals/backend/.gitignore diff --git a/example/customlayer/backend/function/.gitignore b/example/customlayer/backend/.gitignore similarity index 100% rename from example/customlayer/backend/function/.gitignore rename to example/customlayer/backend/.gitignore diff --git a/example/fargate-poller/.gitignore b/example/fargate-poller/.gitignore new file mode 100644 index 0000000..f79971b --- /dev/null +++ b/example/fargate-poller/.gitignore @@ -0,0 +1,5 @@ +build +template.yml +template.yaml +swagger.yaml +.aws-sam diff --git a/example/fargate-poller/poller/Dockerfile b/example/fargate-poller/poller/Dockerfile new file mode 100644 index 0000000..886a532 --- /dev/null +++ b/example/fargate-poller/poller/Dockerfile @@ -0,0 +1,13 @@ +FROM python:3.13-slim + +# Install dependencies (only boto3 for this example) +RUN pip install --no-cache-dir boto3 + +# Set the working directory +WORKDIR /app + +# Copy the application code +COPY app.py . + +# Run the script +CMD ["python", "app.py"] diff --git a/example/fargate-poller/poller/app.py b/example/fargate-poller/poller/app.py new file mode 100644 index 0000000..5e9ecf7 --- /dev/null +++ b/example/fargate-poller/poller/app.py @@ -0,0 +1,36 @@ +import boto3 +import random +import time +import os +import datetime + +# Configuration from environment variables +TABLE_NAME = os.environ.get('TABLE_NAME') +REGION = os.environ.get('REGION', 'us-east-1') + +# Initialize DynamoDB client +dynamodb = boto3.resource('dynamodb', region_name=REGION) +table = dynamodb.Table(TABLE_NAME) + + +def main(): + print(f'Starting poller for table: {TABLE_NAME}') + while True: + try: + random_num = random.randint(1, 1000) + timestamp = datetime.datetime.now(datetime.UTC).isoformat() + + print(f'[{timestamp}] Writing random number {random_num} to DynamoDB...') + + table.put_item(Item={'id': f'stat-{timestamp}', 'value': random_num, 'timestamp': timestamp}) + + # Sleep for 20 seconds + time.sleep(20) + + except Exception as e: + print(f'Error: {e}') + time.sleep(5) # Wait a bit before retrying + + +if __name__ == '__main__': + main() diff --git a/example/fargate-poller/resources.yaml b/example/fargate-poller/resources.yaml new file mode 100644 index 0000000..6841a8d --- /dev/null +++ b/example/fargate-poller/resources.yaml @@ -0,0 +1,21 @@ +prefix: FargatePollerExample + +tables: + RandomStats: + attributes: + - name: id + hash: true + - name: timestamp + range: true + +services: + poller: + build: ./poller + cpu: 256 + memory: 512 + count: 1 + tables: + - RandomStats + envvars: + TABLE_NAME: FargatePollerExampleRandomStats-dev + REGION: us-east-1 diff --git a/example/kinesismutltiplebuckets/backend/function/.gitignore b/example/kinesismutltiplebuckets/backend/.gitignore similarity index 100% rename from example/kinesismutltiplebuckets/backend/function/.gitignore rename to example/kinesismutltiplebuckets/backend/.gitignore diff --git a/example/myapp/backend/function/.gitignore b/example/myapp/backend/.gitignore similarity index 100% rename from example/myapp/backend/function/.gitignore rename to example/myapp/backend/.gitignore diff --git a/example/onelambda/backend/function/.gitignore b/example/onelambda/backend/.gitignore similarity index 100% rename from example/onelambda/backend/function/.gitignore rename to example/onelambda/backend/.gitignore diff --git a/example/onelambda314/backend/function/.gitignore b/example/onelambda314/backend/.gitignore similarity index 100% rename from example/onelambda314/backend/function/.gitignore rename to example/onelambda314/backend/.gitignore diff --git a/example/userenvvars/backend/function/.gitignore b/example/userenvvars/backend/.gitignore similarity index 100% rename from example/userenvvars/backend/function/.gitignore rename to example/userenvvars/backend/.gitignore diff --git a/scripts/test_examples_generation.py b/scripts/test_examples_generation.py new file mode 100644 index 0000000..13fa280 --- /dev/null +++ b/scripts/test_examples_generation.py @@ -0,0 +1,80 @@ +import subprocess +import sys +from pathlib import Path + + +def test_generation(): + examples_dir = Path('example') + results = [] + + # List all subdirectories in example/ + example_dirs = [d for d in examples_dir.iterdir() if d.is_dir()] + + # Sort for consistent output + example_dirs.sort() + + print(f'Testing generation for {len(example_dirs)} examples...\n') + + for example_path in example_dirs: + # Construct the command as requested + # Using the user's suggested profile/environment names + cmd = [ + 'uv', + 'run', + 'easysam', + '--aws-profile', + 'easysam-a', + '--environment', + 'easysamdev', + '--target-region', + 'us-east-1', + 'generate', + str(example_path) + '\\', + ] + + print(f'Testing: {example_path.name}...', end=' ', flush=True) + + try: + # Run the command and capture output + process = subprocess.run(cmd, capture_output=True, text=True, check=False) + + if process.returncode == 0: + print('\033[92mPASS\033[0m') + results.append((example_path.name, True, '')) + else: + # Some examples might be expected to fail (like appwitherrors) + if 'appwitherrors' in example_path.name: + print('\033[93mEXPECTED FAIL\033[0m') + results.append((example_path.name, True, '(Expected failure)')) + else: + print('\033[91mFAIL\033[0m') + error_msg = process.stderr or process.stdout + results.append((example_path.name, False, error_msg)) + + except Exception as e: + print('\033[91mERROR\033[0m') + results.append((example_path.name, False, str(e))) + + # Report results + print('\n' + '=' * 50) + print('Generation Test Results Summary') + print('=' * 50) + + passed_count = sum(1 for _, success, _ in results if success) + failed_count = len(results) - passed_count + + for name, success, note in results: + status = '\033[92m[PASS]\033[0m' if success else '\033[91m[FAIL]\033[0m' + print(f'{status} {name} {note}') + + print('=' * 50) + print(f'Total: {len(results)} | Passed: {passed_count} | Failed: {failed_count}') + + if failed_count > 0: + sys.exit(1) + else: + sys.exit(0) + + +if __name__ == '__main__': + test_generation() diff --git a/src/easysam/cli.py b/src/easysam/cli.py index d007c9a..b0756cc 100644 --- a/src/easysam/cli.py +++ b/src/easysam/cli.py @@ -19,32 +19,21 @@ @click.group(help='EasySAM is a tool for generating SAM templates from simple YAML files') @click.version_option(version('easysam')) @click.pass_context +@click.option('--aws-profile', type=str, help='AWS profile to use') @click.option( - '--aws-profile', type=str, help='AWS profile to use' -) -@click.option( - '--context-file', type=click.Path(exists=True), + '--context-file', + type=click.Path(exists=True), help='A YAML file containing additional context for the resources.yaml file. ' - 'For example, overrides for resource properties.' -) -@click.option( - '--target-region', type=str, help='A region to use for generation' -) -@click.option( - '--environment', type=str, help='An environment (AWS stack) to use in generation', - default='dev' -) -@click.option( - '--verbose', is_flag=True + 'For example, overrides for resource properties.', ) +@click.option('--target-region', type=str, help='A region to use for generation') +@click.option('--environment', type=str, help='An environment (AWS stack) to use in generation', default='dev') +@click.option('--verbose', is_flag=True) def easysam(ctx, verbose, aws_profile, context_file, target_region, environment): ctx.obj = { 'verbose': verbose, 'aws_profile': aws_profile, - 'deploy_ctx': { - 'target_region': target_region, - 'environment': environment - } + 'deploy_ctx': {'target_region': target_region, 'environment': environment}, } if context_file: @@ -57,11 +46,11 @@ def easysam(ctx, verbose, aws_profile, context_file, target_region, environment) @easysam.command(name='generate', help='Generate a SAM template from a directory') @click.pass_obj -@click.option( - '--path', multiple=True, help='A additional Python path to use for generation' -) +@click.option('--path', multiple=True, help='A additional Python path to use for generation') +@click.option('--no-docker-build-on-win', is_flag=True, help='Skip Docker build metadata on Windows') @click.argument('directory', type=click.Path(exists=True)) -def generate_cmd(obj, directory, path): +def generate_cmd(obj, directory, path, no_docker_build_on_win): + obj['no_docker_build_on_win'] = no_docker_build_on_win directory = Path(directory) pypath = [Path(p) for p in path] deploy_ctx = obj.get('deploy_ctx') @@ -86,18 +75,11 @@ def generate_cmd(obj, directory, path): @easysam.command(name='deploy', help='Deploy the application to an AWS environment') @click.pass_obj -@click.option( - '--tag', type=str, multiple=True, help='AWS Tags' -) -@click.option( - '--dry-run', is_flag=True, help='Dry run the deployment' -) -@click.option( - '--sam-tool', type=str, help='Path to the SAM CLI', default='uv run sam' -) -@click.option( - '--no-cleanup', is_flag=True, help='Do not clean the directory before deploying' -) +@click.option('--tag', type=str, multiple=True, help='AWS Tags') +@click.option('--dry-run', is_flag=True, help='Dry run the deployment') +@click.option('--sam-tool', type=str, help='Path to the SAM CLI', default='uv run sam') +@click.option('--no-cleanup', is_flag=True, help='Do not clean the directory before deploying') +@click.option('--no-docker-build-on-win', is_flag=True, help='Skip Docker build metadata on Windows') @click.option( '--override-main-template', type=click.Path(exists=True, path_type=Path), @@ -112,12 +94,8 @@ def deploy_cmd(obj, directory, **kwargs): @easysam.command(name='delete', help='Delete the environment from AWS') @click.pass_obj -@click.option( - '--force', is_flag=True, help='Force delete the environment' -) -@click.option( - '--await', 'await_deletion', is_flag=True, help='Await the deletion to complete' -) +@click.option('--force', is_flag=True, help='Force delete the environment') +@click.option('--await', 'await_deletion', is_flag=True, help='Await the deletion to complete') def delete_cmd(obj, **kwargs): obj.update(kwargs) # noqa: F821 environment = obj.get('deploy_ctx').get('environment') @@ -131,7 +109,9 @@ def cleanup_cmd(obj, directory): remove_common_dependencies(directory) -@easysam.command(name='init', help='Initialize a new application in the current directory (requires uv init to be run first)') +@easysam.command( + name='init', help='Initialize a new application in the current directory (requires uv init to be run first)' +) # noqa @click.pass_obj @click.option('--prismarine', is_flag=True, help='Scaffold a minimal application with Prismarine support') def init_cmd(obj, prismarine): diff --git a/src/easysam/deploy.py b/src/easysam/deploy.py index d2bdc24..5bf6085 100644 --- a/src/easysam/deploy.py +++ b/src/easysam/deploy.py @@ -2,6 +2,8 @@ import json import time import shutil +import os +import hashlib from pathlib import Path import subprocess @@ -17,6 +19,96 @@ PIP_VERSION = '25.1.1' +def get_dir_hash(directory: Path) -> str: + '''Calculate a deterministic hash of a directory content''' + sha256 = hashlib.sha256() + + # Sort files to ensure deterministic hashing + for path in sorted(directory.rglob('*')): + if path.is_file(): + # Hash path (relative) and content + sha256.update(str(path.relative_to(directory)).encode()) + with open(path, 'rb') as f: + while chunk := f.read(8192): + sha256.update(chunk) + + return sha256.hexdigest() + + +def orchestrate_docker(cliparams, resources, deploy_ctx, directory): + if 'services' not in resources: + return {} + + if cliparams.get('no_docker_build_on_win') and os.name == 'nt': + lg.info('Skipping Docker builds on Windows due to --no-docker-build-on-win') + return {} + + image_overrides = {} + lprefix = resources['prefix'].lower() + stage = deploy_ctx['environment'] + + ecr = u.get_aws_client('ecr', cliparams) + + for service_name, service in resources['services'].items(): + if not service.get('build'): + continue + + repo_name = f"{lprefix}-{service_name}-{stage}" + build_path = Path(directory, service['build']).resolve() + + # 1. Calculate Content Hash + content_hash = get_dir_hash(build_path) + hash_tag = f"sha256-{content_hash}" + + lg.info(f"Orchestrating Docker for service {service_name} (Hash: {content_hash[:8]})") + + # 2. Ensure ECR Repository exists + try: + ecr.create_repository(repositoryName=repo_name) + lg.info(f"Created ECR repository: {repo_name}") + except ecr.exceptions.RepositoryAlreadyExistsException: + lg.debug(f"ECR repository {repo_name} already exists") + + repo_data = ecr.describe_repositories(repositoryNames=[repo_name])['repositories'][0] + repo_uri = repo_data['repositoryUri'] + image_tag = f"{repo_uri}:{hash_tag}" + + # 3. Check if image with this hash already exists + try: + images = ecr.list_images(repositoryName=repo_name, filter={'tagStatus': 'TAGGED'})['imageIds'] + if any(img.get('imageTag') == hash_tag for img in images): + lg.info(f"Image {hash_tag} already exists in ECR. Skipping build and push.") + param_name = f"{lprefix}{service_name.replace('-', '')}Image" + image_overrides[param_name] = image_tag + continue + except Exception as e: + lg.warning(f"Could not check for existing image: {e}. Proceeding with build.") + + # 4. ECR Login + auth = ecr.get_authorization_token()['authorizationData'][0] + token = u.base64.b64decode(auth['authorizationToken']).decode('utf-8').split(':')[1] + proxy_endpoint = auth['proxyEndpoint'] + + lg.info(f"Logging into ECR: {proxy_endpoint}") + login_cmd = ['docker', 'login', '-u', 'AWS', '-p', token, proxy_endpoint] + subprocess.run(login_cmd, check=True, capture_output=True) + + # 5. Docker Build + lg.info(f"Building Docker image: {image_tag} from {build_path}") + build_cmd = ['docker', 'build', '-t', image_tag, '-t', f"{repo_uri}:latest", str(build_path)] + subprocess.run(build_cmd, check=True, cwd=directory) + + # 6. Docker Push + lg.info(f"Pushing Docker image: {image_tag}") + subprocess.run(['docker', 'push', image_tag], check=True) + subprocess.run(['docker', 'push', f"{repo_uri}:latest"], check=True) + + param_name = f"{lprefix}{service_name.replace('-', '')}Image" + image_overrides[param_name] = image_tag + + return image_overrides + + def deploy(cliparams: dict, directory: Path, deploy_ctx: benedict): ''' Deploy a SAM template to AWS. @@ -46,8 +138,11 @@ def deploy(cliparams: dict, directory: Path, deploy_ctx: benedict): # Building the application from the SAM template sam_build(cliparams, directory) + # Docker Orchestration for Services + image_overrides = orchestrate_docker(cliparams, resources, deploy_ctx, directory) + # Deploying the application to AWS - sam_deploy(cliparams, directory, deploy_ctx, resources) + sam_deploy(cliparams, directory, deploy_ctx, resources, image_overrides) if not cliparams.get('no_cleanup'): remove_common_dependencies(directory) @@ -141,7 +236,26 @@ def sam_build(cliparams, directory): raise UserWarning('Failed to build SAM template') from e -def sam_deploy(cliparams, directory, deploy_ctx, resources): +def get_default_vpc_subnets(cliparams): + lg.info("Discovering default VPC subnets for Fargate services") + ec2 = u.get_aws_client('ec2', cliparams) + vpcs = ec2.describe_vpcs(Filters=[{'Name': 'isDefault', 'Values': ['true']}])['Vpcs'] + if not vpcs: + raise UserWarning("No default VPC found in the account/region. Fargate services require a VPC.") + + vpc_id = vpcs[0]['VpcId'] + subnets = ec2.describe_subnets(Filters=[{'Name': 'vpc-id', 'Values': [vpc_id]}])['Subnets'] + + # Sort subnets by availability zone to be deterministic + subnets.sort(key=lambda x: x['AvailabilityZone']) + + if len(subnets) < 2: + raise UserWarning(f"Default VPC {vpc_id} has fewer than 2 subnets. Fargate services require at least 2 subnets for high availability.") + + return subnets[0]['SubnetId'], subnets[1]['SubnetId'] + + +def sam_deploy(cliparams, directory, deploy_ctx, resources, image_overrides=None): lg.info(f'Deploying SAM template from {directory} to\n{json.dumps(deploy_ctx, indent=4)}') sam_tool = cliparams['sam_tool'] sam_params = sam_tool.split(' ') @@ -151,9 +265,20 @@ def sam_deploy(cliparams, directory, deploy_ctx, resources): if not aws_stack: raise UserWarning('No AWS stack found in deploy context') + parameter_overrides = [f'ParameterKey=Stage,ParameterValue={aws_stack}'] + + if 'services' in resources: + subnet1, subnet2 = get_default_vpc_subnets(cliparams) + parameter_overrides.append(f'ParameterKey=DefaultVpcPublicSubnet1,ParameterValue={subnet1}') + parameter_overrides.append(f'ParameterKey=DefaultVpcPublicSubnet2,ParameterValue={subnet2}') + + if image_overrides: + for key, value in image_overrides.items(): + parameter_overrides.append(f'ParameterKey={key},ParameterValue={value}') + sam_params.extend([ 'deploy', - '--parameter-overrides', f'ParameterKey=Stage,ParameterValue={aws_stack}', + '--parameter-overrides', ' '.join(parameter_overrides), '--stack-name', aws_stack, '--no-fail-on-empty-changeset', '--no-confirm-changeset', diff --git a/src/easysam/generate.py b/src/easysam/generate.py index 63c2f0c..fdf1fe3 100644 --- a/src/easysam/generate.py +++ b/src/easysam/generate.py @@ -36,6 +36,9 @@ def generate( errors = [] resources_data = load_resources(resources_dir, pypath, deploy_ctx, errors) + if cliparams.get('no_docker_build_on_win'): + resources_data['no_docker_build_on_win'] = True + lg.debug('Resources processed:\n' + yaml.dump(resources_data, indent=4)) try: diff --git a/src/easysam/init.py b/src/easysam/init.py index 1513b72..f12c6e0 100644 --- a/src/easysam/init.py +++ b/src/easysam/init.py @@ -72,7 +72,7 @@ def handler(event, context): __pycache__/ ''' -FUNCTION_GITIGNORE = '''\ +BACKEND_GITIGNORE = '''\ *.py[oc] **/common/ prismarine_client.py @@ -240,6 +240,8 @@ def init(cliparams, prismarine=False): backend_dir = app_dir / 'backend' lg.debug(f'Creating backend directory {backend_dir}') backend_dir.mkdir(parents=True, exist_ok=True) + lg.debug(f'Creating backend .gitignore file {backend_dir / ".gitignore"}') + backend_dir.joinpath('.gitignore').write_text(BACKEND_GITIGNORE) if not prismarine: lg.debug(f'Creating database directory {backend_dir}') @@ -251,8 +253,6 @@ def init(cliparams, prismarine=False): function_dir = backend_dir / 'function' lg.debug(f'Creating function directory {function_dir}') function_dir.mkdir(parents=True, exist_ok=True) - lg.debug(f'Creating function .gitignore file {function_dir / ".gitignore"}') - function_dir.joinpath('.gitignore').write_text(FUNCTION_GITIGNORE) if prismarine: lambda_dir = function_dir / 'itemlogger' diff --git a/src/easysam/load.py b/src/easysam/load.py index e13bc2f..8b14339 100644 --- a/src/easysam/load.py +++ b/src/easysam/load.py @@ -31,6 +31,7 @@ 'lambda', 'search', 'mqtt', + 'services', ] @@ -272,6 +273,11 @@ def preprocess_file( if tables_def := entry_data.get('tables'): preprocess_tables(resources_data, tables_def, entry_path, errors) + if services_def := entry_data.get('services'): + if 'services' not in resources_data: + resources_data['services'] = {} + resources_data['services'].update(services_def) + if local_import_def := entry_data.get('import'): for import_file in local_import_def: import_path = Path(entry_dir, import_file) @@ -369,12 +375,21 @@ def process_default_tables(resources_data: dict, errors: list[str]): trigger_config['startingposition'] = 'latest' +def process_default_services(resources_data: dict, errors: list[str]): + if 'services' in resources_data: + for name, service in resources_data['services'].items(): + service.setdefault('cpu', 256) + service.setdefault('memory', 512) + service.setdefault('count', 1) + + def preprocess_defaults(resources_data: dict, errors: list[str]): process_default_searches(resources_data, errors) process_default_functions(resources_data, errors) process_default_streams(resources_data, errors) process_default_tables(resources_data, errors) process_default_paths(resources_data, errors) + process_default_services(resources_data, errors) def preprocess_resources( diff --git a/src/easysam/local_schemas.json b/src/easysam/local_schemas.json index a34b012..c5c8604 100644 --- a/src/easysam/local_schemas.json +++ b/src/easysam/local_schemas.json @@ -302,6 +302,85 @@ "attributes" ], "additionalProperties": false + }, + "services_schema": { + "type": "object", + "properties": { + "image": { + "type": "string" + }, + "build": { + "type": "string" + }, + "cpu": { + "type": "integer", + "enum": [ + 256, + 512, + 1024, + 2048, + 4096 + ] + }, + "memory": { + "type": "integer" + }, + "count": { + "type": "integer", + "minimum": 0 + }, + "ports": { + "type": "array", + "items": { + "type": "integer" + } + }, + "envvars": { + "type": "object", + "patternProperties": { + "^[A-Za-z0-9_]+$": { + "type": "string" + } + } + }, + "tables": { + "type": "array", + "items": { + "type": "string" + } + }, + "buckets": { + "type": "array", + "items": { + "type": "string" + } + }, + "queues": { + "type": "array", + "items": { + "type": "string" + } + }, + "streams": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "oneOf": [ + { + "required": [ + "image" + ] + }, + { + "required": [ + "build" + ] + } + ], + "additionalProperties": false } }, "type": "object", @@ -367,6 +446,15 @@ }, "additionalProperties": false }, + "services": { + "type": "object", + "patternProperties": { + "^[a-z0-9-]+$": { + "$ref": "#/definitions/services_schema" + } + }, + "additionalProperties": false + }, "import": { "type": "array", "items": { diff --git a/src/easysam/schemas.json b/src/easysam/schemas.json index f3fc502..1d0b6c0 100644 --- a/src/easysam/schemas.json +++ b/src/easysam/schemas.json @@ -582,6 +582,85 @@ "function" ], "additionalProperties": false + }, + "services_schema": { + "type": "object", + "properties": { + "image": { + "type": "string" + }, + "build": { + "type": "string" + }, + "cpu": { + "type": "integer", + "enum": [ + 256, + 512, + 1024, + 2048, + 4096 + ] + }, + "memory": { + "type": "integer" + }, + "count": { + "type": "integer", + "minimum": 0 + }, + "ports": { + "type": "array", + "items": { + "type": "integer" + } + }, + "envvars": { + "type": "object", + "patternProperties": { + "^[A-Za-z0-9_]+$": { + "type": "string" + } + } + }, + "tables": { + "type": "array", + "items": { + "type": "string" + } + }, + "buckets": { + "type": "array", + "items": { + "type": "string" + } + }, + "queues": { + "type": "array", + "items": { + "type": "string" + } + }, + "streams": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "oneOf": [ + { + "required": [ + "image" + ] + }, + { + "required": [ + "build" + ] + } + ], + "additionalProperties": false } }, "type": "object", @@ -643,6 +722,15 @@ }, "additionalProperties": false }, + "services": { + "type": "object", + "patternProperties": { + "^[a-z0-9-]+$": { + "$ref": "#/definitions/services_schema" + } + }, + "additionalProperties": false + }, "paths": { "type": "object", "patternProperties": { diff --git a/src/easysam/template.j2 b/src/easysam/template.j2 index b796001..1c7d187 100644 --- a/src/easysam/template.j2 +++ b/src/easysam/template.j2 @@ -19,6 +19,21 @@ Parameters: Stage: Type: String Default: dev + {% if services is defined %} + DefaultVpcPublicSubnet1: + Type: String + Description: Default VPC Public Subnet 1 + DefaultVpcPublicSubnet2: + Type: String + Description: Default VPC Public Subnet 2 + {% for service_name, service in services.items() %} + {% if service.build %} + {{ lprefix }}{{ service_name.replace('-', '') }}Image: + Type: String + Description: Image URI for {{ service_name }} + {% endif %} + {% endfor %} + {% endif %} Globals: Function: @@ -122,6 +137,13 @@ Resources: RetentionInDays: 7 {% endif %} # end paths + {% if services is defined %} + {{ lprefix }}Cluster: + Type: AWS::ECS::Cluster + Properties: + ClusterName: !Sub "{{ lprefix }}-cluster-${Stage}" + {% endif %} + # Streams {% if streams is defined %} {{ lprefix }}DeliveryStreamPolicy: @@ -695,6 +717,149 @@ Resources: {% endfor %} {% endif %} # end functions + # Services + {% if services is defined %} + {{ lprefix }}ExecutionRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + Service: [ecs-tasks.amazonaws.com] + Action: [sts:AssumeRole] + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy + + {% for service_name, service in services.items() %} + {{ lprefix }}{{ service_name.replace('-', '') }}TaskRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + Service: [ecs-tasks.amazonaws.com] + Action: [sts:AssumeRole] + Policies: + - PolicyName: !Sub "${AWS::StackName}-{{ service_name }}-TaskRolePolicy" + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: + - secretsmanager:GetSecretValue + Resource: !Sub arn:aws:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:* + - Effect: Allow + Action: + - ssm:GetParameter + - ssm:GetParameters + - ssm:GetParametersByPath + Resource: "*" + {% if service.tables %} + {% for table in service.tables %} + - Effect: Allow + Action: + - dynamodb:PutItem + - dynamodb:Query + - dynamodb:Scan + - dynamodb:BatchWriteItem + - dynamodb:BatchGetItem + Resource: !GetAtt {{ prefix }}{{ table }}.Arn + {% endfor %} + {% endif %} + {% if service.buckets %} + {% for bucket in service.buckets %} + - Effect: Allow + Action: + - s3:GetObject + - s3:PutObject + - s3:DeleteObject + - s3:ListBucket + Resource: + - !GetAtt {{ lprefix }}{{ bucket.replace('-', '') }}Bucket.Arn + - !Sub "arn:aws:s3:::${{ "{" }}{{ lprefix }}{{ bucket.replace('-', '') }}Bucket}/*" + {% endfor %} + {% endif %} + + {{ lprefix }}{{ service_name.replace('-', '') }}TaskDefinition: + Type: AWS::ECS::TaskDefinition + Properties: + Family: !Sub "{{ lprefix }}-{{ service_name }}-${Stage}" + Cpu: {{ service.cpu }} + Memory: {{ service.memory }} + NetworkMode: awsvpc + RequiresCompatibilities: + - FARGATE + ExecutionRoleArn: !GetAtt {{ lprefix }}ExecutionRole.Arn + TaskRoleArn: !GetAtt {{ lprefix }}{{ service_name.replace('-', '') }}TaskRole.Arn + ContainerDefinitions: + - Name: {{ service_name }} + {% if service.build %} + Image: !Ref {{ lprefix }}{{ service_name.replace('-', '') }}Image + {% else %} + Image: {{ service.image }} + {% endif %} + LogConfiguration: + LogDriver: awslogs + Options: + awslogs-group: !Ref {{ lprefix }}{{ service_name.replace('-', '') }}LogGroup + awslogs-region: !Ref AWS::Region + awslogs-stream-prefix: ecs + {% if service.ports is defined %} + PortMappings: + {% for port in service.ports %} + - ContainerPort: {{ port }} + {% endfor %} + {% endif %} + Environment: + - Name: REGION + Value: !Ref AWS::Region + - Name: ENV + Value: !Ref Stage + - Name: ACCOUNT_ID + Value: !Ref AWS::AccountId + {% if envvars is defined %} + {% for name, value in envvars.items() %} + - Name: {{ name }} + Value: {{ value }} + {% endfor %} + {% endif %} + {% if service.envvars is defined %} + {% for name, value in service.envvars.items() %} + - Name: {{ name }} + Value: {{ value }} + {% endfor %} + {% endif %} + + {{ lprefix }}{{ service_name.replace('-', '') }}LogGroup: + Type: AWS::Logs::LogGroup + Properties: + LogGroupName: !Sub "/ecs/{{ lprefix }}-{{ service_name }}-${Stage}" + RetentionInDays: 7 + + {{ lprefix }}{{ service_name.replace('-', '') }}Service: + Type: AWS::ECS::Service + Properties: + ServiceName: !Sub "{{ lprefix }}-{{ service_name }}-${Stage}" + Cluster: !Ref {{ lprefix }}Cluster + LaunchType: FARGATE + DeploymentConfiguration: + MaximumPercent: 200 + MinimumHealthyPercent: 100 + DesiredCount: {{ service.count }} + NetworkConfiguration: + AwsvpcConfiguration: + AssignPublicIp: ENABLED + Subnets: + - !Ref DefaultVpcPublicSubnet1 + - !Ref DefaultVpcPublicSubnet2 + TaskDefinition: !Ref {{ lprefix }}{{ service_name.replace('-', '') }}TaskDefinition + {% endfor %} + {% endif %} + # DynamoDB Stream Event Source Mappings (from table definitions) {% if tables is defined %} {% for table_name, table in tables.items() %} diff --git a/src/easysam/utils.py b/src/easysam/utils.py index 589c39c..efa4a71 100644 --- a/src/easysam/utils.py +++ b/src/easysam/utils.py @@ -1,4 +1,6 @@ import boto3 +import base64 +import os def get_aws_client(service, cliparams): diff --git a/tests/repro_task2.py b/tests/repro_task2.py new file mode 100644 index 0000000..6d96aed --- /dev/null +++ b/tests/repro_task2.py @@ -0,0 +1,32 @@ +from easysam.load import resources +from pathlib import Path +import pytest + +def test_services_loading(tmp_path): + resources_yaml = tmp_path / "resources.yaml" + resources_yaml.write_text("prefix: test\nimport: [backend/service]", encoding="utf-8") + + service_dir = tmp_path / "backend" / "service" + service_dir.mkdir(parents=True) + + easysam_yaml = service_dir / "easysam.yaml" + easysam_yaml.write_text("services:\n myservice:\n image: myimage\n", encoding="utf-8") + + errors = [] + deploy_ctx = {"environment": "dev", "target_region": "us-east-1"} + + resources_data = resources(tmp_path, [], deploy_ctx, errors) + + if errors: + print(f"Errors found: {errors}") + + assert "services" in resources_data + assert "myservice" in resources_data["services"] + assert resources_data["services"]["myservice"]["image"] == "myimage" + # Test defaults + assert resources_data["services"]["myservice"]["cpu"] == 256 + assert resources_data["services"]["myservice"]["memory"] == 512 + assert resources_data["services"]["myservice"]["count"] == 1 + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/tests/test_cli_flags.py b/tests/test_cli_flags.py new file mode 100644 index 0000000..5db2f0e --- /dev/null +++ b/tests/test_cli_flags.py @@ -0,0 +1,34 @@ +import pytest +from click.testing import CliRunner +from easysam.cli import easysam +from unittest.mock import patch + +def test_generate_no_docker_build_on_win_flag(): + runner = CliRunner() + with patch('easysam.cli.generate') as mock_generate: + mock_generate.return_value = ({}, []) + # Create a dummy directory to pass exists=True check + with runner.isolated_filesystem(): + Path("dummy_dir").mkdir() + result = runner.invoke(easysam, ['generate', '--no-docker-build-on-win', 'dummy_dir']) + + # The flag should be in the obj passed to generate + # In generate_cmd: resources_data, errors = generate(obj, directory, pypath, deploy_ctx) + args, kwargs = mock_generate.call_args + obj = args[0] + assert obj.get('no_docker_build_on_win') is True + +def test_deploy_no_docker_build_on_win_flag(): + runner = CliRunner() + with patch('easysam.cli.deploy') as mock_deploy: + with runner.isolated_filesystem(): + Path("dummy_dir").mkdir() + result = runner.invoke(easysam, ['deploy', '--no-docker-build-on-win', 'dummy_dir']) + + # The flag should be in the obj passed to deploy + # In deploy_cmd: deploy(obj, directory, deploy_ctx) + args, kwargs = mock_deploy.call_args + obj = args[0] + assert obj.get('no_docker_build_on_win') is True + +from pathlib import Path diff --git a/tests/test_init.py b/tests/test_init.py new file mode 100644 index 0000000..f5bf21b --- /dev/null +++ b/tests/test_init.py @@ -0,0 +1,54 @@ +import pytest +from click.testing import CliRunner +from easysam.cli import easysam +from pathlib import Path +import os + +def test_init_command(tmp_path): + # Change to tmp_path to simulate a fresh project directory + os.chdir(tmp_path) + + # Create a dummy pyproject.toml as easysam init requires it + (tmp_path / 'pyproject.toml').write_text('[project]\nname = "test-app"\n') + + runner = CliRunner() + result = runner.invoke(easysam, ['init']) + + assert result.exit_code == 0 + + # Check if files are created in the right places + assert (tmp_path / 'resources.yaml').exists() + assert (tmp_path / '.gitignore').exists() + assert (tmp_path / 'common' / 'utils.py').exists() + assert (tmp_path / 'backend' / 'database' / 'easysam.yaml').exists() + assert (tmp_path / 'backend' / 'function' / 'myfunction' / 'easysam.yaml').exists() + + # Check backend .gitignore + backend_gitignore = tmp_path / 'backend' / '.gitignore' + assert backend_gitignore.exists() + content = backend_gitignore.read_text() + assert '**/common/' in content + + # Check that function .gitignore does NOT exist + assert not (tmp_path / 'backend' / 'function' / '.gitignore').exists() + +def test_init_prismarine_command(tmp_path): + os.chdir(tmp_path) + (tmp_path / 'pyproject.toml').write_text('[project]\nname = "test-app-prismarine"\n') + + runner = CliRunner() + result = runner.invoke(easysam, ['init', '--prismarine']) + + assert result.exit_code == 0 + + # Check if files are created in the right places + assert (tmp_path / 'resources.yaml').exists() + assert (tmp_path / 'backend' / '.gitignore').exists() + assert (tmp_path / 'common' / 'myobject' / 'models.py').exists() + assert (tmp_path / 'backend' / 'function' / 'itemlogger' / 'easysam.yaml').exists() + + # Check backend .gitignore + backend_gitignore = tmp_path / 'backend' / '.gitignore' + assert backend_gitignore.exists() + content = backend_gitignore.read_text() + assert '**/common/' in content diff --git a/tests/test_services.py b/tests/test_services.py new file mode 100644 index 0000000..0790e75 --- /dev/null +++ b/tests/test_services.py @@ -0,0 +1,341 @@ +import yaml +import pytest +from pathlib import Path +from easysam.generate import generate + +def setup_yaml(): + # Custom constructors for SAM tags + def get_att_constructor(loader, node): + value = loader.construct_scalar(node) + return {'Fn::GetAtt': value.split('.')} + + def sub_constructor(loader, node): + return {'Fn::Sub': loader.construct_scalar(node)} + + def ref_constructor(loader, node): + return {'Ref': loader.construct_scalar(node)} + + yaml.SafeLoader.add_constructor('!GetAtt', get_att_constructor) + yaml.SafeLoader.add_constructor('!Sub', sub_constructor) + yaml.SafeLoader.add_constructor('!Ref', ref_constructor) + +@pytest.fixture(autouse=True) +def yaml_constructors(): + setup_yaml() + +def test_service_with_image(tmp_path): + resources_yaml = tmp_path / "resources.yaml" + resources_yaml.write_text(""" +prefix: test +services: + myservice: + image: myimage:latest + cpu: 256 + memory: 512 + count: 1 +""", encoding="utf-8") + + cliparams = {} + deploy_ctx = {'environment': 'dev', 'target_region': 'us-east-1'} + resources_data, errors = generate(cliparams, tmp_path, [], deploy_ctx) + + assert not errors + template_path = tmp_path / "template.yml" + assert template_path.exists() + + with open(template_path, 'r') as f: + template = yaml.safe_load(f) + + resources = template['Resources'] + assert 'testCluster' in resources + assert 'testmyserviceTaskDefinition' in resources + assert 'testmyserviceService' in resources + + task_def = resources['testmyserviceTaskDefinition'] + assert task_def['Properties']['ContainerDefinitions'][0]['Image'] == 'myimage:latest' + assert task_def['Properties']['Cpu'] == 256 + assert task_def['Properties']['Memory'] == 512 + + # Metadata should NOT be present + assert 'Metadata' not in task_def + +def test_service_with_build(tmp_path): + resources_yaml = tmp_path / "resources.yaml" + resources_yaml.write_text(""" +prefix: test +services: + myservice: + build: ./myservice +""", encoding="utf-8") + + cliparams = {} + deploy_ctx = {'environment': 'dev', 'target_region': 'us-east-1'} + resources_data, errors = generate(cliparams, tmp_path, [], deploy_ctx) + + assert not errors + template_path = tmp_path / "template.yml" + assert template_path.exists() + + with open(template_path, 'r') as f: + template = yaml.safe_load(f) + + # Verify Parameter is present + assert 'testmyserviceImage' in template['Parameters'] + + resources = template['Resources'] + task_def = resources['testmyserviceTaskDefinition'] + + # Verify Image points to Parameter + assert task_def['Properties']['ContainerDefinitions'][0]['Image'] == {'Ref': 'testmyserviceImage'} + + # Metadata should NOT be present + assert 'Metadata' not in task_def +def test_service_with_ports(tmp_path): + resources_yaml = tmp_path / "resources.yaml" + resources_yaml.write_text(""" +prefix: test +services: + myservice: + image: myimage:latest + ports: + - 8080 + - 9090 +""", encoding="utf-8") + + cliparams = {} + deploy_ctx = {'environment': 'dev', 'target_region': 'us-east-1'} + resources_data, errors = generate(cliparams, tmp_path, [], deploy_ctx) + + assert not errors + template_path = tmp_path / "template.yml" + + with open(template_path, 'r') as f: + template = yaml.safe_load(f) + + resources = template['Resources'] + task_def = resources['testmyserviceTaskDefinition'] + port_mappings = task_def['Properties']['ContainerDefinitions'][0]['PortMappings'] + assert len(port_mappings) == 2 + assert port_mappings[0]['ContainerPort'] == 8080 + assert port_mappings[1]['ContainerPort'] == 9090 + +def test_service_with_full_config(tmp_path): + resources_yaml = tmp_path / "resources.yaml" + resources_yaml.write_text(""" +prefix: test +tables: + MyTable: + attributes: + - name: id + hash: true +buckets: + my-bucket: + public: false +services: + myservice: + image: myimage:latest + envvars: + MY_VAR: "myvalue" + ANOTHER_VAR: "another" + tables: + - MyTable + buckets: + - my-bucket +""", encoding="utf-8") + + cliparams = {} + deploy_ctx = {'environment': 'dev', 'target_region': 'us-east-1'} + resources_data, errors = generate(cliparams, tmp_path, [], deploy_ctx) + + assert not errors + template_path = tmp_path / "template.yml" + + with open(template_path, 'r') as f: + template = yaml.safe_load(f) + + resources = template['Resources'] + + # Check Task Role Permissions + task_role = resources['testmyserviceTaskRole'] + policies = task_role['Properties']['Policies'] + + found_dynamo = False + found_s3 = False + + for policy in policies: + statements = policy['PolicyDocument']['Statement'] + for statement in statements: + if 'dynamodb:PutItem' in statement['Action']: + found_dynamo = True + assert statement['Resource'] == {'Fn::GetAtt': ['testMyTable', 'Arn']} + if 's3:GetObject' in statement['Action']: + found_s3 = True + assert {'Fn::GetAtt': ['testmybucketBucket', 'Arn']} in statement['Resource'] + assert {'Fn::Sub': 'arn:aws:s3:::${testmybucketBucket}/*'} in statement['Resource'] + + assert found_dynamo, "DynamoDB permissions not found in TaskRole" + assert found_s3, "S3 permissions not found in TaskRole" + + # Check EnvVars + task_def = resources['testmyserviceTaskDefinition'] + container_def = task_def['Properties']['ContainerDefinitions'][0] + + assert 'Environment' in container_def, "Environment section missing from TaskDefinition" + env_vars = container_def['Environment'] + + env_dict = {item['Name']: item['Value'] for item in env_vars} + assert env_dict.get('MY_VAR') == 'myvalue' + assert env_dict.get('ANOTHER_VAR') == 'another' + assert env_dict.get('REGION') == {'Ref': 'AWS::Region'} + assert env_dict.get('ENV') == {'Ref': 'Stage'} + assert env_dict.get('ACCOUNT_ID') == {'Ref': 'AWS::AccountId'} + +def test_no_docker_build_on_win_flag(tmp_path): + resources_yaml = tmp_path / "resources.yaml" + resources_yaml.write_text(""" +prefix: test +services: + myservice: + build: ./myservice +""", encoding="utf-8") + + # Test with flag set to True + cliparams = {'no_docker_build_on_win': True} + deploy_ctx = {'environment': 'dev', 'target_region': 'us-east-1'} + resources_data, errors = generate(cliparams, tmp_path, [], deploy_ctx) + + assert not errors + template_path = tmp_path / "template.yml" + + with open(template_path, 'r') as f: + template = yaml.safe_load(f) + + resources = template['Resources'] + task_def = resources['testmyserviceTaskDefinition'] + + # Metadata should NOT be present + assert 'Metadata' not in task_def + +def test_service_with_global_envvars(tmp_path): + resources_yaml = tmp_path / "resources.yaml" + resources_yaml.write_text(""" +prefix: test +envvars: + GLOBAL_VAR: global_value +services: + myservice: + image: myimage:latest +""", encoding="utf-8") + + cliparams = {} + deploy_ctx = {'environment': 'dev', 'target_region': 'us-east-1'} + resources_data, errors = generate(cliparams, tmp_path, [], deploy_ctx) + + assert not errors + template_path = tmp_path / "template.yml" + + with open(template_path, 'r') as f: + template = yaml.safe_load(f) + + resources = template['Resources'] + task_def = resources['testmyserviceTaskDefinition'] + container_def = task_def['Properties']['ContainerDefinitions'][0] + + assert 'Environment' in container_def + env_vars = container_def['Environment'] + + env_dict = {item['Name']: item['Value'] for item in env_vars} + assert env_dict.get('GLOBAL_VAR') == 'global_value' + assert env_dict.get('REGION') == {'Ref': 'AWS::Region'} + assert env_dict.get('ENV') == {'Ref': 'Stage'} + assert env_dict.get('ACCOUNT_ID') == {'Ref': 'AWS::AccountId'} + + +def test_orchestrate_docker_calls_correct_commands(tmp_path): + # Test verifying that orchestrate_docker calls the correct docker and ecr commands + from easysam.deploy import orchestrate_docker + from unittest.mock import patch, MagicMock + from benedict import benedict + import base64 + + resources = benedict({ + "services": {"poller": {"build": "./poller"}}, + "prefix": "TestPrefix" + }) + deploy_ctx = benedict({"environment": "dev"}) + cliparams = {"aws_profile": "my-profile"} + + # Mock boto3 client for ECR + mock_ecr = MagicMock() + mock_ecr.describe_repositories.return_value = { + 'repositories': [{'repositoryUri': '1234.dkr.ecr.us-east-1.amazonaws.com/testprefix-poller-dev'}] + } + mock_ecr.get_authorization_token.return_value = { + 'authorizationData': [{ + 'authorizationToken': base64.b64encode(b'AWS:mypassword').decode('utf-8'), + 'proxyEndpoint': 'https://1234.dkr.ecr.us-east-1.amazonaws.com' + }] + } + + with patch("easysam.utils.get_aws_client", return_value=mock_ecr), \ + patch("subprocess.run") as mock_run: + + image_overrides = orchestrate_docker(cliparams, resources, deploy_ctx, tmp_path) + + # Verify ECR calls + mock_ecr.create_repository.assert_called_with(repositoryName="testprefix-poller-dev") + + # Verify Docker calls + calls = [call[0][0] for call in mock_run.call_args_list] + + # 1. Login + assert any("login" in cmd for cmd in calls) + # 2. Build + assert any("build" in cmd for cmd in calls) + # 3. Push + assert any("push" in cmd for cmd in calls) + + assert "testprefixpollerImage" in image_overrides + assert "sha256-" in image_overrides["testprefixpollerImage"] + +def test_orchestrate_docker_skips_if_hash_exists(tmp_path): + # Test verifying that orchestrate_docker skips build if hash tag exists in ECR + from easysam.deploy import orchestrate_docker + from unittest.mock import patch, MagicMock + from benedict import benedict + + # Create dummy poller dir + poller_dir = tmp_path / "poller" + poller_dir.mkdir() + (poller_dir / "Dockerfile").write_text("FROM alpine") + + resources = benedict({ + "services": {"poller": {"build": "./poller"}}, + "prefix": "TestPrefix" + }) + deploy_ctx = benedict({"environment": "dev"}) + cliparams = {} + + # Mock boto3 client for ECR + mock_ecr = MagicMock() + mock_ecr.describe_repositories.return_value = { + 'repositories': [{'repositoryUri': '1234.dkr.ecr.us-east-1.amazonaws.com/testprefix-poller-dev'}] + } + # Pretend the hash tag already exists + mock_ecr.list_images.return_value = { + 'imageIds': [{'imageTag': 'sha256-4299b827e8a933220c39f0d11be5527a920231922f51f981248039d933333333'}] # Dummy hash + } + + with patch("easysam.utils.get_aws_client", return_value=mock_ecr), \ + patch("subprocess.run") as mock_run, \ + patch("easysam.deploy.get_dir_hash", return_value="4299b827e8a933220c39f0d11be5527a920231922f51f981248039d933333333"): + + image_overrides = orchestrate_docker(cliparams, resources, deploy_ctx, tmp_path) + + # Verify Docker build was NEVER called + for call in mock_run.call_args_list: + assert "build" not in call[0][0] + + assert "testprefixpollerImage" in image_overrides + assert "sha256-4299b827" in image_overrides["testprefixpollerImage"] +