From 68bdc0d282a97d351cbecbea7fbcc3e4706e3fc1 Mon Sep 17 00:00:00 2001 From: modupei Date: Wed, 17 Dec 2025 21:42:05 +0100 Subject: [PATCH 1/7] Fix nested mcp directory and move MCP_Registry_Plan.md --- mcp/MCP_Registry_Plan.md | 288 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 288 insertions(+) create mode 100644 mcp/MCP_Registry_Plan.md diff --git a/mcp/MCP_Registry_Plan.md b/mcp/MCP_Registry_Plan.md new file mode 100644 index 0000000..b6c305d --- /dev/null +++ b/mcp/MCP_Registry_Plan.md @@ -0,0 +1,288 @@ +# ZSoftly MCP Registry - Technical Plan + +## 1. Executive Summary + +This document outlines a technical plan for building a **ZSoftly Model Context Protocol (MCP) Registry**. The goal is to create a private, internal registry to host and manage ZSoftly's proprietary AI Agents and Tools. This registry will function as a central discovery mechanism, allowing applications and other AI agents to dynamically find, understand, and connect to the capabilities offered by registered agents. + +The proposed solution is a lightweight, secure, and cloud-native RESTful service that manages metadata for our AI agents. This plan covers the core concepts, proposed architecture, agent lifecycle management, security considerations, and deployment strategy. It is intended to be a roadmap for an engineering team to implement the registry. + +## 2. MCP Registry Concept Explained + +An MCP Registry is **not** a hosting platform for AI models. Instead, it is a **discovery service** or a **catalog**. Its primary purpose is to answer the question: "What AI capabilities are available, and how do I use them?" + +It works by storing metadata about "MCP Servers" (in our case, ZSoftly AI Agents). Each agent is described by a manifest file (e.g., `agent.json`) which contains: + +- **Identity:** Name, version, owner, description. +- **Contract:** An OpenAPI (or similar) specification that defines the agent's capabilities as a set of callable tools or functions. +- **Location:** The network endpoint (URL) where the agent can be reached. + +Client applications query the registry to find agents that meet their needs (e.g., "find an agent that can summarize text"). The registry returns the manifest of the matching agent(s), and the client then uses that information to communicate directly with the agent. + +For ZSoftly, a private MCP registry provides a secure, centralized way to manage our growing ecosystem of internal AI agents, promoting reuse, standardization, and governance. + +## 3. Agent Lifecycle Management + +The lifecycle of a ZSoftly AI Agent within the registry involves three key stages: **Registration**, **Discovery**, and **Versioning**. This process is managed via a RESTful API and a standardized manifest file. + +### 3.1 The `agent.json` Manifest + +This file is the source of truth for an agent's metadata. Each agent project MUST include an `agent.json` file at its root. + +**Fields:** + +- `name` (string, required): A unique, machine-readable name for the agent (e.g., `document-summarizer`, `user-profile-service`). +- `version` (string, required): The semantic version of the agent (e.g., `1.0.0`). +- `displayName` (string, optional): A human-readable name for display in UIs. +- `description` (string, required): A clear, concise summary of the agent's purpose and capabilities. +- `owner` (string, required): The ZSoftly team or individual responsible for the agent (e.g., `team-alpha`, `user@zsoftly.com`). +- `endpoint` (string, required): The base URL where the agent's service can be accessed. +- `openapi` (object, required): A valid OpenAPI 3.x specification object describing the agent's available `paths` and `components`. This is the contract for using the agent. +- `environment` (enum, required): The environment this agent is deployed to. (`dev`, `stage`, or `prod`). This is critical for multi-environment support. +- `tags` (array of strings, optional): Keywords to improve discoverability (e.g., `["nlp", "text", "summary"]`). + +### 3.2 Registry API Endpoints + +The registry will expose the following core endpoints: + +**Agent Registration** + +- `POST /agents` + - **Action:** Publishes or updates an agent. The request body will be the `agent.json` manifest. + - **Logic:** The registry validates the manifest against a schema. If an agent with the same `name` and `version` already exists, it is updated. Otherwise, a new version is created. The `name` and `version` from the manifest are used as the unique identifier. + - **Auth:** Requires a publisher role. + +**Agent Discovery** + +- `GET /agents` + - **Action:** Returns a paginated list of all registered agents (latest version of each). + - **Auth:** Requires a reader role. + +- `GET /agents/{name}` + - **Action:** Returns a list of all available versions for a specific agent. + - **Auth:** Requires a reader role. + +- `GET /agents/{name}/{version}` + - **Action:** Returns the specific `agent.json` for the given name and version (e.g., `/agents/document-summarizer/1.2.0`). Use `latest` as the version to get the most recently published version. + - **Auth:** Requires a reader role. + +**Agent Search** + +- `GET /search?q={query}` + - **Action:** Searches for agents. The query will match against `name`, `displayName`, `description`, and `tags`. + - **Example:** `/search?q=summarize` + - **Auth:** Requires a reader role. + +**Agent Deletion** + +- `DELETE /agents/{name}/{version}` + - **Action:** Deletes a specific version of an agent. This is a soft-delete to prevent breaking dependencies. A hard-delete mechanism can be decided on later. + - **Auth:** Requires a publisher/admin role. + +### 3.3 Versioning Strategy + +- The registry will use **Semantic Versioning (SemVer)** as specified in the `agent.json` (`major.minor.patch`). +- A combination of `name` and `version` uniquely identifies an agent manifest. +- Clients should be encouraged to request a specific major version (e.g., `1.x`) or use the `latest` tag with caution in development environments. The registry can resolve `1.x` to the latest non-breaking version. +- The `latest` tag will always point to the most recently published version, regardless of whether it's a breaking change. + +## 4. Proposed Architecture + +The MCP Registry will be a simple, robust, and scalable system composed of three core components. The architecture prioritizes statelessness and cloud-native principles. + +The MCP Registry will be a simple, robust, and scalable system composed of three core components. The architecture prioritizes statelessness and cloud-native principles. + +``` ++---------------------------------------------------------------------------------------+ +| MCP Registry | +| | +| +---------------------+ +---------------------+ +---------------------+ | +| | Client Apps |<---->| REST API |<---->| Database | | +| | (Agent Consumers) | | (Registry Service) | | (PostgreSQL/RDS) | | +| +---------------------+ | | | | | +| | - Handles Requests | | - Stores Agent | | +| | - Authentication | | Manifests | | +| | - DB Communication | | - Stores Versions | | +| +---------------------+ | - Supports Search | | +| ^ +---------------------+ | +| | | +| | (Validation Logic) | +| | | +| +---------------------+ | +| | Validation Logic | | +| | - agent.json Schema | | +| | - OpenAPI Spec | | +| +---------------------+ | +| | ++---------------------------------------------------------------------------------------+ +``` + +### 4.1 System Components + +1. **REST API (Registry Service):** + - **Description:** The heart of the system, this service exposes all the endpoints defined in the "Agent Lifecycle Management" section. It handles incoming requests, authentication, and communication with the database. + - **Technology:** A stateless web application. + - **Recommendation:** **FastAPI (Python)**. Its native support for Pydantic models makes validation of incoming `agent.json` manifests trivial. Its automatic generation of OpenAPI documentation is also a significant benefit. + +2. **Database:** + - **Description:** A persistent storage layer for all agent manifests and their versions. It needs to support structured data, indexing for fast lookups, and full-text search capabilities for the `GET /search` endpoint. + - **Recommendation:** **PostgreSQL**. Its robustness, support for JSONB data types, and powerful indexing features make it a perfect fit. Using a managed cloud service (like AWS RDS) is highly recommended. + +3. **Validation Logic:** + - **Description:** Before any data is written to the database, incoming `agent.json` manifests must be validated. This includes checking for required fields, correct data types, and ensuring the embedded OpenAPI contract is syntactically valid. + - **Implementation:** This logic will be implemented directly within the REST API service, leveraging the data validation features of the chosen framework (e.g., Pydantic models in FastAPI). + +### 4.2 Data Model + +A primary `agents` table will store the core metadata. + +- `id` (PK) +- `name` (Indexed) +- `version` (Indexed) +- `display_name` +- `description` +- `owner` +- `endpoint` +- `environment` (Indexed) +- `tags` (Indexed, using GIN index on a JSONB field) +- `openapi_spec` (JSONB) +- `created_at` +- `is_deleted` (for soft deletes) + +A unique constraint will be placed on `(name, version)` to enforce uniqueness. + +## 5. Hosting and Deployment + +Our hosting strategy should be cloud-first, prioritizing managed services to reduce operational overhead. + +### 5.1 Primary Recommendation: Serverless Containers + +This approach offers the best balance of performance, scalability, and ease of management. + +- **Compute:** **AWS Fargate**. + - The FastAPI application will be packaged into a **Docker container**. + - These platforms automatically manage scaling (including scaling to zero), patching, and server infrastructure. +- **Database:** **AWS RDS for PostgreSQL**. + - A managed database service handles backups, failover, and maintenance. +- **CI/CD:** The process of publishing the registry itself (and agents to it) should be automated. A GitHub Actions or GitLab CI pipeline would build the Docker image, push it to a container registry (e.g., ECR), and trigger a deployment in Fargate. + +### 5.2 Alternative: Serverless Functions + +For potentially lower-traffic scenarios or as a cost-saving measure, a purely serverless function architecture could be used. + +- **Compute:** **AWS Lambda**. + - Each API endpoint or a group of related endpoints would be a separate function. +- **Gateway:** **Amazon API Gateway** would be used to manage the public-facing API routes and trigger the appropriate functions. +- **Database:** **AWS Aurora Serverless** or **DynamoDB**. + +**Conclusion:** The **Serverless Container** approach is the recommended path as it offers a more cohesive development experience and simpler application structure compared to managing numerous individual functions. + +## 6. Security and Access Control + +Security is a critical component of the registry, ensuring that only authorized users and services can publish or access agent information. + +### 6.1 Authentication: API Keys + +All requests to the registry API must be authenticated. The proposed method is via **API Keys**. + +- **Mechanism:** Clients will pass their key in the `Authorization: Bearer ` HTTP header. +- **Management:** + - API keys will be generated and associated with a specific role (`Reader`, `Publisher`, or `Admin`). + - Keys will be stored securely in the database using a strong hashing algorithm (e.g., SHA-256). They will not be retrievable in plaintext. + - An internal administrative interface or CLI command will be created for managing keys. + +### 6.2 Authorization: Role-Based Access Control (RBAC) + +Once authenticated, the registry will authorize actions based on pre-defined roles. + +- **Roles:** + 1. **Reader:** Can perform read-only operations (`GET` endpoints for discovery and search). This role is intended for agent consumers (applications, other agents). + 2. **Publisher:** Has `Reader` permissions plus the ability to create, update, and delete agent registrations (`POST`, `DELETE`). This role is intended for developers and CI/CD pipelines responsible for publishing agents. + 3. **Admin:** Has full permissions, including the ability to manage API keys. + +- **Enforcement:** This will be implemented as a middleware within the API service. The middleware will inspect the role associated with the authenticated API key and verify it against the required role for the requested endpoint. + +### 6.3 Multi-Environment Support + +The registry will support multiple environments (`dev`, `stage`, `prod`) within a single deployed instance. + +- **Mechanism:** The `environment` field in the `agent.json` manifest is the primary mechanism for distinguishing agents. +- **Access Strategy:** + - Discovery endpoints (`/agents`, `/search`) will be enhanced with an optional `environment` query parameter (e.g., `GET /agents?environment=prod`). + - Clients (agent consumers) are responsible for requesting the correct environment. For example, a service running in production should be configured to only fetch agents from the `prod` environment. + - This approach provides flexibility. A future enhancement could be to create environment-specific API keys (e.g., `prod-reader`) to enforce stricter isolation if needed. + +### 6.4 Network Security + +- **VPC:** The entire registry service (API, database) should be deployed within a secure Virtual Private Cloud (VPC). +- **Access:** By default, the registry API endpoint should only be accessible from within the VPC. If access from outside the VPC is required (e.g., for developers), it should be managed through a secure API Gateway with appropriate controls. + +## 7. Tooling and Dependencies + +This section summarizes the recommended technologies and tools required to build, deploy, and maintain the MCP Registry, focusing on an AWS-native implementation. + +### 7.1 Core Application + +- **Programming Language:** Python (3.11+) +- **API Framework:** FastAPI +- **Data Validation:** Pydantic (included with FastAPI) +- **Database Interaction:** SQLAlchemy (with `asyncpg` for asynchronous support) +- **Web Server:** Uvicorn + +### 7.2 Database + +- **System:** PostgreSQL (15+) +- **Recommended Hosting:** AWS RDS for PostgreSQL + +### 7.3 Infrastructure and Deployment + +- **Containerization:** Docker +- **Container Registry:** AWS Elastic Container Registry (ECR) +- **Compute Hosting:** AWS Fargate +- **CI/CD Pipeline:** GitHub Actions or GitLab CI + +### 7.4 Developer Experience + +- **Registry CLI:** A critical support tool will be a dedicated CLI to streamline interaction with the registry. + - **Technology:** **Typer** or **Click** (Python). + - **Features:** + - `mcp-cli validate`: Validates the structure and schema of a local `agent.json` file. + - `mcp-cli publish`: Publishes an `agent.json` file to the registry. Reads API key from an environment variable. + - `mcp-cli search `: A command-line interface for searching the registry. +- **Code Quality:** + - **Linter:** Ruff + - **Formatter:** Black + - **Testing Framework:** Pytest + +## 8. Implementation Roadmap + +This roadmap outlines a phased approach to delivering the MCP Registry. + +### Phase 1: Core MVP (2-3 Sprints) + +- **Goal:** A functional, deployed registry service for a single environment. +- **Tasks:** + - Set up the core FastAPI application structure. + - Implement the data model in PostgreSQL. + - Develop API endpoints for `POST /agents` and `GET /agents/{name}/{version}`. + - Implement API Key authentication (`Reader` and `Publisher` roles). + - Set up the initial CI/CD pipeline to deploy the service as a container on AWS Fargate. + - Onboard 1-2 pilot AI agent teams. + +### Phase 2: CLI and Search (1-2 Sprints) + +- **Goal:** Improve developer experience and discoverability. +- **Tasks:** + - Develop the `mcp-cli` tool with `validate` and `publish` commands. + - Integrate the `publish` command into the CI/CD pipelines of the pilot agent teams. + - Implement the remaining `GET` endpoints (`/agents`, `/agents/{name}`). + - Implement the `GET /search` endpoint with full-text search on key metadata fields. + +### Phase 3: Hardening and Rollout (Ongoing) + +- **Goal:** Prepare for full production use and wider adoption. +- **Tasks:** + - Implement soft-delete (`DELETE`) functionality. + - Add the `Admin` role and a secure mechanism for managing API keys. + - Enhance monitoring, logging, and alerting for the registry service. + - Create comprehensive user documentation for both the API and the CLI. + - Onboard all ZSoftly AI agent teams. \ No newline at end of file From 0d8fb55df2df470fb7a8d29a3e37fd63d6587542 Mon Sep 17 00:00:00 2001 From: modupei Date: Wed, 17 Dec 2025 22:12:54 +0100 Subject: [PATCH 2/7] Fix markdown formatting issues --- mcp/MCP_Registry_Plan.md | 261 ++++++++++++++++++++------------------- 1 file changed, 132 insertions(+), 129 deletions(-) diff --git a/mcp/MCP_Registry_Plan.md b/mcp/MCP_Registry_Plan.md index b6c305d..486a288 100644 --- a/mcp/MCP_Registry_Plan.md +++ b/mcp/MCP_Registry_Plan.md @@ -12,9 +12,9 @@ An MCP Registry is **not** a hosting platform for AI models. Instead, it is a ** It works by storing metadata about "MCP Servers" (in our case, ZSoftly AI Agents). Each agent is described by a manifest file (e.g., `agent.json`) which contains: -- **Identity:** Name, version, owner, description. -- **Contract:** An OpenAPI (or similar) specification that defines the agent's capabilities as a set of callable tools or functions. -- **Location:** The network endpoint (URL) where the agent can be reached. +- **Identity:** Name, version, owner, description. +- **Contract:** An OpenAPI (or similar) specification that defines the agent's capabilities as a set of callable tools or functions. +- **Location:** The network endpoint (URL) where the agent can be reached. Client applications query the registry to find agents that meet their needs (e.g., "find an agent that can summarize text"). The registry returns the manifest of the matching agent(s), and the client then uses that information to communicate directly with the agent. @@ -30,68 +30,68 @@ This file is the source of truth for an agent's metadata. Each agent project MUS **Fields:** -- `name` (string, required): A unique, machine-readable name for the agent (e.g., `document-summarizer`, `user-profile-service`). -- `version` (string, required): The semantic version of the agent (e.g., `1.0.0`). -- `displayName` (string, optional): A human-readable name for display in UIs. -- `description` (string, required): A clear, concise summary of the agent's purpose and capabilities. -- `owner` (string, required): The ZSoftly team or individual responsible for the agent (e.g., `team-alpha`, `user@zsoftly.com`). -- `endpoint` (string, required): The base URL where the agent's service can be accessed. -- `openapi` (object, required): A valid OpenAPI 3.x specification object describing the agent's available `paths` and `components`. This is the contract for using the agent. -- `environment` (enum, required): The environment this agent is deployed to. (`dev`, `stage`, or `prod`). This is critical for multi-environment support. -- `tags` (array of strings, optional): Keywords to improve discoverability (e.g., `["nlp", "text", "summary"]`). + - `name` (string, required): A unique, machine-readable name for the agent (e.g., `document-summarizer`, `user-profile-service`). + - `version` (string, required): The semantic version of the agent (e.g., `1.0.0`). + - `displayName` (string, optional): A human-readable name for display in UIs. + - `description` (string, required): A clear, concise summary of the agent's purpose and capabilities. + - `owner` (string, required): The ZSoftly team or individual responsible for the agent (e.g., `team-alpha`, `user@zsoftly.com`). + - `endpoint` (string, required): The base URL where the agent's service can be accessed. + - `openapi` (object, required): A valid OpenAPI 3.x specification object describing the agent's available `paths` and `components`. This is the contract for using the agent. + - `environment` (enum, required): The environment this agent is deployed to. (`dev`, `stage`, or `prod`). This is critical for multi-environment support. + - `tags` (array of strings, optional): Keywords to improve discoverability (e.g., `["nlp", "text", "summary"]`). ### 3.2 Registry API Endpoints The registry will expose the following core endpoints: -**Agent Registration** +#### Agent Registration -- `POST /agents` - - **Action:** Publishes or updates an agent. The request body will be the `agent.json` manifest. - - **Logic:** The registry validates the manifest against a schema. If an agent with the same `name` and `version` already exists, it is updated. Otherwise, a new version is created. The `name` and `version` from the manifest are used as the unique identifier. - - **Auth:** Requires a publisher role. +- `POST /agents` + - **Action:** Publishes or updates an agent. The request body will be the `agent.json` manifest. + - **Logic:** The registry validates the manifest against a schema. If an agent with the same `name` and `version` already exists, it is updated. Otherwise, a new version is created. The `name` and `version` from the manifest are used as the unique identifier. + - **Auth:** Requires a publisher role. -**Agent Discovery** +#### Agent Discovery -- `GET /agents` - - **Action:** Returns a paginated list of all registered agents (latest version of each). - - **Auth:** Requires a reader role. +- `GET /agents` -- `GET /agents/{name}` - - **Action:** Returns a list of all available versions for a specific agent. - - **Auth:** Requires a reader role. + - **Action:** Returns a paginated list of all registered agents (latest version of each). + - **Auth:** Requires a reader role. -- `GET /agents/{name}/{version}` - - **Action:** Returns the specific `agent.json` for the given name and version (e.g., `/agents/document-summarizer/1.2.0`). Use `latest` as the version to get the most recently published version. - - **Auth:** Requires a reader role. +- `GET /agents/{name}` -**Agent Search** + - **Action:** Returns a list of all available versions for a specific agent. + - **Auth:** Requires a reader role. -- `GET /search?q={query}` - - **Action:** Searches for agents. The query will match against `name`, `displayName`, `description`, and `tags`. - - **Example:** `/search?q=summarize` - - **Auth:** Requires a reader role. +- `GET /agents/{name}/{version}` + - **Action:** Returns the specific `agent.json` for the given name and version (e.g., `/agents/document-summarizer/1.2.0`). Use `latest` as the version to get the most recently published version. + - **Auth:** Requires a reader role. -**Agent Deletion** +#### Agent Search -- `DELETE /agents/{name}/{version}` - - **Action:** Deletes a specific version of an agent. This is a soft-delete to prevent breaking dependencies. A hard-delete mechanism can be decided on later. - - **Auth:** Requires a publisher/admin role. +- `GET /search?q={query}` + - **Action:** Searches for agents. The query will match against `name`, `displayName`, `description`, and `tags`. + - **Example:** `/search?q=summarize` + - **Auth:** Requires a reader role. + +#### Agent Deletion + +- `DELETE /agents/{name}/{version}` + - **Action:** Deletes a specific version of an agent. This is a soft-delete to prevent breaking dependencies. A hard-delete mechanism can be decided on later. + - **Auth:** Requires a publisher/admin role. ### 3.3 Versioning Strategy -- The registry will use **Semantic Versioning (SemVer)** as specified in the `agent.json` (`major.minor.patch`). -- A combination of `name` and `version` uniquely identifies an agent manifest. -- Clients should be encouraged to request a specific major version (e.g., `1.x`) or use the `latest` tag with caution in development environments. The registry can resolve `1.x` to the latest non-breaking version. -- The `latest` tag will always point to the most recently published version, regardless of whether it's a breaking change. +- The registry will use **Semantic Versioning (SemVer)** as specified in the `agent.json` (`major.minor.patch`). +- A combination of `name` and `version` uniquely identifies an agent manifest. +- Clients should be encouraged to request a specific major version (e.g., `1.x`) or use the `latest` tag with caution in development environments. The registry can resolve `1.x` to the latest non-breaking version. +- The `latest` tag will always point to the most recently published version, regardless of whether it's a breaking change. ## 4. Proposed Architecture The MCP Registry will be a simple, robust, and scalable system composed of three core components. The architecture prioritizes statelessness and cloud-native principles. -The MCP Registry will be a simple, robust, and scalable system composed of three core components. The architecture prioritizes statelessness and cloud-native principles. - -``` +```text +---------------------------------------------------------------------------------------+ | MCP Registry | | | @@ -119,34 +119,36 @@ The MCP Registry will be a simple, robust, and scalable system composed of three ### 4.1 System Components 1. **REST API (Registry Service):** - - **Description:** The heart of the system, this service exposes all the endpoints defined in the "Agent Lifecycle Management" section. It handles incoming requests, authentication, and communication with the database. - - **Technology:** A stateless web application. - - **Recommendation:** **FastAPI (Python)**. Its native support for Pydantic models makes validation of incoming `agent.json` manifests trivial. Its automatic generation of OpenAPI documentation is also a significant benefit. + + - **Description:** The heart of the system, this service exposes all the endpoints defined in the "Agent Lifecycle Management" section. It handles incoming requests, authentication, and communication with the database. + - **Technology:** A stateless web application. + - **Recommendation:** **FastAPI (Python)**. Its native support for Pydantic models makes validation of incoming `agent.json` manifests trivial. Its automatic generation of OpenAPI documentation is also a significant benefit. 2. **Database:** - - **Description:** A persistent storage layer for all agent manifests and their versions. It needs to support structured data, indexing for fast lookups, and full-text search capabilities for the `GET /search` endpoint. - - **Recommendation:** **PostgreSQL**. Its robustness, support for JSONB data types, and powerful indexing features make it a perfect fit. Using a managed cloud service (like AWS RDS) is highly recommended. + + - **Description:** A persistent storage layer for all agent manifests and their versions. It needs to support structured data, indexing for fast lookups, and full-text search capabilities for the `GET /search` endpoint. + - **Recommendation:** **PostgreSQL**. Its robustness, support for JSONB data types, and powerful indexing features make it a perfect fit. Using a managed cloud service (like AWS RDS) is highly recommended. 3. **Validation Logic:** - - **Description:** Before any data is written to the database, incoming `agent.json` manifests must be validated. This includes checking for required fields, correct data types, and ensuring the embedded OpenAPI contract is syntactically valid. - - **Implementation:** This logic will be implemented directly within the REST API service, leveraging the data validation features of the chosen framework (e.g., Pydantic models in FastAPI). + - **Description:** Before any data is written to the database, incoming `agent.json` manifests must be validated. This includes checking for required fields, correct data types, and ensuring the embedded OpenAPI contract is syntactically valid. + - **Implementation:** This logic will be implemented directly within the REST API service, leveraging the data validation features of the chosen framework (e.g., Pydantic models in FastAPI). ### 4.2 Data Model A primary `agents` table will store the core metadata. -- `id` (PK) -- `name` (Indexed) -- `version` (Indexed) -- `display_name` -- `description` -- `owner` -- `endpoint` -- `environment` (Indexed) -- `tags` (Indexed, using GIN index on a JSONB field) -- `openapi_spec` (JSONB) -- `created_at` -- `is_deleted` (for soft deletes) + - `id` (PK) + - `name` (Indexed) + - `version` (Indexed) + - `display_name` + - `description` + - `owner` + - `endpoint` + - `environment` (Indexed) + - `tags` (Indexed, using GIN index on a JSONB field) + - `openapi_spec` (JSONB) + - `created_at` + - `is_deleted` (for soft deletes) A unique constraint will be placed on `(name, version)` to enforce uniqueness. @@ -158,21 +160,21 @@ Our hosting strategy should be cloud-first, prioritizing managed services to red This approach offers the best balance of performance, scalability, and ease of management. -- **Compute:** **AWS Fargate**. - - The FastAPI application will be packaged into a **Docker container**. - - These platforms automatically manage scaling (including scaling to zero), patching, and server infrastructure. -- **Database:** **AWS RDS for PostgreSQL**. - - A managed database service handles backups, failover, and maintenance. -- **CI/CD:** The process of publishing the registry itself (and agents to it) should be automated. A GitHub Actions or GitLab CI pipeline would build the Docker image, push it to a container registry (e.g., ECR), and trigger a deployment in Fargate. + - **Compute:** **AWS Fargate**. + - The FastAPI application will be packaged into a **Docker container**. + - These platforms automatically manage scaling (including scaling to zero), patching, and server infrastructure. + - **Database:** **AWS RDS for PostgreSQL**. + - A managed database service handles backups, failover, and maintenance. + - **CI/CD:** The process of publishing the registry itself (and agents to it) should be automated. A GitHub Actions or GitLab CI pipeline would build the Docker image, push it to a container registry (e.g., ECR), and trigger a deployment in Fargate. ### 5.2 Alternative: Serverless Functions For potentially lower-traffic scenarios or as a cost-saving measure, a purely serverless function architecture could be used. -- **Compute:** **AWS Lambda**. - - Each API endpoint or a group of related endpoints would be a separate function. -- **Gateway:** **Amazon API Gateway** would be used to manage the public-facing API routes and trigger the appropriate functions. -- **Database:** **AWS Aurora Serverless** or **DynamoDB**. + - **Compute:** **AWS Lambda**. + - Each API endpoint or a group of related endpoints would be a separate function. + - **Gateway:** **Amazon API Gateway** would be used to manage the public-facing API routes and trigger the appropriate functions. + - **Database:** **AWS Aurora Serverless** or **DynamoDB**. **Conclusion:** The **Serverless Container** approach is the recommended path as it offers a more cohesive development experience and simpler application structure compared to managing numerous individual functions. @@ -182,39 +184,40 @@ Security is a critical component of the registry, ensuring that only authorized ### 6.1 Authentication: API Keys -All requests to the registry API must be authenticated. The proposed method is via **API Keys**. +All requests to the registry API must be authenticated. The proposed method is via **API Keys**. -- **Mechanism:** Clients will pass their key in the `Authorization: Bearer ` HTTP header. -- **Management:** - - API keys will be generated and associated with a specific role (`Reader`, `Publisher`, or `Admin`). - - Keys will be stored securely in the database using a strong hashing algorithm (e.g., SHA-256). They will not be retrievable in plaintext. - - An internal administrative interface or CLI command will be created for managing keys. + - **Mechanism:** Clients will pass their key in the `Authorization: Bearer ` HTTP header. + - **Management:** + - API keys will be generated and associated with a specific role (`Reader`, `Publisher`, or `Admin`). + - Keys will be stored securely in the database using a strong hashing algorithm (e.g., SHA-256). They will not be retrievable in plaintext. + - An internal administrative interface or CLI command will be created for managing keys. ### 6.2 Authorization: Role-Based Access Control (RBAC) Once authenticated, the registry will authorize actions based on pre-defined roles. -- **Roles:** + - **Roles:** + 1. **Reader:** Can perform read-only operations (`GET` endpoints for discovery and search). This role is intended for agent consumers (applications, other agents). 2. **Publisher:** Has `Reader` permissions plus the ability to create, update, and delete agent registrations (`POST`, `DELETE`). This role is intended for developers and CI/CD pipelines responsible for publishing agents. 3. **Admin:** Has full permissions, including the ability to manage API keys. -- **Enforcement:** This will be implemented as a middleware within the API service. The middleware will inspect the role associated with the authenticated API key and verify it against the required role for the requested endpoint. + - **Enforcement:** This will be implemented as a middleware within the API service. The middleware will inspect the role associated with the authenticated API key and verify it against the required role for the requested endpoint. ### 6.3 Multi-Environment Support The registry will support multiple environments (`dev`, `stage`, `prod`) within a single deployed instance. -- **Mechanism:** The `environment` field in the `agent.json` manifest is the primary mechanism for distinguishing agents. -- **Access Strategy:** - - Discovery endpoints (`/agents`, `/search`) will be enhanced with an optional `environment` query parameter (e.g., `GET /agents?environment=prod`). - - Clients (agent consumers) are responsible for requesting the correct environment. For example, a service running in production should be configured to only fetch agents from the `prod` environment. - - This approach provides flexibility. A future enhancement could be to create environment-specific API keys (e.g., `prod-reader`) to enforce stricter isolation if needed. + - **Mechanism:** The `environment` field in the `agent.json` manifest is the primary mechanism for distinguishing agents. + - **Access Strategy:** + - Discovery endpoints (`/agents`, `/search`) will be enhanced with an optional `environment` query parameter (e.g., `GET /agents?environment=prod`). + - Clients (agent consumers) are responsible for requesting the correct environment. For example, a service running in production should be configured to only fetch agents from the `prod` environment. + - This approach provides flexibility. A future enhancement could be to create environment-specific API keys (e.g., `prod-reader`) to enforce stricter isolation if needed. ### 6.4 Network Security -- **VPC:** The entire registry service (API, database) should be deployed within a secure Virtual Private Cloud (VPC). -- **Access:** By default, the registry API endpoint should only be accessible from within the VPC. If access from outside the VPC is required (e.g., for developers), it should be managed through a secure API Gateway with appropriate controls. + - **VPC:** The entire registry service (API, database) should be deployed within a secure Virtual Private Cloud (VPC). + - **Access:** By default, the registry API endpoint should only be accessible from within the VPC. If access from outside the VPC is required (e.g., for developers), it should be managed through a secure API Gateway with appropriate controls. ## 7. Tooling and Dependencies @@ -222,36 +225,36 @@ This section summarizes the recommended technologies and tools required to build ### 7.1 Core Application -- **Programming Language:** Python (3.11+) -- **API Framework:** FastAPI -- **Data Validation:** Pydantic (included with FastAPI) -- **Database Interaction:** SQLAlchemy (with `asyncpg` for asynchronous support) -- **Web Server:** Uvicorn + - **Programming Language:** Python (3.11+) + - **API Framework:** FastAPI + - **Data Validation:** Pydantic (included with FastAPI) + - **Database Interaction:** SQLAlchemy (with `asyncpg` for asynchronous support) + - **Web Server:** Uvicorn ### 7.2 Database -- **System:** PostgreSQL (15+) -- **Recommended Hosting:** AWS RDS for PostgreSQL + - **System:** PostgreSQL (15+) + - **Recommended Hosting:** AWS RDS for PostgreSQL ### 7.3 Infrastructure and Deployment -- **Containerization:** Docker -- **Container Registry:** AWS Elastic Container Registry (ECR) -- **Compute Hosting:** AWS Fargate -- **CI/CD Pipeline:** GitHub Actions or GitLab CI + - **Containerization:** Docker + - **Container Registry:** AWS Elastic Container Registry (ECR) + - **Compute Hosting:** AWS Fargate + - **CI/CD Pipeline:** GitHub Actions or GitLab CI ### 7.4 Developer Experience -- **Registry CLI:** A critical support tool will be a dedicated CLI to streamline interaction with the registry. - - **Technology:** **Typer** or **Click** (Python). - - **Features:** - - `mcp-cli validate`: Validates the structure and schema of a local `agent.json` file. - - `mcp-cli publish`: Publishes an `agent.json` file to the registry. Reads API key from an environment variable. - - `mcp-cli search `: A command-line interface for searching the registry. -- **Code Quality:** - - **Linter:** Ruff - - **Formatter:** Black - - **Testing Framework:** Pytest + - **Registry CLI:** A critical support tool will be a dedicated CLI to streamline interaction with the registry. + - **Technology:** **Typer** or **Click** (Python). + - **Features:** + - `mcp-cli validate`: Validates the structure and schema of a local `agent.json` file. + - `mcp-cli publish`: Publishes an `agent.json` file to the registry. Reads API key from an environment variable. + - `mcp-cli search `: A command-line interface for searching the registry. + - **Code Quality:** + - **Linter:** Ruff + - **Formatter:** Black + - **Testing Framework:** Pytest ## 8. Implementation Roadmap @@ -259,30 +262,30 @@ This roadmap outlines a phased approach to delivering the MCP Registry. ### Phase 1: Core MVP (2-3 Sprints) -- **Goal:** A functional, deployed registry service for a single environment. -- **Tasks:** - - Set up the core FastAPI application structure. - - Implement the data model in PostgreSQL. - - Develop API endpoints for `POST /agents` and `GET /agents/{name}/{version}`. - - Implement API Key authentication (`Reader` and `Publisher` roles). - - Set up the initial CI/CD pipeline to deploy the service as a container on AWS Fargate. - - Onboard 1-2 pilot AI agent teams. + - **Goal:** A functional, deployed registry service for a single environment. + - **Tasks:** + - Set up the core FastAPI application structure. + - Implement the data model in PostgreSQL. + - Develop API endpoints for `POST /agents` and `GET /agents/{name}/{version}`. + - Implement API Key authentication (`Reader` and `Publisher` roles). + - Set up the initial CI/CD pipeline to deploy the service as a container on AWS Fargate. + - Onboard 1-2 pilot AI agent teams. ### Phase 2: CLI and Search (1-2 Sprints) -- **Goal:** Improve developer experience and discoverability. -- **Tasks:** - - Develop the `mcp-cli` tool with `validate` and `publish` commands. - - Integrate the `publish` command into the CI/CD pipelines of the pilot agent teams. - - Implement the remaining `GET` endpoints (`/agents`, `/agents/{name}`). - - Implement the `GET /search` endpoint with full-text search on key metadata fields. + - **Goal:** Improve developer experience and discoverability. + - **Tasks:** + - Develop the `mcp-cli` tool with `validate` and `publish` commands. + - Integrate the `publish` command into the CI/CD pipelines of the pilot agent teams. + - Implement the remaining `GET` endpoints (`/agents`, `/agents/{name}`). + - Implement the `GET /search` endpoint with full-text search on key metadata fields. ### Phase 3: Hardening and Rollout (Ongoing) -- **Goal:** Prepare for full production use and wider adoption. -- **Tasks:** - - Implement soft-delete (`DELETE`) functionality. - - Add the `Admin` role and a secure mechanism for managing API keys. - - Enhance monitoring, logging, and alerting for the registry service. - - Create comprehensive user documentation for both the API and the CLI. - - Onboard all ZSoftly AI agent teams. \ No newline at end of file + - **Goal:** Prepare for full production use and wider adoption. + - **Tasks:** + - Implement soft-delete (`DELETE`) functionality. + - Add the `Admin` role and a secure mechanism for managing API keys. + - Enhance monitoring, logging, and alerting for the registry service. + - Create comprehensive user documentation for both the API and the CLI. + - Onboard all ZSoftly AI agent teams. From a76b24a6404127b257ef146ca833ebc96c5b6f62 Mon Sep 17 00:00:00 2001 From: modupei Date: Fri, 19 Dec 2025 00:27:05 +0100 Subject: [PATCH 3/7] Incorporate feedback from code review --- mcp/MCP_Registry_Plan.md | 212 +++++++++++++++++++++++++++++++++++---- 1 file changed, 192 insertions(+), 20 deletions(-) diff --git a/mcp/MCP_Registry_Plan.md b/mcp/MCP_Registry_Plan.md index 486a288..5ac64ea 100644 --- a/mcp/MCP_Registry_Plan.md +++ b/mcp/MCP_Registry_Plan.md @@ -126,8 +126,9 @@ The MCP Registry will be a simple, robust, and scalable system composed of three 2. **Database:** - - **Description:** A persistent storage layer for all agent manifests and their versions. It needs to support structured data, indexing for fast lookups, and full-text search capabilities for the `GET /search` endpoint. - - **Recommendation:** **PostgreSQL**. Its robustness, support for JSONB data types, and powerful indexing features make it a perfect fit. Using a managed cloud service (like AWS RDS) is highly recommended. + - **Description:** A persistent storage layer for all agent **metadata**. While initially proposed to store the full `openapi_spec` in the database, a more scalable and robust approach is to offload large objects to a dedicated object store. The database will continue to support structured data, indexing for fast lookups, and full-text search capabilities for the `GET /search` endpoint. + - **Recommendation:** **PostgreSQL**. Its robustness, support for JSONB data types for metadata, and powerful indexing features make it a perfect fit. Using a managed cloud service (like AWS RDS) is highly recommended. + - **OpenAPI Spec Storage:** To improve auditability and reduce database load, OpenAPI specifications will be stored in **Amazon S3**. The database will store a reference to the S3 object, including a checksum, to ensure integrity. This approach also enables version immutability, as a new version of an agent will point to a new, immutable S3 object. 3. **Validation Logic:** - **Description:** Before any data is written to the database, incoming `agent.json` manifests must be validated. This includes checking for required fields, correct data types, and ensuring the embedded OpenAPI contract is syntactically valid. @@ -146,12 +147,38 @@ A primary `agents` table will store the core metadata. - `endpoint` - `environment` (Indexed) - `tags` (Indexed, using GIN index on a JSONB field) - - `openapi_spec` (JSONB) + - `openapi_spec_s3_uri` (string) + - `openapi_spec_checksum` (string) - `created_at` - `is_deleted` (for soft deletes) A unique constraint will be placed on `(name, version)` to enforce uniqueness. +### 4.3 Environment Isolation Strategy + +The initial plan proposed a single registry instance for all environments, relying on an `environment` field for logical separation. However, a production-grade implementation requires stronger isolation to prevent environment bleed-through and simplify security management. + +**Strong Recommendation: One Registry per Environment** + +To achieve robust isolation, we will deploy a separate, independent registry instance for each environment: + +- `mcp-registry-dev` +- `mcp-registry-stage` +- `mcp-registry-prod` + +This approach provides several key advantages: + +- **Data Isolation:** Each environment has its own dedicated database, preventing accidental access to or modification of production agent metadata from lower environments. +- **Simplified IAM:** AWS IAM policies can be scoped directly to the resources of a single environment, simplifying access control and reducing the risk of misconfiguration. +- **Incident Containment:** An issue in the `dev` or `stage` registry will not impact the `prod` registry, limiting the blast radius of any incidents. +- **Clear API Scoping:** Consumers of the registry will use a different endpoint for each environment, making it explicit which set of agents they are interacting with. + +If, for operational reasons, a single instance must be retained, the following controls would be mandatory: + +- **Environment-Scoped API Keys:** API keys would be generated with a specific environment scope, and the API would enforce that a `dev` key cannot be used to publish to the `prod` environment. +- **Environment-Based Authorization:** Middleware would be required to enforce authorization based on the environment, ensuring that a user's role and the environment of the resource being accessed are both validated. + + ## 5. Hosting and Deployment Our hosting strategy should be cloud-first, prioritizing managed services to reduce operational overhead. @@ -163,8 +190,17 @@ This approach offers the best balance of performance, scalability, and ease of m - **Compute:** **AWS Fargate**. - The FastAPI application will be packaged into a **Docker container**. - These platforms automatically manage scaling (including scaling to zero), patching, and server infrastructure. + - **Best Practices:** + - Enable autoscaling based on request count or CPU utilization to handle fluctuating loads. + - Run at least two tasks per service for high availability. + - Implement graceful shutdown hooks in FastAPI to ensure that in-flight requests are not dropped during deployments. - **Database:** **AWS RDS for PostgreSQL**. - A managed database service handles backups, failover, and maintenance. + - **Best Practices:** + - Enable Multi-AZ for high availability and automated failover. + - Configure automated backups and Point-In-Time-Recovery (PITR) to protect against data loss. + - Enable encryption at rest using AWS KMS to protect sensitive data. + - Implement connection pooling using PgBouncer or by tuning the SQLAlchemy pool settings to efficiently manage database connections. - **CI/CD:** The process of publishing the registry itself (and agents to it) should be automated. A GitHub Actions or GitLab CI pipeline would build the Docker image, push it to a container registry (e.g., ECR), and trigger a deployment in Fargate. ### 5.2 Alternative: Serverless Functions @@ -176,35 +212,82 @@ For potentially lower-traffic scenarios or as a cost-saving measure, a purely se - **Gateway:** **Amazon API Gateway** would be used to manage the public-facing API routes and trigger the appropriate functions. - **Database:** **AWS Aurora Serverless** or **DynamoDB**. -**Conclusion:** The **Serverless Container** approach is the recommended path as it offers a more cohesive development experience and simpler application structure compared to managing numerous individual functions. +### 5.3 Infrastructure as Code + +The entire infrastructure for the MCP Registry will be managed using Infrastructure as Code (IaC) to ensure consistency, repeatability, and version control. + +- **Technology:** We will use **Terraform** or **OpenTofu** to define and manage all cloud resources, including: + - VPC, subnets, and networking rules + - ECS cluster and Fargate services + - RDS database + - IAM roles and policies + - Secrets in AWS Secrets Manager +- **Structure:** The IaC code will be organized into versioned modules, with separate state files for each environment (`dev`, `stage`, `prod`). This will allow for safe and predictable deployments to each environment. + +### 5.4 Conclusion ## 6. Security and Access Control Security is a critical component of the registry, ensuring that only authorized users and services can publish or access agent information. -### 6.1 Authentication: API Keys +### 6.1 Authentication: From API Keys to IAM and OIDC + +The initial plan proposed API keys for authentication. While simple to implement, API keys are not sufficient for a production-grade internal platform due to the risks of key leakage, lack of identity context, and difficult rotation. -All requests to the registry API must be authenticated. The proposed method is via **API Keys**. +For production, we will adopt a more secure, identity-based authentication model. - - **Mechanism:** Clients will pass their key in the `Authorization: Bearer ` HTTP header. - - **Management:** - - API keys will be generated and associated with a specific role (`Reader`, `Publisher`, or `Admin`). - - Keys will be stored securely in the database using a strong hashing algorithm (e.g., SHA-256). They will not be retrievable in plaintext. - - An internal administrative interface or CLI command will be created for managing keys. +**Recommended Authentication Model (Best Practice)** + +- **Primary Method: IAM + SigV4 Authentication** + - The registry will be deployed behind an Application Load Balancer (ALB) or API Gateway, and access will be controlled via AWS IAM. + - Human users and service accounts (e.g., CI/CD pipelines, other agents) will be granted specific IAM roles that allow them to assume a role to access the registry. + - All requests to the registry API must be signed with AWS Signature Version 4 (SigV4), which provides strong authentication and identity context. + - This model eliminates the need for static secrets like API keys, as authentication is based on short-lived credentials obtained by assuming an IAM role. + +- **Alternative (if IAM is not feasible): OAuth2 / OIDC** + - If a pure IAM-based approach is not feasible, an alternative is to use an OpenID Connect (OIDC) provider, such as GitHub Actions OIDC or an internal Identity Provider (IdP). + - Clients would authenticate with the IdP to obtain a short-lived JSON Web Token (JWT), which would then be passed to the registry API. + - The API would validate the JWT and extract the user's identity and permissions. + +- **API Keys (Limited Use)** + - API keys will be deprecated for production use. + - They may be retained for limited use cases, such as: + - **Local development:** To provide a simple way for developers to interact with the `dev` registry. + - **Transitional usage:** To support legacy clients during a migration period. + - **CLI bootstrap:** To perform initial setup or administrative tasks. + - If used, API keys must have a clear owner, an expiration date, and be stored securely in AWS Secrets Manager. ### 6.2 Authorization: Role-Based Access Control (RBAC) -Once authenticated, the registry will authorize actions based on pre-defined roles. +Once a user or service is authenticated, the registry will authorize actions based on pre-defined roles. While the initial plan defined roles at the endpoint level, a production-grade system requires more granular, resource-level authorization. - **Roles:** 1. **Reader:** Can perform read-only operations (`GET` endpoints for discovery and search). This role is intended for agent consumers (applications, other agents). - 2. **Publisher:** Has `Reader` permissions plus the ability to create, update, and delete agent registrations (`POST`, `DELETE`). This role is intended for developers and CI/CD pipelines responsible for publishing agents. - 3. **Admin:** Has full permissions, including the ability to manage API keys. + 2. **Publisher:** Has `Reader` permissions plus the ability to create and update agent registrations (`POST`). + 3. **Admin:** Has full permissions, including the ability to manage API keys and perform administrative actions. + + - **Resource-Level Authorization:** + - A key improvement is to enforce ownership-based permissions. A `Publisher` should only be able to publish or update agents they own. The `owner` field in the `agent.json` manifest will be used to enforce this. + - Admin actions, such as deleting an agent version or modifying a user's role, must be fully audited. + + - **Enforcement:** This will be implemented as a middleware within the API service. The middleware will inspect the user's identity and roles, and then verify them against the requested action and the resource being accessed. + + - **Immutability Rules:** + - To ensure auditability and prevent accidental overwrites, the registry will enforce version immutability. + - It will be forbidden to `POST` an agent with the same `name` and `version` as an existing agent. + - To update an agent, a new version must be published. This forces a clear version history and simplifies rollbacks. + +### 6.3 Secrets Management - - **Enforcement:** This will be implemented as a middleware within the API service. The middleware will inspect the role associated with the authenticated API key and verify it against the required role for the requested endpoint. +A robust secrets management strategy is critical to the security of the registry. The following principles will be applied: -### 6.3 Multi-Environment Support +- **Secrets Storage:** All secrets, including API keys, database credentials, and any other sensitive information, will be stored in **AWS Secrets Manager**. +- **No Hardcoded Secrets:** Secrets will never be hardcoded in the application code, configuration files, or environment variables. +- **IAM Integration:** The application will be granted IAM permissions to retrieve secrets from Secrets Manager at runtime. This avoids the need to manage secrets within the application itself. +- **Database Credentials:** If IAM authentication is used for the database, no credentials need to be managed. If username/password authentication is used, the credentials will be stored in Secrets Manager and rotated regularly. + +### 6.4 Multi-Environment Support The registry will support multiple environments (`dev`, `stage`, `prod`) within a single deployed instance. @@ -214,10 +297,16 @@ The registry will support multiple environments (`dev`, `stage`, `prod`) within - Clients (agent consumers) are responsible for requesting the correct environment. For example, a service running in production should be configured to only fetch agents from the `prod` environment. - This approach provides flexibility. A future enhancement could be to create environment-specific API keys (e.g., `prod-reader`) to enforce stricter isolation if needed. -### 6.4 Network Security +### 6.5 Network Security + +The registry will be deployed with a defense-in-depth network security strategy. - - **VPC:** The entire registry service (API, database) should be deployed within a secure Virtual Private Cloud (VPC). - - **Access:** By default, the registry API endpoint should only be accessible from within the VPC. If access from outside the VPC is required (e.g., for developers), it should be managed through a secure API Gateway with appropriate controls. + - **VPC:** The entire registry service (API, database) will be deployed within a secure Virtual Private Cloud (VPC). + - **Private ALB:** The API will be fronted by a private Application Load Balancer (ALB), ensuring that it is not directly exposed to the public internet. + - **Security Groups:** Security groups will be used to enforce the principle of least privilege. For example, the ALB security group will only allow inbound traffic on the required port from specific sources, and the Fargate service security group will only allow inbound traffic from the ALB. + - **NACLs:** Network Access Control Lists (NACLs) will be used as a stateless firewall to provide an additional layer of security at the subnet level. + - **WAF:** If public access is ever required, AWS WAF will be used to protect the API from common web exploits. + - **Access:** By default, the registry API endpoint will only be accessible from within the VPC. If access from outside the VPC is required (e.g., for developers), it will be managed through a secure API Gateway with appropriate controls. ## 7. Tooling and Dependencies @@ -256,7 +345,90 @@ This section summarizes the recommended technologies and tools required to build - **Formatter:** Black - **Testing Framework:** Pytest -## 8. Implementation Roadmap +## 8. CI/CD & Supply Chain Security + +A secure and reliable CI/CD pipeline is critical for both the registry itself and the agents that are published to it. + +### 8.1 Registry Deployment Pipeline + +The pipeline that builds and deploys the MCP Registry will include the following security gates: + +- **SAST (Static Application Security Testing):** Tools like `Bandit` and `Semgrep` will be used to scan the Python code for security vulnerabilities. +- **Dependency Scanning:** The pipeline will scan all third-party dependencies for known vulnerabilities. +- **Container Image Scanning:** The Docker image will be scanned for vulnerabilities using tools like `Trivy` or `Grype`. +- **Signed Images:** All container images will be signed using `cosign` to ensure their integrity and authenticity. + +### 8.2 Agent Publishing Pipeline + +The pipeline that publishes new agents to the registry is a critical control point. To prevent the publishing of malicious or non-compliant agents, the following controls will be implemented: + +- **Schema Validation:** The pipeline will validate the `agent.json` manifest against a strict schema. +- **Version Immutability:** The pipeline will enforce the version immutability rule, failing any attempt to overwrite an existing version. +- **Approval Gates for Production:** Any deployment to the `prod` registry will require a manual approval step from a designated approver. +- **OIDC-Based Identity:** The CI/CD pipeline will use an OIDC-based identity to authenticate with the registry, eliminating the need for static API keys. + +## 9. Observability & Operations + +A comprehensive observability strategy is essential for maintaining the health and performance of the MCP Registry. + +### 9.1 Metrics + +The following key metrics will be collected and monitored: + +- **Request Count:** The number of requests to each API endpoint. +- **Error Rate:** The percentage of requests that result in an error. +- **Publish Frequency:** The number of new agents published over time. +- **Search Latency:** The latency of the search API. + +### 9.2 Logs + +- **Structured JSON Logs:** All logs will be in a structured JSON format to facilitate searching and analysis. +- **Key Events:** The application will log key events, including: + - Agent publish/delete events + - Authentication failures + - Authorization failures + +### 9.3 Tracing + +- **OpenTelemetry Support:** The application will include optional support for OpenTelemetry to enable distributed tracing. + +### 9.4 Alerting + +Alerts will be configured for the following conditions: + +- **Registry Unavailable:** The registry is not responding to requests. +- **DB Connectivity Failure:** The application cannot connect to the database. +- **Unauthorized Publish Attempts:** A user or service attempts to publish an agent without the required permissions. + +## 10. Data Governance & Compliance + +A clear data governance and compliance strategy is essential for ensuring the integrity and auditability of the MCP Registry. + +### 10.1 Audit Log + +- **Audit Log Table:** A dedicated audit log table will be created in the database to record all significant events, including: + - Who published an agent + - When the agent was published + - The source IP address of the publish request + - What changed in the agent manifest +- **Read-Only Historical View:** A read-only view of the audit log will be provided for compliance and debugging purposes. + +### 10.2 Data Retention + +- **Retention Policy:** A retention policy will be enforced for soft-deleted agents. After a defined period, soft-deleted agents will be hard-deleted from the database. +- **Legal Holds:** The system will provide a mechanism to place a legal hold on an agent, preventing it from being hard-deleted. + +## 11. MCP-Specific Design Considerations + +### 11.1 Registry Semantics + +The registry's primary responsibility is discovery, not model hosting. The following enhancements will be made to the registry's semantics: + +- **Capability-Based Search:** In addition to free-text search, the registry will support capability-based search. This will allow clients to search for agents that support a specific tool or function, as defined in their OpenAPI spec. +- **Deprecation Metadata:** The `agent.json` manifest will be extended to include deprecation metadata, such as `deprecated: true` and `sunset_date`. This will allow clients to gracefully migrate away from deprecated agents. +- **Health Metadata:** The registry will optionally store health metadata for each agent, such as the last time the agent was seen alive. This will allow clients to avoid routing requests to unhealthy agents. + +## 12. Implementation Roadmap This roadmap outlines a phased approach to delivering the MCP Registry. From b93b0980af309ebef1bfc1ed72ca2b6d8bfd4b81 Mon Sep 17 00:00:00 2001 From: modupei Date: Mon, 22 Dec 2025 18:26:55 +0100 Subject: [PATCH 4/7] Updated with environment isolation consistency, authentication boundaries --- mcp/MCP_Registry_Plan.md | 40 +++++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/mcp/MCP_Registry_Plan.md b/mcp/MCP_Registry_Plan.md index 5ac64ea..916fbaf 100644 --- a/mcp/MCP_Registry_Plan.md +++ b/mcp/MCP_Registry_Plan.md @@ -82,11 +82,11 @@ The registry will expose the following core endpoints: ### 3.3 Versioning Strategy -- The registry will use **Semantic Versioning (SemVer)** as specified in the `agent.json` (`major.minor.patch`). -- A combination of `name` and `version` uniquely identifies an agent manifest. -- Clients should be encouraged to request a specific major version (e.g., `1.x`) or use the `latest` tag with caution in development environments. The registry can resolve `1.x` to the latest non-breaking version. -- The `latest` tag will always point to the most recently published version, regardless of whether it's a breaking change. - + - The registry will use **Semantic Versioning (SemVer)** as specified in the `agent.json` (`major.minor.patch`). + - A combination of `name` and `version` uniquely identifies an agent manifest. + - Clients should be encouraged to request a specific major version (e.g., `1.x`) or use the `latest` tag with caution. + - **Version Range Resolution (`1.x`):** When a client requests a major version like `1.x`, the registry will resolve this to the **highest available patch version of the highest available minor version within that major version**. (e.g., for `1.x`, if `1.2.0` and `1.3.5` exist, it resolves to `1.3.5`). This ensures the most up-to-date non-breaking version is provided. + - **`latest` Tag:** The `latest` tag will always point to the most recently published version, **regardless of whether it introduces breaking changes**. Consumers are advised to use `latest` with extreme caution in production environments, as it offers no guarantee of backward compatibility. ## 4. Proposed Architecture The MCP Registry will be a simple, robust, and scalable system composed of three core components. The architecture prioritizes statelessness and cloud-native principles. @@ -249,13 +249,19 @@ For production, we will adopt a more secure, identity-based authentication model - Clients would authenticate with the IdP to obtain a short-lived JSON Web Token (JWT), which would then be passed to the registry API. - The API would validate the JWT and extract the user's identity and permissions. -- **API Keys (Limited Use)** - - API keys will be deprecated for production use. - - They may be retained for limited use cases, such as: - - **Local development:** To provide a simple way for developers to interact with the `dev` registry. - - **Transitional usage:** To support legacy clients during a migration period. - - **CLI bootstrap:** To perform initial setup or administrative tasks. - - If used, API keys must have a clear owner, an expiration date, and be stored securely in AWS Secrets Manager. +- **API Keys (Limited Use)** + + - API keys will be deprecated for production use and are considered a **transitional mechanism for Phase 1 MVP and development environments only**. + + - They may be retained for limited, temporary use cases, such as: + + - **Local development:** To provide a simple way for developers to interact with the `dev` registry. + + - **CLI bootstrap:** To perform initial setup or administrative tasks in development. + + - **Strict Policy:** All production environments and CI/CD pipelines (even for development environments) MUST utilize IAM + SigV4 or OIDC-based authentication. A clear migration path from API keys to IAM/OIDC will be established as part of Phase 2. + + - If used, API keys must have a clear owner, an expiration date, and be stored securely in AWS Secrets Manager. ### 6.2 Authorization: Role-Based Access Control (RBAC) @@ -289,14 +295,10 @@ A robust secrets management strategy is critical to the security of the registry ### 6.4 Multi-Environment Support -The registry will support multiple environments (`dev`, `stage`, `prod`) within a single deployed instance. - - - **Mechanism:** The `environment` field in the `agent.json` manifest is the primary mechanism for distinguishing agents. - - **Access Strategy:** - - Discovery endpoints (`/agents`, `/search`) will be enhanced with an optional `environment` query parameter (e.g., `GET /agents?environment=prod`). - - Clients (agent consumers) are responsible for requesting the correct environment. For example, a service running in production should be configured to only fetch agents from the `prod` environment. - - This approach provides flexibility. A future enhancement could be to create environment-specific API keys (e.g., `prod-reader`) to enforce stricter isolation if needed. + The plan mandates a **"One Registry per Environment"** strategy (as detailed in Section 4.3). This means each environment (`dev`, `stage`, `prod`) will have its own entirely separate and independent registry instance. + - **Mechanism:** The `environment` field in the `agent.json` manifest will be used as metadata within its dedicated registry instance to denote the agent's target environment (e.g., an agent deployed to the `prod` AWS account would have `"environment": "prod"` in its manifest, and would be published only to the `mcp-registry-prod` instance). + - **Access Strategy:** Clients (agent consumers) must be configured to interact with the correct environment-specific registry endpoint. For example, a service running in the `prod` AWS account will query `mcp-registry-prod` exclusively. This physically isolates environments, simplifying security and preventing cross-environment data contamination. ### 6.5 Network Security The registry will be deployed with a defense-in-depth network security strategy. From 7f91a2d51bfe6c1eebe56305a05ec3f1493d0ec7 Mon Sep 17 00:00:00 2001 From: modupei Date: Thu, 8 Jan 2026 00:49:18 +0100 Subject: [PATCH 5/7] feat: Add initial MCP Registry POC files --- mcp/.gitignore | 133 ++++++++++++++++++++++++++++++++ mcp/Dockerfile | 16 ++++ mcp/mcp_registry/crud.py | 52 +++++++++++++ mcp/mcp_registry/database.py | 79 +++++++++++++++++++ mcp/mcp_registry/main.py | 54 +++++++++++++ mcp/mcp_registry/schemas.py | 42 ++++++++++ mcp/pyproject.toml | 20 +++++ mcp/terraform/alb.tf | 76 ++++++++++++++++++ mcp/terraform/ecr.tf | 16 ++++ mcp/terraform/ecs.tf | 78 +++++++++++++++++++ mcp/terraform/iam.tf | 83 ++++++++++++++++++++ mcp/terraform/main.tf | 109 ++++++++++++++++++++++++++ mcp/terraform/outputs.tf | 4 + mcp/terraform/providers.tf | 15 ++++ mcp/terraform/rds.tf | 49 ++++++++++++ mcp/terraform/s3.tf | 38 +++++++++ mcp/terraform/secretsmanager.tf | 18 +++++ mcp/terraform/variables.tf | 51 ++++++++++++ 18 files changed, 933 insertions(+) create mode 100644 mcp/.gitignore create mode 100644 mcp/Dockerfile create mode 100644 mcp/mcp_registry/crud.py create mode 100644 mcp/mcp_registry/database.py create mode 100644 mcp/mcp_registry/main.py create mode 100644 mcp/mcp_registry/schemas.py create mode 100644 mcp/pyproject.toml create mode 100644 mcp/terraform/alb.tf create mode 100644 mcp/terraform/ecr.tf create mode 100644 mcp/terraform/ecs.tf create mode 100644 mcp/terraform/iam.tf create mode 100644 mcp/terraform/main.tf create mode 100644 mcp/terraform/outputs.tf create mode 100644 mcp/terraform/providers.tf create mode 100644 mcp/terraform/rds.tf create mode 100644 mcp/terraform/s3.tf create mode 100644 mcp/terraform/secretsmanager.tf create mode 100644 mcp/terraform/variables.tf diff --git a/mcp/.gitignore b/mcp/.gitignore new file mode 100644 index 0000000..1641d70 --- /dev/null +++ b/mcp/.gitignore @@ -0,0 +1,133 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyderworkspace + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# Terraform +.terraform/ +.terraform.lock.hcl +terraform.tfstate +terraform.tfstate.backup +tfplan +terraform.tfvars +*.plan +*.tfstate +*.tfstate.backup + +# VSCode +.vscode/ + +# PyCharm +.idea/ diff --git a/mcp/Dockerfile b/mcp/Dockerfile new file mode 100644 index 0000000..d3db9c7 --- /dev/null +++ b/mcp/Dockerfile @@ -0,0 +1,16 @@ +# Use an official Python runtime as a parent image +FROM python:3.11-slim + +# Set the working directory in the container +WORKDIR /app + +# Copy the pyproject.toml file and install dependencies +COPY pyproject.toml . +COPY ./mcp_registry ./mcp_registry +RUN pip install --no-cache-dir . + +# Expose the port the app runs on +EXPOSE 80 + +# Run the application +CMD ["uvicorn", "mcp_registry.main:app", "--host", "0.0.0.0", "--port", "80"] diff --git a/mcp/mcp_registry/crud.py b/mcp/mcp_registry/crud.py new file mode 100644 index 0000000..38d56d1 --- /dev/null +++ b/mcp/mcp_registry/crud.py @@ -0,0 +1,52 @@ +from sqlalchemy.orm import Session +from . import models, schemas + +def get_agent(db: Session, agent_id: int): + return db.query(models.Agent).filter(models.Agent.id == agent_id).first() + +def get_agents(db: Session, skip: int = 0, limit: int = 100): + return db.query(models.Agent).offset(skip).limit(limit).all() + +def create_agent(db: Session, agent: schemas.AgentCreate): + db_agent = models.Agent( + name=agent.name, + version=agent.version, + display_name=agent.display_name, + description=agent.description, + owner=agent.owner, + endpoint=agent.endpoint, + openapi_spec=agent.openapi_spec, + environment=agent.environment, + tags=agent.tags, + openapi_spec_s3_uri=agent.openapi_spec_s3_uri, + openapi_spec_checksum=agent.openapi_spec_checksum + ) + for capability_data in agent.capabilities: + capability = get_capability_by_name(db, name=capability_data.name) + if not capability: + capability = create_capability(db, capability_data) + db_agent.capabilities.append(capability) + + db.add(db_agent) + db.commit() + db.refresh(db_agent) + return db_agent + +def get_capability(db: Session, capability_id: int): + return db.query(models.Capability).filter(models.Capability.id == capability_id).first() + +def get_capability_by_name(db: Session, name: str): + return db.query(models.Capability).filter(models.Capability.name == name).first() + +def get_capabilities(db: Session, skip: int = 0, limit: int = 100): + return db.query(models.Capability).offset(skip).limit(limit).all() + +def create_capability(db: Session, capability: schemas.CapabilityCreate): + db_capability = models.Capability(**capability.dict()) + db.add(db_capability) + db.commit() + db.refresh(db_capability) + return db_capability + +def get_agents_by_capability(db: Session, capability_name: str): + return db.query(models.Agent).join(models.Agent.capabilities).filter(models.Capability.name == capability_name).all() diff --git a/mcp/mcp_registry/database.py b/mcp/mcp_registry/database.py new file mode 100644 index 0000000..98c6672 --- /dev/null +++ b/mcp/mcp_registry/database.py @@ -0,0 +1,79 @@ +import os +from dotenv import load_dotenv + +from sqlalchemy import create_engine, Column, String, Text, Boolean, DateTime, ARRAY, Integer, UniqueConstraint, Table, ForeignKey +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import sessionmaker, relationship +from datetime import datetime + +load_dotenv() # Load environment variables from .env file + +# Database connection string from environment variable +DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://user:password@localhost/mcp_registry") + +engine = create_engine(DATABASE_URL) +SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) +Base = declarative_base() + +agent_capability_association = Table( + 'agent_capability_association', + Base.metadata, + Column('agent_id', Integer, ForeignKey('agents.id')), + Column('capability_id', Integer, ForeignKey('capabilities.id')) +) + +class Agent(Base): + __tablename__ = "agents" + + id = Column(Integer, primary_key=True, index=True) # Auto-incrementing primary key + name = Column(String, index=True, nullable=False) + version = Column(String, index=True, nullable=False) + display_name = Column(String, nullable=True) + description = Column(Text, nullable=False) + owner = Column(String, nullable=False) + endpoint = Column(String, nullable=False) + openapi_spec = Column(JSONB, nullable=False) # Storing full OpenAPI spec as JSONB for now + environment = Column(String, index=True, nullable=False) + tags = Column(ARRAY(String), nullable=True) + openapi_spec_s3_uri = Column(String, nullable=True) + openapi_spec_checksum = Column(String, nullable=True) + created_at = Column(DateTime, default=datetime.utcnow) + is_deleted = Column(Boolean, default=False) + + capabilities = relationship( + "Capability", + secondary=agent_capability_association, + back_populates="agents" + ) + + # Unique constraint on name and version + __table_args__ = ( + UniqueConstraint('name', 'version', name='_name_version_uc'), + ) + + def __repr__(self): + return f"" + +class Capability(Base): + __tablename__ = 'capabilities' + + id = Column(Integer, primary_key=True, index=True) + name = Column(String, unique=True, index=True) + description = Column(Text) + + agents = relationship( + "Agent", + secondary=agent_capability_association, + back_populates="capabilities" + ) + +def get_db(): + db = SessionLocal() + try: + yield db + finally: + db.close() + +def create_db_and_tables(): + Base.metadata.create_all(bind=engine) diff --git a/mcp/mcp_registry/main.py b/mcp/mcp_registry/main.py new file mode 100644 index 0000000..9932c31 --- /dev/null +++ b/mcp/mcp_registry/main.py @@ -0,0 +1,54 @@ +# main.py + +from typing import List +from fastapi import FastAPI, Depends +from sqlalchemy.orm import Session + +from . import crud, models, schemas +from .database import SessionLocal, engine, create_db_and_tables + +models.Base.metadata.create_all(bind=engine) + +app = FastAPI() + + +# Dependency +def get_db(): + db = SessionLocal() + try: + yield db + finally: + db.close() + + +@app.on_event("startup") +def on_startup(): + create_db_and_tables() + + +@app.post("/agents/", response_model=schemas.Agent) +def create_agent(agent: schemas.AgentCreate, db: Session = Depends(get_db)): + return crud.create_agent(db=db, agent=agent) + + +@app.get("/agents/", response_model=List[schemas.Agent]) +def read_agents(skip: int = 0, limit: int = 100, db: Session = Depends(get_db)): + agents = crud.get_agents(db, skip=skip, limit=limit) + return agents + + +@app.get("/search/", response_model=List[schemas.Agent]) +def search_agents_by_capability(capability_name: str, db: Session = Depends(get_db)): + return crud.get_agents_by_capability(db=db, capability_name=capability_name) + + +@app.post("/capabilities/", response_model=schemas.Capability) +def create_capability( + capability: schemas.CapabilityCreate, db: Session = Depends(get_db) +): + return crud.create_capability(db=db, capability=capability) + + +@app.get("/") +async def root(): + return {"message": "Hello MCP Registry!"} diff --git a/mcp/mcp_registry/schemas.py b/mcp/mcp_registry/schemas.py new file mode 100644 index 0000000..7691924 --- /dev/null +++ b/mcp/mcp_registry/schemas.py @@ -0,0 +1,42 @@ +from typing import List, Optional +from pydantic import BaseModel + +class CapabilityBase(BaseModel): + name: str + description: Optional[str] = None + +class CapabilityCreate(CapabilityBase): + pass + +class Capability(CapabilityBase): + id: int + agents: List["Agent"] = [] + + class Config: + orm_mode = True + +class AgentBase(BaseModel): + name: str + version: str + display_name: Optional[str] = None + description: str + owner: str + endpoint: str + openapi_spec: dict + environment: str + tags: Optional[List[str]] = None + openapi_spec_s3_uri: Optional[str] = None + openapi_spec_checksum: Optional[str] = None + +class AgentCreate(AgentBase): + capabilities: List[CapabilityCreate] = [] + +class Agent(AgentBase): + id: int + capabilities: List[Capability] = [] + + class Config: + orm_mode = True + +Capability.update_forward_refs() +Agent.update_forward_refs() diff --git a/mcp/pyproject.toml b/mcp/pyproject.toml new file mode 100644 index 0000000..f75766a --- /dev/null +++ b/mcp/pyproject.toml @@ -0,0 +1,20 @@ +[project] +name = "mcp_registry" +version = "0.1.0" +description = "ZSoftly Model Context Protocol (MCP) Registry" +dependencies = [ + "fastapi", + "uvicorn", + "psycopg2-binary", + "SQLAlchemy", + "python-dotenv", +] +requires-python = ">=3.11" +license = "MIT" + +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[tool.setuptools] +packages = ["mcp_registry"] diff --git a/mcp/terraform/alb.tf b/mcp/terraform/alb.tf new file mode 100644 index 0000000..2eadd85 --- /dev/null +++ b/mcp/terraform/alb.tf @@ -0,0 +1,76 @@ +# terraform/alb.tf + +# Security group for the application (for the ALB) +resource "aws_security_group" "app" { + name = "${var.project_name}-app-sg" + description = "Allow traffic to the application" + vpc_id = aws_vpc.main.id + + # Allow inbound traffic from the internet on port 80 + ingress { + from_port = 80 + to_port = 80 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + + # Allow all outbound traffic + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + tags = { + Name = "${var.project_name}-app-sg" + } +} + +# Application Load Balancer +resource "aws_lb" "main" { + name = "${var.project_name}-alb" + internal = false + load_balancer_type = "application" + security_groups = [aws_security_group.app.id] + subnets = aws_subnet.public.*.id + + tags = { + Name = "${var.project_name}-alb" + } +} + +# Target group for the ALB +resource "aws_lb_target_group" "main" { + name = "${var.project_name}-tg" + port = 80 + protocol = "HTTP" + vpc_id = aws_vpc.main.id + target_type = "ip" + + health_check { + path = "/" + protocol = "HTTP" + matcher = "200" + interval = 30 + timeout = 5 + healthy_threshold = 2 + unhealthy_threshold = 2 + } + + tags = { + Name = "${var.project_name}-tg" + } +} + +# Listener for the ALB +resource "aws_lb_listener" "http" { + load_balancer_arn = aws_lb.main.arn + port = "80" + protocol = "HTTP" + + default_action { + type = "forward" + target_group_arn = aws_lb_target_group.main.arn + } +} diff --git a/mcp/terraform/ecr.tf b/mcp/terraform/ecr.tf new file mode 100644 index 0000000..39ca711 --- /dev/null +++ b/mcp/terraform/ecr.tf @@ -0,0 +1,16 @@ +# terraform/ecr.tf + +# ECR repository for the application +resource "aws_ecr_repository" "main" { + name = "${var.project_name}-repo" + image_tag_mutability = "MUTABLE" + force_delete = true + + image_scanning_configuration { + scan_on_push = true + } + + tags = { + Name = "${var.project_name}-repo" + } +} diff --git a/mcp/terraform/ecs.tf b/mcp/terraform/ecs.tf new file mode 100644 index 0000000..529566d --- /dev/null +++ b/mcp/terraform/ecs.tf @@ -0,0 +1,78 @@ +# terraform/ecs.tf + +# ECS Cluster +resource "aws_ecs_cluster" "main" { + name = "${var.project_name}-cluster" + + tags = { + Name = "${var.project_name}-cluster" + } +} + +# ECS Task Definition +resource "aws_ecs_task_definition" "main" { + family = "${var.project_name}-task" + network_mode = "awsvpc" + requires_compatibilities = ["FARGATE"] + cpu = "256" + memory = "512" + execution_role_arn = aws_iam_role.task_execution.arn + task_role_arn = aws_iam_role.task.arn + + container_definitions = jsonencode([ + { + name = "${var.project_name}-container" + image = "${aws_ecr_repository.main.repository_url}:latest" + essential = true + portMappings = [ + { + containerPort = 80 + hostPort = 80 + } + ] + environment = [ + { + name = "DATABASE_URL" + value = "secretsmanager:${aws_secretsmanager_secret.db_credentials.name}" + } + ] + secrets = [ + { + name = "DB_CREDENTIALS" + valueFrom = aws_secretsmanager_secret.db_credentials.arn + } + ] + } + ]) + + tags = { + Name = "${var.project_name}-task-def" + } +} + +# ECS Service +resource "aws_ecs_service" "main" { + name = "${var.project_name}-service" + cluster = aws_ecs_cluster.main.id + task_definition = aws_ecs_task_definition.main.arn + desired_count = 1 + launch_type = "FARGATE" + + network_configuration { + subnets = aws_subnet.private.*.id + security_groups = [aws_security_group.app.id] + assign_public_ip = false + } + + load_balancer { + target_group_arn = aws_lb_target_group.main.arn + container_name = "${var.project_name}-container" + container_port = 80 + } + + depends_on = [aws_lb_listener.http] + + tags = { + Name = "${var.project_name}-service" + } +} diff --git a/mcp/terraform/iam.tf b/mcp/terraform/iam.tf new file mode 100644 index 0000000..c139aa5 --- /dev/null +++ b/mcp/terraform/iam.tf @@ -0,0 +1,83 @@ +# terraform/iam.tf + +# IAM role for the ECS task +resource "aws_iam_role" "task" { + name = "${var.project_name}-task-role" + assume_role_policy = jsonencode({ + Version = "2008-10-17", + Statement = [ + { + Action = "sts:AssumeRole", + Principal = { + Service = "ecs-tasks.amazonaws.com" + }, + Effect = "Allow", + Sid = "" + } + ] + }) + + tags = { + Name = "${var.project_name}-task-role" + } +} + +# IAM role for the ECS task execution +resource "aws_iam_role" "task_execution" { + name = "${var.project_name}-task-execution-role" + assume_role_policy = jsonencode({ + Version = "2008-10-17", + Statement = [ + { + Action = "sts:AssumeRole", + Principal = { + Service = "ecs-tasks.amazonaws.com" + }, + Effect = "Allow", + Sid = "" + } + ] + }) + + tags = { + Name = "${var.project_name}-task-execution-role" + } +} + +# Attach the AmazonECSTaskExecutionRolePolicy to the task execution role +resource "aws_iam_role_policy_attachment" "task_execution" { + role = aws_iam_role.task_execution.name + policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy" +} + +# IAM policy for accessing secrets and S3 +resource "aws_iam_policy" "task_policy" { + name = "${var.project_name}-task-policy" + description = "Policy for the ECS task to access secrets and S3" + policy = jsonencode({ + Version = "2012-10-17", + Statement = [ + { + Action = [ + "secretsmanager:GetSecretValue" + ], + Effect = "Allow", + Resource = [aws_secretsmanager_secret.db_credentials.arn] + }, + { + Action = [ + "s3:GetObject", + "s3:PutObject" + ], + Effect = "Allow", + Resource = ["${aws_s3_bucket.main.arn}/*"] + } + ] + }) +} + +# Attach the policy to the task role +resource "aws_iam_role_policy_attachment" "task_policy" { + role = aws_iam_role.task.name + policy_arn = aws_iam_policy.task_policy.arn +} diff --git a/mcp/terraform/main.tf b/mcp/terraform/main.tf new file mode 100644 index 0000000..9c53d30 --- /dev/null +++ b/mcp/terraform/main.tf @@ -0,0 +1,109 @@ +# terraform/main.tf + +# VPC +resource "aws_vpc" "main" { + cidr_block = var.vpc_cidr_block + enable_dns_hostnames = true + enable_dns_support = true + + tags = { + Name = "${var.project_name}-vpc" + } +} + +# Public Subnets +resource "aws_subnet" "public" { + count = length(var.public_subnet_cidrs) + vpc_id = aws_vpc.main.id + cidr_block = var.public_subnet_cidrs[count.index] + availability_zone = data.aws_availability_zones.available.names[count.index] + map_public_ip_on_launch = true + + tags = { + Name = "${var.project_name}-public-subnet-${count.index + 1}" + } +} + +# Private Subnets +resource "aws_subnet" "private" { + count = length(var.private_subnet_cidrs) + vpc_id = aws_vpc.main.id + cidr_block = var.private_subnet_cidrs[count.index] + availability_zone = data.aws_availability_zones.available.names[count.index] + + tags = { + Name = "${var.project_name}-private-subnet-${count.index + 1}" + } +} + +# Internet Gateway +resource "aws_internet_gateway" "main" { + vpc_id = aws_vpc.main.id + + tags = { + Name = "${var.project_name}-igw" + } +} + +# Elastic IP for NAT Gateway +resource "aws_eip" "nat" { + domain = "vpc" +} + +# NAT Gateway +resource "aws_nat_gateway" "main" { + allocation_id = aws_eip.nat.id + subnet_id = aws_subnet.public[0].id + + tags = { + Name = "${var.project_name}-nat-gw" + } + + depends_on = [aws_internet_gateway.main] +} + +# Public Route Table +resource "aws_route_table" "public" { + vpc_id = aws_vpc.main.id + + route { + cidr_block = "0.0.0.0/0" + gateway_id = aws_internet_gateway.main.id + } + + tags = { + Name = "${var.project_name}-public-rt" + } +} + +# Associate Public Subnets with Public Route Table +resource "aws_route_table_association" "public" { + count = length(var.public_subnet_cidrs) + subnet_id = aws_subnet.public[count.index].id + route_table_id = aws_route_table.public.id +} + +# Private Route Table +resource "aws_route_table" "private" { + vpc_id = aws_vpc.main.id + + route { + cidr_block = "0.0.0.0/0" + nat_gateway_id = aws_nat_gateway.main.id + } + + tags = { + Name = "${var.project_name}-private-rt" + } +} + +# Associate Private Subnets with Private Route Table +resource "aws_route_table_association" "private" { + count = length(var.private_subnet_cidrs) + subnet_id = aws_subnet.private[count.index].id + route_table_id = aws_route_table.private.id +} + +data "aws_availability_zones" "available" { + state = "available" +} diff --git a/mcp/terraform/outputs.tf b/mcp/terraform/outputs.tf new file mode 100644 index 0000000..bfc0136 --- /dev/null +++ b/mcp/terraform/outputs.tf @@ -0,0 +1,4 @@ +output "ecr_repository_url" { + description = "The URL of the ECR repository" + value = aws_ecr_repository.main.repository_url +} diff --git a/mcp/terraform/providers.tf b/mcp/terraform/providers.tf new file mode 100644 index 0000000..b84f2fe --- /dev/null +++ b/mcp/terraform/providers.tf @@ -0,0 +1,15 @@ +# terraform/providers.tf + +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + } +} + +provider "aws" { + region = var.aws_region + profile = var.aws_profile +} diff --git a/mcp/terraform/rds.tf b/mcp/terraform/rds.tf new file mode 100644 index 0000000..68c8f63 --- /dev/null +++ b/mcp/terraform/rds.tf @@ -0,0 +1,49 @@ +# terraform/rds.tf + +# Security group for the RDS instance +resource "aws_security_group" "db" { + name = "${var.project_name}-db-sg" + description = "Allow traffic to the RDS instance" + vpc_id = aws_vpc.main.id + + # Allow inbound traffic from the application security group + ingress { + from_port = 5432 + to_port = 5432 + protocol = "tcp" + security_groups = [aws_security_group.app.id] + } + + tags = { + Name = "${var.project_name}-db-sg" + } +} + +# Subnet group for the RDS instance +resource "aws_db_subnet_group" "main" { + name = "${var.project_name}-db-subnet-group" + subnet_ids = aws_subnet.private.*.id + + tags = { + Name = "${var.project_name}-db-subnet-group" + } +} + +# RDS instance +resource "aws_db_instance" "main" { + allocated_storage = 20 + storage_type = "gp2" + engine = "postgres" + engine_version = "18.1" + instance_class = "db.t3.micro" + db_name = "mcpdatabase" + username = var.db_username + password = var.db_password + db_subnet_group_name = aws_db_subnet_group.main.name + vpc_security_group_ids = [aws_security_group.db.id] + skip_final_snapshot = true + + tags = { + Name = "${var.project_name}-db" + } +} diff --git a/mcp/terraform/s3.tf b/mcp/terraform/s3.tf new file mode 100644 index 0000000..7817933 --- /dev/null +++ b/mcp/terraform/s3.tf @@ -0,0 +1,38 @@ +# terraform/s3.tf + +# S3 bucket for storing OpenAPI specification files +resource "aws_s3_bucket" "main" { + bucket = "${var.project_name}-openapi-specs" + + tags = { + Name = "${var.project_name}-openapi-specs" + } +} + +# Block public access to the S3 bucket +resource "aws_s3_bucket_public_access_block" "main" { + bucket = aws_s3_bucket.main.id + + block_public_acls = true + block_public_policy = true + ignore_public_acls = true + restrict_public_buckets = true +} + +# S3 bucket policy to allow access from the application +resource "aws_s3_bucket_policy" "main" { + bucket = aws_s3_bucket.main.id + policy = data.aws_iam_policy_document.s3_access.json +} + +# IAM policy document for S3 bucket access +data "aws_iam_policy_document" "s3_access" { + statement { + principals { + type = "AWS" + identifiers = [aws_iam_role.task.arn] + } + actions = ["s3:GetObject", "s3:PutObject"] + resources = ["${aws_s3_bucket.main.arn}/*"] + } +} diff --git a/mcp/terraform/secretsmanager.tf b/mcp/terraform/secretsmanager.tf new file mode 100644 index 0000000..9a59fd1 --- /dev/null +++ b/mcp/terraform/secretsmanager.tf @@ -0,0 +1,18 @@ +# terraform/secretsmanager.tf + +# Secrets Manager secret for the database credentials +resource "aws_secretsmanager_secret" "db_credentials" { + name = "${var.project_name}-db-credentials" + + tags = { + Name = "${var.project_name}-db-credentials" + } +} + +resource "aws_secretsmanager_secret_version" "db_credentials" { + secret_id = aws_secretsmanager_secret.db_credentials.id + secret_string = jsonencode({ + username = var.db_username + password = var.db_password + }) +} diff --git a/mcp/terraform/variables.tf b/mcp/terraform/variables.tf new file mode 100644 index 0000000..1431ab2 --- /dev/null +++ b/mcp/terraform/variables.tf @@ -0,0 +1,51 @@ +# terraform/variables.tf + +variable "aws_region" { + description = "The AWS region to deploy the resources to." + type = string + default = "ca-central-1" +} + +variable "aws_profile" { + description = "The AWS profile to deploy the resources to." + type = string + default = "main" +} + + +variable "project_name" { + description = "The name of the project." + type = string + default = "mcp-registry" +} + +variable "vpc_cidr_block" { + description = "The CIDR block for the VPC." + type = string + default = "10.0.0.0/16" +} + +variable "public_subnet_cidrs" { + description = "The CIDR blocks for the public subnets." + type = list(string) + default = ["10.0.1.0/24", "10.0.2.0/24"] +} + +variable "private_subnet_cidrs" { + description = "The CIDR blocks for the private subnets." + type = list(string) + default = ["10.0.3.0/24", "10.0.4.0/24"] +} + +variable "db_username" { + description = "The username for the database." + type = string + default = "mcpadmin" + sensitive = true +} + +variable "db_password" { + description = "The password for the database." + type = string + sensitive = true +} From fce884d60b48ffaa863c53e392d5194d823e7cd1 Mon Sep 17 00:00:00 2001 From: modupei Date: Thu, 8 Jan 2026 01:14:36 +0100 Subject: [PATCH 6/7] feat: Apply various fixes and improvements to AI agents configuration --- mcp/mcp_registry/crud.py | 8 +++----- mcp/mcp_registry/database.py | 8 +++++--- mcp/mcp_registry/main.py | 12 ++---------- mcp/mcp_registry/schemas.py | 20 ++++++++++++++++---- mcp/terraform/alb.tf | 28 +++++++++++++++++++++++++++- mcp/terraform/ecr.tf | 4 ++-- mcp/terraform/ecs.tf | 10 ++++------ mcp/terraform/main.tf | 4 ++-- 8 files changed, 61 insertions(+), 33 deletions(-) diff --git a/mcp/mcp_registry/crud.py b/mcp/mcp_registry/crud.py index 38d56d1..e3def7b 100644 --- a/mcp/mcp_registry/crud.py +++ b/mcp/mcp_registry/crud.py @@ -2,10 +2,10 @@ from . import models, schemas def get_agent(db: Session, agent_id: int): - return db.query(models.Agent).filter(models.Agent.id == agent_id).first() + return db.query(models.Agent).filter(models.Agent.id == agent_id, models.Agent.is_deleted == False).first() def get_agents(db: Session, skip: int = 0, limit: int = 100): - return db.query(models.Agent).offset(skip).limit(limit).all() + return db.query(models.Agent).filter(models.Agent.is_deleted == False).offset(skip).limit(limit).all() def create_agent(db: Session, agent: schemas.AgentCreate): db_agent = models.Agent( @@ -44,9 +44,7 @@ def get_capabilities(db: Session, skip: int = 0, limit: int = 100): def create_capability(db: Session, capability: schemas.CapabilityCreate): db_capability = models.Capability(**capability.dict()) db.add(db_capability) - db.commit() - db.refresh(db_capability) return db_capability def get_agents_by_capability(db: Session, capability_name: str): - return db.query(models.Agent).join(models.Agent.capabilities).filter(models.Capability.name == capability_name).all() + return db.query(models.Agent).join(models.Agent.capabilities).filter(models.Capability.name == capability_name, models.Agent.is_deleted == False).all() diff --git a/mcp/mcp_registry/database.py b/mcp/mcp_registry/database.py index 98c6672..6237188 100644 --- a/mcp/mcp_registry/database.py +++ b/mcp/mcp_registry/database.py @@ -10,7 +10,9 @@ load_dotenv() # Load environment variables from .env file # Database connection string from environment variable -DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://user:password@localhost/mcp_registry") +DATABASE_URL = os.getenv("DATABASE_URL") +if not DATABASE_URL: + raise ValueError("DATABASE_URL environment variable not set") engine = create_engine(DATABASE_URL) SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) @@ -19,8 +21,8 @@ agent_capability_association = Table( 'agent_capability_association', Base.metadata, - Column('agent_id', Integer, ForeignKey('agents.id')), - Column('capability_id', Integer, ForeignKey('capabilities.id')) + Column('agent_id', Integer, ForeignKey('agents.id'), primary_key=True), + Column('capability_id', Integer, ForeignKey('capabilities.id'), primary_key=True) ) class Agent(Base): diff --git a/mcp/mcp_registry/main.py b/mcp/mcp_registry/main.py index 9932c31..e4e2a46 100644 --- a/mcp/mcp_registry/main.py +++ b/mcp/mcp_registry/main.py @@ -5,20 +5,12 @@ from sqlalchemy.orm import Session from . import crud, models, schemas -from .database import SessionLocal, engine, create_db_and_tables +from .database import engine, create_db_and_tables, get_db -models.Base.metadata.create_all(bind=engine) -app = FastAPI() +app = FastAPI() -# Dependency -def get_db(): - db = SessionLocal() - try: - yield db - finally: - db.close() @app.on_event("startup") diff --git a/mcp/mcp_registry/schemas.py b/mcp/mcp_registry/schemas.py index 7691924..a16c925 100644 --- a/mcp/mcp_registry/schemas.py +++ b/mcp/mcp_registry/schemas.py @@ -1,5 +1,5 @@ from typing import List, Optional -from pydantic import BaseModel +from pydantic import BaseModel, Field class CapabilityBase(BaseModel): name: str @@ -8,9 +8,8 @@ class CapabilityBase(BaseModel): class CapabilityCreate(CapabilityBase): pass -class Capability(CapabilityBase): +class CapabilitySimple(CapabilityBase): id: int - agents: List["Agent"] = [] class Config: orm_mode = True @@ -28,12 +27,25 @@ class AgentBase(BaseModel): openapi_spec_s3_uri: Optional[str] = None openapi_spec_checksum: Optional[str] = None +class AgentSimple(AgentBase): + id: int + + class Config: + orm_mode = True + class AgentCreate(AgentBase): capabilities: List[CapabilityCreate] = [] class Agent(AgentBase): id: int - capabilities: List[Capability] = [] + capabilities: List[CapabilitySimple] = Field(default_factory=list) + + class Config: + orm_mode = True + +class Capability(CapabilityBase): + id: int + agents: List[AgentSimple] = Field(default_factory=list) class Config: orm_mode = True diff --git a/mcp/terraform/alb.tf b/mcp/terraform/alb.tf index 2eadd85..8c21c13 100644 --- a/mcp/terraform/alb.tf +++ b/mcp/terraform/alb.tf @@ -14,6 +14,13 @@ resource "aws_security_group" "app" { cidr_blocks = ["0.0.0.0/0"] } + ingress { + from_port = 443 + to_port = 443 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + # Allow all outbound traffic egress { from_port = 0 @@ -63,12 +70,31 @@ resource "aws_lb_target_group" "main" { } } -# Listener for the ALB +# Listener for the ALB - HTTP to HTTPS redirect resource "aws_lb_listener" "http" { load_balancer_arn = aws_lb.main.arn port = "80" protocol = "HTTP" + default_action { + type = "redirect" + + redirect { + port = "443" + protocol = "HTTPS" + status_code = "HTTP_301" + } + } +} + +# Listener for the ALB - HTTPS +resource "aws_lb_listener" "https" { + load_balancer_arn = aws_lb.main.arn + port = "443" + protocol = "HTTPS" + ssl_policy = "ELBSecurityPolicy-2016-08" + certificate_arn = "arn:aws:acm:us-east-1:123456789012:certificate/your-certificate-id" # TODO: Replace with your certificate ARN + default_action { type = "forward" target_group_arn = aws_lb_target_group.main.arn diff --git a/mcp/terraform/ecr.tf b/mcp/terraform/ecr.tf index 39ca711..f2ec2d4 100644 --- a/mcp/terraform/ecr.tf +++ b/mcp/terraform/ecr.tf @@ -3,8 +3,8 @@ # ECR repository for the application resource "aws_ecr_repository" "main" { name = "${var.project_name}-repo" - image_tag_mutability = "MUTABLE" - force_delete = true + image_tag_mutability = "IMMUTABLE" + force_delete = false image_scanning_configuration { scan_on_push = true diff --git a/mcp/terraform/ecs.tf b/mcp/terraform/ecs.tf index 529566d..a6df72c 100644 --- a/mcp/terraform/ecs.tf +++ b/mcp/terraform/ecs.tf @@ -30,16 +30,14 @@ resource "aws_ecs_task_definition" "main" { hostPort = 80 } ] - environment = [ - { - name = "DATABASE_URL" - value = "secretsmanager:${aws_secretsmanager_secret.db_credentials.name}" - } - ] secrets = [ { name = "DB_CREDENTIALS" valueFrom = aws_secretsmanager_secret.db_credentials.arn + }, + { + name = "DATABASE_URL" + valueFrom = aws_secretsmanager_secret.db_credentials.arn } ] } diff --git a/mcp/terraform/main.tf b/mcp/terraform/main.tf index 9c53d30..01d6903 100644 --- a/mcp/terraform/main.tf +++ b/mcp/terraform/main.tf @@ -16,7 +16,7 @@ resource "aws_subnet" "public" { count = length(var.public_subnet_cidrs) vpc_id = aws_vpc.main.id cidr_block = var.public_subnet_cidrs[count.index] - availability_zone = data.aws_availability_zones.available.names[count.index] + availability_zone = data.aws_availability_zones.available.names[min(count.index, length(data.aws_availability_zones.available.names) - 1)] map_public_ip_on_launch = true tags = { @@ -29,7 +29,7 @@ resource "aws_subnet" "private" { count = length(var.private_subnet_cidrs) vpc_id = aws_vpc.main.id cidr_block = var.private_subnet_cidrs[count.index] - availability_zone = data.aws_availability_zones.available.names[count.index] + availability_zone = data.aws_availability_zones.available.names[min(count.index, length(data.aws_availability_zones.available.names) - 1)] tags = { Name = "${var.project_name}-private-subnet-${count.index + 1}" From c12231f67fac4975916dbfb6427e45de9459b73c Mon Sep 17 00:00:00 2001 From: modupei Date: Thu, 8 Jan 2026 01:16:24 +0100 Subject: [PATCH 7/7] feat: Complete remaining fixes for AI agents configuration --- mcp/terraform/s3.tf | 12 ++++++++++++ mcp/terraform/variables.tf | 1 - 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/mcp/terraform/s3.tf b/mcp/terraform/s3.tf index 7817933..51d3bc1 100644 --- a/mcp/terraform/s3.tf +++ b/mcp/terraform/s3.tf @@ -4,6 +4,18 @@ resource "aws_s3_bucket" "main" { bucket = "${var.project_name}-openapi-specs" + versioning { + enabled = true + } + + server_side_encryption_configuration { + rule { + apply_server_side_encryption_by_default { + sse_algorithm = "AES256" + } + } + } + tags = { Name = "${var.project_name}-openapi-specs" } diff --git a/mcp/terraform/variables.tf b/mcp/terraform/variables.tf index 1431ab2..31b14c2 100644 --- a/mcp/terraform/variables.tf +++ b/mcp/terraform/variables.tf @@ -40,7 +40,6 @@ variable "private_subnet_cidrs" { variable "db_username" { description = "The username for the database." type = string - default = "mcpadmin" sensitive = true }