Make import work for BitBucket, CodeCommit and GitLab

.vscode/launch.json

@@ -199,6 +199,17 @@
             ],
             "justMyCode": true
         },
+        {
+            "name": "Iambic: Simulate Generic Git Provider",
+            "type": "python",
+            "request": "launch",
+            "program": "${workspaceFolder}/dev_tools/simulate_generic_git_in_aws_lambda/simulate_generic_git.py",
+            "args": [
+            ],
+            "console": "integratedTerminal",
+            "justMyCode": true,
+            "envFile": "${workspaceFolder}/.env",
+        },
         {
             "name": "Iambic: Simulate Lambda GitHub Webhook",
             "type": "python",

deployment/upgrade_iambic_version_for_generic_git_provider_lambda/iambic-github-app-updater-role-template.yaml

@@ -0,0 +1,53 @@
+template_type: NOQ::AWS::IAM::Role
+template_schema_url: https://docs.iambic.org/reference/schemas/aws_iam_role_template
+included_accounts:
+  - "REPLACE_THIS_WITH_YOUR_AWS_ACCOUNT_NAME_THAT_CONTAINS_IAMBIC_GENERIC_GIT_PROVIDER_LAMBDA_CODE"
+identifier: iambic_generic_git_provider_updater
+properties:
+  description: "Use to update IAMbic Generic Git Provider integration on AWS Lambda"
+  assume_role_policy_document:
+    statement:
+      - action:
+          - sts:AssumeRole
+          - sts:TagSession
+        effect: Allow
+        principal:
+          aws: "REPLACE_THIS_WITH_CI_CD_ROLE_THAT_WOULD_RUN_THE_UPDATER"
+    version: '2012-10-17'
+  inline_policies:
+    - policy_name: CloudFormation
+      statement:
+        - action: cloudformation:ListStacks
+          effect: Allow
+          resource: '*'
+          sid: ListPermissions
+        - action:
+          - cloudformation:DescribeStacks
+          - cloudformation:UpdateStack
+          effect: Allow
+          resource: arn:aws:cloudformation:*:{{var.account_id}}:stack/IAMbicGenericGitProviderLambda/*
+      version: '2012-10-17'
+    - policy_name: CodeBuild
+      statement:
+        - action:
+            - codebuild:BatchGetBuilds
+            - codebuild:StartBuild
+          effect: Allow
+          resource: arn:aws:codebuild:*:{{var.account_id}}:project/iambic_code_build
+      version: '2012-10-17'
+    - policy_name: ECR
+      statement:
+        - action: ecr:DescribeImages
+          effect: Allow
+          resource: arn:aws:ecr:*:{{var.account_id}}:repository/iambic-ecr-public/iambic/iambic
+      version: '2012-10-17'
+    - policy_name: Lambda
+      statement:
+        - action:
+            - lambda:GetFunctionUrlConfig
+            - lambda:ListTags
+            - lambda:UpdateFunctionCode
+          effect: Allow
+          resource: arn:aws:lambda:*:{{var.account_id}}:function:iambic_generic_git_provider_webhook
+      version: '2012-10-17'
+  role_name: iambic_generic_git_provider_updater

deployment/upgrade_iambic_version_for_generic_git_provider_lambda/upgrade_lambda.py

@@ -0,0 +1,151 @@
+from __future__ import annotations
+
+import logging
+import os
+import time
+
+import boto3
+
+REGION_NAME = os.environ.get("AWS_REGION", "us-east-1")
+IAMBIC_CODE_BUILD_PROJECT_NAME = os.environ.get(
+    "IAMBIC_CODE_BUILD_PROJECT_NAME", "iambic_code_build"
+)
+IAMBIC_FUNCTION_NAME = os.environ.get(
+    "IAMBIC_FUNCTION_NAME", "iambic_generic_git_provider_webhook"
+)
+IAMBIC_REPOSITORY_NAME = os.environ.get(
+    "IAMBIC_REPOSITORY_NAME", "iambic-ecr-public/iambic/iambic"
+)
+IAMBIC_CF_LAMBDA_STACK_NAME = os.environ.get(
+    "IAMBIC_CF_LAMBDA_STACK_NAME", "IAMbicGenericGitProviderLambda"
+)
+IAMBIC_TARGET_VERSION = os.environ.get("IAMBIC_TARGET_VERSION", "latest")
+
+
+def start_code_build_with_pin_version(ver):
+    code_build_client = boto3.client("codebuild", region_name=REGION_NAME)
+
+    response = code_build_client.start_build(
+        projectName=IAMBIC_CODE_BUILD_PROJECT_NAME,
+        environmentVariablesOverride=[
+            {
+                "name": "IMAGE_TAG",
+                "value": ver,
+                "type": "PLAINTEXT",
+            },
+        ],
+    )
+
+    build_id = response["build"]["id"]
+    logging.info("Preparing container image. This process should take around 2 minutes")
+    for _ in range(6):
+        resp = code_build_client.batch_get_builds(ids=[build_id])
+        build_status = resp["builds"][0]["buildStatus"]
+        if build_status == "IN_PROGRESS":
+            time.sleep(30)
+            continue
+        elif build_status == "SUCCEEDED":
+            break
+        else:
+            raise ValueError(f"build status is {build_status}")
+
+
+def is_image_label_ready(ecr_client, ver):
+    if ver == "latest":
+        raise ValueError(
+            "We do not support `latest` as image label because ECR cache maybe out of date. Please point to a specific version"
+        )
+    repository_name = IAMBIC_REPOSITORY_NAME
+    try:
+        resp = ecr_client.describe_images(
+            repositoryName=repository_name, imageIds=[{"imageTag": ver}]
+        )
+        if len(resp["imageDetails"]) == 0:
+            return False
+        else:
+            return True
+    except ecr_client.exceptions.ImageNotFoundException:
+        return False
+
+
+def wait_until_image_is_ready(ecr_client, ver):
+    print("Waiting for image label to be ready")
+    for _ in range(6):
+        if is_image_label_ready(ecr_client, ver):
+            break
+        else:
+            time.sleep(30)
+            continue
+
+
+def update_lambda_code(ver):
+    client = boto3.client("lambda", region_name=REGION_NAME)
+    response = client.get_function(
+        FunctionName=IAMBIC_FUNCTION_NAME,
+    )
+    image_uri = response["Code"]["ImageUri"]
+    base_uri, current_ver = image_uri.split(":")
+    assert base_uri
+    assert current_ver
+    new_image_uri = f"{base_uri}:{ver}"
+    print(f"new image uri: {new_image_uri}")
+    response = client.update_function_code(
+        FunctionName=IAMBIC_FUNCTION_NAME,
+        ImageUri=new_image_uri,
+        Publish=True,
+    )
+
+
+def update_cf_lambda_ver(ver):
+    client = boto3.client("cloudformation", region_name=REGION_NAME)
+    response = client.describe_stacks(
+        StackName=IAMBIC_CF_LAMBDA_STACK_NAME,
+    )
+    existing_parameters = response["Stacks"][0]["Parameters"]
+    new_parameters = []
+    image_uri = None
+    for param in existing_parameters:
+        if param["ParameterKey"] != "ImageUri":
+            new_parameters.append(
+                {"ParameterKey": param["ParameterKey"], "UsePreviousValue": True}
+            )
+        else:
+            image_uri = param["ParameterValue"]
+    assert image_uri
+    base_uri, current_ver = image_uri.split(":")
+    assert base_uri
+    assert current_ver
+    new_image_uri = f"{base_uri}:{ver}"
+    print(f"new image uri: {new_image_uri}")
+    new_parameters.append({"ParameterKey": "ImageUri", "ParameterValue": new_image_uri})
+    response = client.update_stack(
+        StackName=IAMBIC_CF_LAMBDA_STACK_NAME,
+        UsePreviousTemplate=True,
+        Parameters=new_parameters,
+    )
+    for _ in range(6):
+        response = client.describe_stacks(
+            StackName=IAMBIC_CF_LAMBDA_STACK_NAME,
+        )
+        stack_status = response["Stacks"][0]["StackStatus"]
+        if stack_status != "UPDATE_IN_PROGRESS":
+            print(f"stack status: {stack_status}")
+            break
+        else:
+            print("waiting for stack to finish updating")
+            time.sleep(60)
+
+
+def upgrade_lambda(ver):
+    ecr_client = boto3.client("ecr", region_name=REGION_NAME)
+    if not is_image_label_ready(ecr_client, ver):
+        # only trigger the pull if the version is not already in ECR
+        # this helps speed up rollback
+        start_code_build_with_pin_version(ver)
+    # the wait is required due to eventual consistency
+    wait_until_image_is_ready(ecr_client, ver)
+    update_cf_lambda_ver(ver)
+
+
+if __name__ == "__main__":
+    upgrade_lambda(IAMBIC_TARGET_VERSION)

dev_tools/simulate_generic_git_in_aws_lambda/simulate_generic_git.py

@@ -0,0 +1,58 @@
+from __future__ import annotations
+
+import os
+from functools import cache
+from unittest.mock import patch
+
+import boto3
+import yaml
+
+from iambic.plugins.v0_1_0.generic_git_provider.aws_lambda_handler import run_handler
+
+DEV_REGION = os.environ.get("DEV_REGION", "us-west-2")
+DEV_EMAIL_DOMAIN_SUFFIX = os.environ.get("DEV_EMAIL_DOMAIN_SUFFIX", "@example.com")
+DEV_ACCOUNT_ID = os.environ.get("DEV_ACCOUNT_ID", "")
+GIT_PROVIDER_UNDER_TEST = os.environ.get("GIT_PROVIDER_UNDER_TEST", "")
+
+# GIT_PROVIDER_UNDER_TEST valid ones are
+# bitbucket
+# codecommit
+# gitlab
+
+# You cannot proceed without these values. Check your environment setup.
+assert DEV_ACCOUNT_ID
+assert GIT_PROVIDER_UNDER_TEST
+
+
+@cache
+def _get_app_secrets_as_lambda_context_current() -> dict:
+    # Create a Secrets Manager client
+    session = boto3.session.Session()
+    client = session.client(service_name="secretsmanager", region_name=DEV_REGION)
+
+    try:
+        get_secret_value_response = client.get_secret_value(
+            SecretId="iambic-dev/generic-git-providers-secrets"
+        )
+    except Exception as e:
+        # For a list of exceptions thrown, see
+        # https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_GetSecretValue.html
+        raise e
+
+    # Decrypts secret using the associated KMS key.
+    return yaml.safe_load(get_secret_value_response["SecretString"])["git_providers"][
+        GIT_PROVIDER_UNDER_TEST
+    ]
+
+
+if __name__ == "__main__":
+    # to simulate lambda, we are pretending to be a lambda function
+    os.environ["AWS_LAMBDA_FUNCTION_NAME"] = "simulate_generic_git.py"
+
+    req = {"source": "EventBridgeCron", "command": "import"}
+
+    with patch(
+        "iambic.plugins.v0_1_0.generic_git_provider.aws_lambda_handler._get_app_secrets_as_lambda_context_current",
+        new=_get_app_secrets_as_lambda_context_current,
+    ):
+        run_handler(req, None)

docs/cep/000-cep-template.md

@@ -1,5 +1,8 @@
 # CEP xxx - Code Enhancement Proposal Title
 
+## Champion
+Who will help organize the effort of getting this implemented?
+
 ## Summary
 Short summary about a code enhancement proposal
 

docs/cep/004-generic-git-provider-support.md

@@ -0,0 +1,53 @@
+# CEP 004 - Generic Git Provider Support
+
+## Champion
+smoy
+
+## Summary
+Quickly add support to other Git Provider that is not GitHub
+
+## Rationale
+The GitHub integration took sometime because it uses GitHub App interaction model. Such app
+support is not universal in other Git providers. We want to maximize other Git provider support
+with minimum complexity.
+
+The most supported mechanism is git checkout repository using https. For the sake of concrete
+examples, we will attempt to make this generic git provider at least support BitBucket,
+AWS CodeCommit and GitLab. It's not limited to just these 3 providers. A Git provider
+that supports git clone via https should be sufficient.
+
+https git clone for private repository typically involves http basic auth. We
+recommend users use an repository scoped token for authentication. We strongly
+advise against using actual username, password combination. Access token is
+less prone to re-use across other services.
+
+Road Map
+1. Launch import support with generic git provider.
+2. Recruit additional help to implement Git Provider specific interactions.
+
+Git Provider specific interactions
+
+1. Each provider has different webhook event implementation details.
+1. Each provider has different REST API
+1. Each provider has different authentication + authorization model
+
+## Customer Experience
+1. User will still use `iambic setup` to install a lambda function
+1. The lambda function will be driven by AWS EventBridge to periodic
+import.
+1. During install, user will need to provide the following
+
+* username
+* token
+* clone url (must be https:// based)
+* repo full name (typically company_name/repo_name )
+* default branch name (typically main or master)
+
+## Alternative
+Is there alternative considered?
+
+## Implementation
+What's needed on the implementation?
+
+## Compatibility concern
+Is there any compatibility concern?