diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index c04f6f059..e69de29bb 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,3 +0,0 @@ -# https://docs.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners - -* @aws-observability/adot-sdk-maintainers diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml deleted file mode 100644 index 66cd13c4f..000000000 --- a/.github/workflows/canary.yml +++ /dev/null @@ -1,202 +0,0 @@ -name: Canary (published artifacts) build - -on: - schedule: - - cron: '25 */8 * * *' # every 8 hours - workflow_dispatch: - -env: - AMP_REGIONS: us-west-2,us-east-1,us-east-2,eu-central-1,eu-west-1 - -permissions: - id-token: write - contents: read - -jobs: - canary-test: - runs-on: ubuntu-22.04 - name: Canary Test - (${{ matrix.aws_region }} - ${{ matrix.language }} - ${{ matrix.sample-app }} - ${{ matrix.instrumentation-type }} - ${{ matrix.architecture }}) - strategy: - fail-fast: false - matrix: - aws_region: [ - "us-east-1", - "us-east-2", - "us-west-1", - "us-west-2", - "ap-south-1", - "ap-northeast-2", - "ap-southeast-1", - "ap-southeast-2", - "ap-northeast-1", - "ca-central-1", - "eu-central-1", - "eu-west-1", - "eu-west-2", - "eu-west-3", - "eu-north-1", - "sa-east-1" - ] - language: [ dotnet, go, java, nodejs, python ] - sample-app: [ aws-sdk, okhttp ] - instrumentation-type: [ agent, wrapper ] - architecture: [ amd64, arm64 ] - exclude: - - language: dotnet - sample-app: okhttp - - language: dotnet - instrumentation-type: agent - - language: go - sample-app: okhttp - - language: go - instrumentation-type: agent - - language: nodejs - sample-app: okhttp - - language: nodejs - instrumentation-type: agent - - language: python - sample-app: okhttp - - language: python - instrumentation-type: agent - - language: java - sample-app: okhttp - instrumentation-type: agent - steps: - - uses: actions/checkout@v4 - with: - submodules: recursive - - uses: actions/setup-go@v5 - with: - go-version: '~1.24.3' - check-latest: true - - uses: actions/setup-java@v4 - if: ${{ matrix.language == 'java' }} - with: - distribution: corretto - java-version: '17' - - name: Cache (Java) - uses: actions/cache@v4 - if: ${{ matrix.language == 'java' }} - with: - path: | - ~/go/pkg/mod - ~/.gradle/caches - ~/.gradle/wrapper - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-gradle- - - uses: actions/setup-node@v4 - if: ${{ matrix.language == 'nodejs' }} - with: - node-version: '18' - - name: Cache (NodeJS) - uses: actions/cache@v4 - if: ${{ matrix.language == 'nodejs' }} - with: - path: | - ~/go/pkg/mod - ~/.npm - key: ${{ runner.os }}-node-${{ hashFiles('**/package.json') }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-node- - - uses: actions/setup-python@v5 - if: ${{ matrix.language == 'python' }} - with: - python-version: '3.x' - - name: Cache (Python) - uses: actions/cache@v4 - if: ${{ matrix.language == 'python' }} - with: - path: | - ~/go/pkg/mod - ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-pip- - - uses: actions/setup-dotnet@v4 - if: ${{ matrix.language == 'dotnet' }} - with: - dotnet-version: '6.0.405' - - name: Patch ADOT - run: ./patch-upstream.sh - - name: Build functions - run: GOARCH=${{ matrix.architecture }} ./build.sh ${{ matrix.architecture }} - working-directory: ${{ matrix.language }} - - uses: aws-actions/configure-aws-credentials@v4.0.2 - with: - role-to-assume: ${{ secrets.INTEG_TEST_LAMBDA_ROLE_ARN }} - role-duration-seconds: 7200 - aws-region: ${{ matrix.aws_region }} - - name: Get terraform directory - run: | - echo TERRAFORM_DIRECTORY=${{ matrix.language }}/sample-apps/${{ matrix.sample-app }}/deploy/${{ matrix.instrumentation-type }} | - tee --append $GITHUB_ENV - - uses: hashicorp/setup-terraform@v2 - - name: Initialize terraform - run: terraform init - working-directory: ${{ env.TERRAFORM_DIRECTORY }} - - name: Get Lambda Layer `amd64` architecture value - if: ${{ matrix.architecture == 'amd64' }} - run: echo LAMBDA_FUNCTION_ARCH=x86_64 | tee --append $GITHUB_ENV - - name: Get Lambda Layer `arm64` architecture value - if: ${{ matrix.architecture == 'arm64' }} - run: echo LAMBDA_FUNCTION_ARCH=arm64 | tee --append $GITHUB_ENV - - name: Apply terraform - run: terraform apply -auto-approve - working-directory: ${{ env.TERRAFORM_DIRECTORY }} - env: - TF_VAR_function_name: lambda-${{ matrix.language }}-${{ matrix.sample-app }}-${{ matrix.instrumentation-type }}-${{ matrix.architecture }}-${{ github.run_id }}-${{ matrix.aws_region }} - TF_VAR_architecture: ${{ env.LAMBDA_FUNCTION_ARCH }} - - name: Extract endpoint - id: extract-endpoint - run: terraform output -raw api-gateway-url - working-directory: ${{ env.TERRAFORM_DIRECTORY }} - - name: Extract AMP endpoint - if: ${{ matrix.language == 'java' && matrix.sample-app == 'aws-sdk' && matrix.instrumentation-type == 'agent' && contains(env.AMP_REGIONS, matrix.aws_region) }} - id: extract-amp-endpoint - run: terraform output -raw amp_endpoint - working-directory: ${{ env.TERRAFORM_DIRECTORY }} - - name: Send request to endpoint - run: curl -sS ${{ steps.extract-endpoint.outputs.stdout }} - - name: Checkout test framework - uses: actions/checkout@v4 - with: - repository: aws-observability/aws-otel-test-framework - path: test-framework - - name: validate trace sample - uses: nick-invision/retry@v3 - with: - timeout_seconds: 300 - max_attempts: 3 - command: | - cp adot/utils/expected-templates/${{ matrix.language }}-${{ matrix.sample-app }}-${{ matrix.instrumentation-type }}.json \ - test-framework/validator/src/main/resources/expected-data-template/lambdaExpectedTrace.mustache - cd test-framework - ./gradlew :validator:run --args="-c default-lambda-validation.yml --endpoint ${{ steps.extract-endpoint.outputs.stdout }} --region $AWS_REGION" - # add back in once the java-agent layer is published and the ARNs are available - # - name: validate java agent metric sample - # if: ${{ matrix.language == 'java' && matrix.sample-app == 'aws-sdk' && matrix.instrumentation-type == 'agent' && contains(env.AMP_REGIONS, matrix.aws_region) }} - # run: | - # cp adot/utils/expected-templates/${{ matrix.language }}-${{ matrix.sample-app }}-${{ matrix.instrumentation-type }}-metric.json \ - # test-framework/validator/src/main/resources/expected-data-template/ampExpectedMetric.mustache - # cd test-framework - # ./gradlew :validator:run --args="-c prometheus-static-metric-validation.yml --cortex-instance-endpoint ${{ steps.extract-amp-endpoint.outputs.stdout }} --region $AWS_REGION" - - name: Destroy terraform - if: always() - run: terraform destroy -auto-approve - working-directory: ${{ env.TERRAFORM_DIRECTORY }} - env: - TF_VAR_architecture: ${{ env.LAMBDA_FUNCTION_ARCH }} - publish-canary-status: - needs: [canary-test] - if: ${{ always() }} - uses: ./.github/workflows/publish-status.yml - with: - namespace: 'ADOT/GitHubActions' - repository: ${{ github.repository }} - branch: ${{ github.ref_name }} - workflow: canary - success: ${{ needs.canary-test.result == 'success' }} - region: us-west-2 - secrets: - roleArn: ${{ secrets.METRICS_ROLE_ARN }} diff --git a/.github/workflows/docker-build-lambda-soak.yml b/.github/workflows/docker-build-lambda-soak.yml deleted file mode 100644 index ed59bd587..000000000 --- a/.github/workflows/docker-build-lambda-soak.yml +++ /dev/null @@ -1,44 +0,0 @@ -on: - push: - branches: - - main - paths: - - adot/utils/soak/** - - .github/workflows/docker-build-lambda-soak.yml - -permissions: - id-token: write - -jobs: - build-lambda-soak: - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v4 - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4.0.2 - with: - role-to-assume: ${{ secrets.INTEG_TEST_LAMBDA_ROLE_ARN }} - role-duration-seconds: 1200 - aws-region: us-east-1 - - name: Login to ECR - uses: docker/login-action@v3 - with: - registry: public.ecr.aws - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Cache Docker layers - uses: actions/cache@v4 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - - name: Build docker image - uses: docker/build-push-action@v6 - with: - push: true - context: adot/utils/soak - tags: | - public.ecr.aws/aws-otel-test/lambda-soak:latest - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache diff --git a/.github/workflows/main-build-java.yml b/.github/workflows/main-build-java.yml deleted file mode 100644 index 126b762cb..000000000 --- a/.github/workflows/main-build-java.yml +++ /dev/null @@ -1,189 +0,0 @@ -name: Java Layer Integration Test -on: - workflow_call: - inputs: - caller-workflow-name: - required: true - type: string - workflow_dispatch: - -concurrency: - group: main-build-java-${{ github.ref_name }} - cancel-in-progress: true - -permissions: - id-token: write - contents: read - -jobs: - integration-test: - runs-on: ubuntu-22.04 - name: java-${{ matrix.sample-app }}-${{ matrix.instrumentation-type }}-${{ matrix.architecture }}-${{matrix.confmap}} - strategy: - fail-fast: false - matrix: - architecture: [ amd64, arm64 ] - runtime: [java17, java11] - sample-app: [ aws-sdk, okhttp ] - instrumentation-type: [ wrapper ] - confmap: [ "noop" ] - include: - - sample-app: aws-sdk - instrumentation-type: agent - architecture: amd64 - confmap: noop - runtime: java17 - - sample-app: aws-sdk - instrumentation-type: agent - architecture: amd64 - confmap: noop - runtime: java11 - - sample-app: aws-sdk - instrumentation-type: agent - architecture: arm64 - confmap: noop - runtime: java17 - - sample-app: aws-sdk - instrumentation-type: agent - architecture: arm64 - confmap: noop - runtime: java11 - - sample-app: aws-sdk - instrumentation-type: agent-confmap - architecture: amd64 - confmap: s3 - runtime: java11 - - sample-app: aws-sdk - instrumentation-type: agent-confmap - architecture: arm64 - confmap: s3 - runtime: java11 - - sample-app: aws-sdk - instrumentation-type: agent-confmap - architecture: amd64 - confmap: http - runtime: java11 - - sample-app: aws-sdk - instrumentation-type: agent-confmap - architecture: arm64 - confmap: http - runtime: java11 - - sample-app: aws-sdk - instrumentation-type: agent-confmap - architecture: amd64 - confmap: https - runtime: java11 - - sample-app: aws-sdk - instrumentation-type: agent-confmap - architecture: arm64 - confmap: https - runtime: java11 - - steps: - - uses: actions/checkout@v4 - with: - submodules: recursive - - uses: actions/setup-go@v5 - with: - go-version: '~1.24.3' - check-latest: true - - uses: actions/setup-java@v4 - with: - distribution: corretto - java-version: '17' - - name: Cache (Java) - uses: actions/cache@v4 - with: - path: | - ~/go/pkg/mod - ~/.gradle/caches - ~/.gradle/wrapper - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-gradle- - - - uses: aws-actions/configure-aws-credentials@v4.0.2 - with: - role-to-assume: ${{ secrets.INTEG_TEST_LAMBDA_ROLE_ARN }} - role-duration-seconds: 7200 - aws-region: us-east-1 - - name: Generate UUID - id: generate-uuid - run: | - UUID=$(uuidgen | cut -c1-4) - echo "uuid=$UUID" >> $GITHUB_OUTPUT - - name: Patch ADOT - run: ./patch-upstream.sh - - name: Build layers / functions - run: GOARCH=${{ matrix.architecture }} ./build.sh ${{ matrix.architecture }} - working-directory: java - - name: Get Lambda Layer amd64 architecture value - if: ${{ matrix.architecture == 'amd64' }} - run: echo LAMBDA_FUNCTION_ARCH=x86_64 | tee --append $GITHUB_ENV - - name: Get Lambda Layer arm64 architecture value - if: ${{ matrix.architecture == 'arm64' }} - run: echo LAMBDA_FUNCTION_ARCH=arm64 | tee --append $GITHUB_ENV - - name: Get terraform directory - run: | - echo TERRAFORM_DIRECTORY=java/integration-tests/${{ matrix.sample-app }}/${{ matrix.instrumentation-type }} | - tee --append $GITHUB_ENV - - uses: hashicorp/setup-terraform@v2 - - name: Initialize terraform - run: terraform init - working-directory: ${{ env.TERRAFORM_DIRECTORY }} - - name: Get terraform Lambda function name - run: | - echo TERRAFORM_LAMBDA_FUNCTION_NAME=lambda-java-${{ matrix.sample-app }}-${{ matrix.instrumentation-type }}-${{ matrix.confmap }}-${{ matrix.architecture }}-${{ steps.generate-uuid.outputs.uuid }} | - tee --append $GITHUB_ENV - - name: Apply terraform - run: terraform apply -auto-approve - working-directory: ${{ env.TERRAFORM_DIRECTORY }} - env: - TF_VAR_sdk_layer_name: opentelemetry-java-${{ matrix.sample-app }}-${{ matrix.instrumentation-type }}-${{ matrix.architecture }}-${{ github.run_id }} - TF_VAR_function_name: ${{ env.TERRAFORM_LAMBDA_FUNCTION_NAME }} - TF_VAR_architecture: ${{ env.LAMBDA_FUNCTION_ARCH }} - TF_VAR_configuration_source: ${{ matrix.confmap }} - TF_VAR_runtime: ${{ matrix.runtime }} - - name: Extract endpoint - id: extract-endpoint - run: terraform output -raw api-gateway-url - working-directory: ${{ env.TERRAFORM_DIRECTORY }} - - name: Extract AMP endpoint - id: extract-amp-endpoint - if: ${{ matrix.sample-app == 'aws-sdk' && startsWith(matrix.instrumentation-type, 'agent') }} - run: terraform output -raw amp_endpoint - working-directory: ${{ env.TERRAFORM_DIRECTORY }} - - name: Extract SDK layer arn - id: extract-sdk-layer-arn - run: terraform output -raw sdk_layer_arn - working-directory: ${{ env.TERRAFORM_DIRECTORY }} - - name: Output annotations - run: | - echo "::warning::Function: ${{ env.TERRAFORM_LAMBDA_FUNCTION_NAME }}" - echo "::warning::SDK Layer ARN: ${{ steps.extract-sdk-layer-arn.outputs.stdout }}" - - name: Send request to endpoint - run: curl -sS ${{ steps.extract-endpoint.outputs.stdout }} - - name: Checkout test framework - uses: actions/checkout@v4 - with: - repository: aws-observability/aws-otel-test-framework - path: test-framework - - name: validate trace sample - run: | - cp adot/utils/expected-templates/java-${{ matrix.sample-app }}-${{ matrix.instrumentation-type }}.json \ - test-framework/validator/src/main/resources/expected-data-template/lambdaExpectedTrace.mustache - cd test-framework - ./gradlew :validator:run --args="-c default-lambda-validation.yml --endpoint ${{ steps.extract-endpoint.outputs.stdout }} --region $AWS_REGION" - - name: validate java agent metric sample - if: ${{ matrix.sample-app == 'aws-sdk' && startsWith(matrix.instrumentation-type, 'agent') }} - run: | - cp adot/utils/expected-templates/java-${{ matrix.sample-app }}-${{ matrix.instrumentation-type }}-metric.json \ - test-framework/validator/src/main/resources/expected-data-template/ampExpectedMetric.mustache - cd test-framework - ./gradlew :validator:run --args="-c prometheus-static-metric-validation.yml --cortex-instance-endpoint ${{ steps.extract-amp-endpoint.outputs.stdout }} --region $AWS_REGION" - - name: Destroy terraform - if: always() - run: terraform destroy -auto-approve - working-directory: ${{ env.TERRAFORM_DIRECTORY }} - env: - TF_VAR_architecture: ${{ env.LAMBDA_FUNCTION_ARCH }} diff --git a/.github/workflows/main-build-nodejs.yml b/.github/workflows/main-build-nodejs.yml deleted file mode 100644 index d99e6534d..000000000 --- a/.github/workflows/main-build-nodejs.yml +++ /dev/null @@ -1,122 +0,0 @@ -name: NodeJS Layer Integration Test -on: - workflow_call: - inputs: - caller-workflow-name: - required: true - type: string - workflow_dispatch: - -concurrency: - group: main-build-nodejs-${{ github.ref_name }} - cancel-in-progress: true - -permissions: - id-token: write - contents: read - -jobs: - integration-test: - runs-on: ubuntu-22.04 - name: nodejs-wrapper-${{ matrix.architecture }} - strategy: - fail-fast: false - matrix: - architecture: [ amd64, arm64 ] - runtime: [nodejs18.x, nodejs20.x, nodejs22.x] - steps: - - uses: actions/checkout@v4 - with: - submodules: recursive - - uses: actions/setup-go@v5 - with: - go-version: '~1.24.3' - check-latest: true - - uses: actions/setup-node@v4 - with: - node-version: '18' - - name: Cache (NodeJS) - uses: actions/cache@v4 - with: - path: | - ~/go/pkg/mod - ~/.npm - key: ${{ runner.os }}-node-${{ hashFiles('**/package.json') }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-node- - - uses: aws-actions/configure-aws-credentials@v4.0.2 - with: - role-to-assume: ${{ secrets.INTEG_TEST_LAMBDA_ROLE_ARN }} - role-duration-seconds: 7200 - aws-region: us-east-1 - - name: Generate UUID - id: generate-uuid - run: | - UUID=$(uuidgen | cut -c1-4) - echo "uuid=$UUID" >> $GITHUB_OUTPUT - - name: Patch ADOT - run: ./patch-upstream.sh - - name: Login to Public ECR - uses: docker/login-action@v3 - with: - registry: public.ecr.aws - - name: Build layers / functions - run: GOARCH=${{ matrix.architecture }} ./build.sh ${{ matrix.architecture }} - working-directory: nodejs - - name: Get Lambda Layer amd64 architecture value - if: ${{ matrix.architecture == 'amd64' }} - run: echo LAMBDA_FUNCTION_ARCH=x86_64 | tee --append $GITHUB_ENV - - name: Get Lambda Layer arm64 architecture value - if: ${{ matrix.architecture == 'arm64' }} - run: echo LAMBDA_FUNCTION_ARCH=arm64 | tee --append $GITHUB_ENV - - name: Get terraform directory - run: | - echo TERRAFORM_DIRECTORY=nodejs/integration-tests/aws-sdk/wrapper | - tee --append $GITHUB_ENV - - uses: hashicorp/setup-terraform@v2 - - name: Initialize terraform - run: terraform init - working-directory: ${{ env.TERRAFORM_DIRECTORY }} - - name: Get terraform Lambda function name - run: | - echo TERRAFORM_LAMBDA_FUNCTION_NAME=lambda-nodejs-${{ matrix.architecture }}-${{ steps.generate-uuid.outputs.uuid }} | - tee --append $GITHUB_ENV - - name: Apply terraform - run: terraform apply -auto-approve - working-directory: ${{ env.TERRAFORM_DIRECTORY }} - env: - TF_VAR_sdk_layer_name: opentelemetry-nodejs-aws-sdk-wrapper-${{ matrix.architecture }} - TF_VAR_function_name: ${{ env.TERRAFORM_LAMBDA_FUNCTION_NAME }} - TF_VAR_architecture: ${{ env.LAMBDA_FUNCTION_ARCH }} - TF_VAR_runtime: ${{ matrix.runtime }} - - name: Extract endpoint - id: extract-endpoint - run: terraform output -raw api-gateway-url - working-directory: ${{ env.TERRAFORM_DIRECTORY }} - - name: Extract SDK layer arn - id: extract-sdk-layer-arn - run: terraform output -raw sdk_layer_arn - working-directory: ${{ env.TERRAFORM_DIRECTORY }} - - name: Output annotations - run: | - echo "::warning::Function: ${{ env.TERRAFORM_LAMBDA_FUNCTION_NAME }}" - echo "::warning::SDK Layer ARN: ${{ steps.extract-sdk-layer-arn.outputs.stdout }}" - - name: Send request to endpoint - run: curl -sS ${{ steps.extract-endpoint.outputs.stdout }} - - name: Checkout test framework - uses: actions/checkout@v4 - with: - repository: aws-observability/aws-otel-test-framework - path: test-framework - - name: validate trace sample - run: | - cp adot/utils/expected-templates/nodejs-aws-sdk-wrapper.json \ - test-framework/validator/src/main/resources/expected-data-template/lambdaExpectedTrace.mustache - cd test-framework - ./gradlew :validator:run --args="-c default-lambda-validation.yml --endpoint ${{ steps.extract-endpoint.outputs.stdout }} --region $AWS_REGION" - - name: Destroy terraform - if: always() - run: terraform destroy -auto-approve - working-directory: ${{ env.TERRAFORM_DIRECTORY }} - env: - TF_VAR_architecture: ${{ env.LAMBDA_FUNCTION_ARCH }} diff --git a/.github/workflows/main-build-python.yml b/.github/workflows/main-build-python.yml deleted file mode 100644 index b326316ee..000000000 --- a/.github/workflows/main-build-python.yml +++ /dev/null @@ -1,122 +0,0 @@ -name: Python Layer Integration Test -on: - workflow_call: - inputs: - caller-workflow-name: - required: true - type: string - workflow_dispatch: - -concurrency: - group: main-build-python-${{ github.ref_name }} - cancel-in-progress: true - -permissions: - id-token: write - contents: read - -jobs: - integration-test: - runs-on: ubuntu-22.04 - name: python-wrapper-${{ matrix.architecture }} - strategy: - fail-fast: false - matrix: - architecture: [ amd64, arm64 ] - runtime: [python3.9, python3.10, python3.11, python3.12, python3.13] - steps: - - uses: actions/checkout@v4 - with: - submodules: recursive - - uses: actions/setup-go@v5 - with: - go-version: '~1.24.3' - check-latest: true - - uses: actions/setup-python@v5 - with: - python-version: '3.x' - - name: Cache (Python) - uses: actions/cache@v4 - with: - path: | - ~/go/pkg/mod - ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-pip- - - uses: aws-actions/configure-aws-credentials@v4.0.2 - with: - role-to-assume: ${{ secrets.INTEG_TEST_LAMBDA_ROLE_ARN }} - role-duration-seconds: 7200 - aws-region: us-east-1 - - name: Generate UUID - id: generate-uuid - run: | - UUID=$(uuidgen | cut -c1-4) - echo "uuid=$UUID" >> $GITHUB_OUTPUT - - name: Patch ADOT - run: ./patch-upstream.sh - - name: Login to Public ECR - uses: docker/login-action@v3 - with: - registry: public.ecr.aws - - name: Build layers / functions - run: GOARCH=${{ matrix.architecture }} ./build.sh ${{ matrix.architecture }} - working-directory: python - - name: Get Lambda Layer amd64 architecture value - if: ${{ matrix.architecture == 'amd64' }} - run: echo LAMBDA_FUNCTION_ARCH=x86_64 | tee --append $GITHUB_ENV - - name: Get Lambda Layer arm64 architecture value - if: ${{ matrix.architecture == 'arm64' }} - run: echo LAMBDA_FUNCTION_ARCH=arm64 | tee --append $GITHUB_ENV - - name: Get terraform directory - run: | - echo TERRAFORM_DIRECTORY=python/integration-tests/aws-sdk/wrapper | - tee --append $GITHUB_ENV - - uses: hashicorp/setup-terraform@v2 - - name: Initialize terraform - run: terraform init - working-directory: ${{ env.TERRAFORM_DIRECTORY }} - - name: Get terraform Lambda function name - run: | - echo TERRAFORM_LAMBDA_FUNCTION_NAME=lambda-python-${{ matrix.architecture }}-${{ steps.generate-uuid.outputs.uuid }} | - tee --append $GITHUB_ENV - - name: Apply terraform - run: terraform apply -auto-approve - working-directory: ${{ env.TERRAFORM_DIRECTORY }} - env: - TF_VAR_sdk_layer_name: opentelemetry-python-aws-sdk-wrapper-${{ matrix.architecture }} - TF_VAR_function_name: ${{ env.TERRAFORM_LAMBDA_FUNCTION_NAME }} - TF_VAR_architecture: ${{ env.LAMBDA_FUNCTION_ARCH }} - TF_VAR_runtime: ${{ matrix.runtime }} - - name: Extract endpoint - id: extract-endpoint - run: terraform output -raw api-gateway-url - working-directory: ${{ env.TERRAFORM_DIRECTORY }} - - name: Extract SDK layer arn - id: extract-sdk-layer-arn - run: terraform output -raw sdk_layer_arn - working-directory: ${{ env.TERRAFORM_DIRECTORY }} - - name: Output annotations - run: | - echo "::warning::Function: ${{ env.TERRAFORM_LAMBDA_FUNCTION_NAME }}" - echo "::warning::SDK Layer ARN: ${{ steps.extract-sdk-layer-arn.outputs.stdout }}" - - name: Send request to endpoint - run: curl -sS ${{ steps.extract-endpoint.outputs.stdout }} - - name: Checkout test framework - uses: actions/checkout@v4 - with: - repository: aws-observability/aws-otel-test-framework - path: test-framework - - name: validate trace sample - run: | - cp adot/utils/expected-templates/python-aws-sdk-wrapper.json \ - test-framework/validator/src/main/resources/expected-data-template/lambdaExpectedTrace.mustache - cd test-framework - ./gradlew :validator:run --args="-c default-lambda-validation.yml --endpoint ${{ steps.extract-endpoint.outputs.stdout }} --region $AWS_REGION" - - name: Destroy terraform - if: always() - run: terraform destroy -auto-approve - working-directory: ${{ env.TERRAFORM_DIRECTORY }} - env: - TF_VAR_architecture: ${{ env.LAMBDA_FUNCTION_ARCH }} diff --git a/.github/workflows/main-build.yml b/.github/workflows/main-build.yml deleted file mode 100644 index 7060250d1..000000000 --- a/.github/workflows/main-build.yml +++ /dev/null @@ -1,157 +0,0 @@ -name: Main Build - -on: - push: - branches: - - main - - release/* - paths-ignore: - - '.github/**' - - '!.github/workflows/main-build.yml' - - '!.github/workflows/main-build-python.yml' - - '!.github/workflows/main-build-java.yml' - - '!.github/workflows/main-build-nodejs.yml' - - '**.md' - workflow_dispatch: - -concurrency: - group: main-build-${{ github.ref_name }} - cancel-in-progress: true - -permissions: - id-token: write - contents: read - -jobs: - integration-test: - runs-on: ubuntu-22.04 - name: ${{ matrix.language }}-${{ matrix.instrumentation-type }}-${{ matrix.architecture }} - strategy: - fail-fast: false - matrix: - architecture: [ amd64, arm64 ] - language: [ dotnet, go ] - sample-app: [ aws-sdk ] - instrumentation-type: [ wrapper ] - - steps: - - uses: actions/checkout@v4 - with: - submodules: recursive - - uses: actions/setup-go@v5 - with: - go-version: '~1.24.3' - check-latest: true - - uses: actions/setup-dotnet@v4 - if: ${{ matrix.language == 'dotnet' }} - with: - dotnet-version: '6.0.405' - - uses: aws-actions/configure-aws-credentials@v4.0.2 - with: - role-to-assume: ${{ secrets.INTEG_TEST_LAMBDA_ROLE_ARN }} - role-duration-seconds: 7200 - aws-region: us-east-1 - - name: Patch ADOT - run: ./patch-upstream.sh - - name: Build layers / functions - run: GOARCH=${{ matrix.architecture }} ./build.sh ${{ matrix.architecture }} - working-directory: ${{ matrix.language }} - - name: Get Lambda Layer amd64 architecture value - if: ${{ matrix.architecture == 'amd64' }} - run: echo LAMBDA_FUNCTION_ARCH=x86_64 | tee --append $GITHUB_ENV - - name: Get Lambda Layer arm64 architecture value - if: ${{ matrix.architecture == 'arm64' }} - run: echo LAMBDA_FUNCTION_ARCH=arm64 | tee --append $GITHUB_ENV - - name: Get terraform directory - run: | - echo TERRAFORM_DIRECTORY=${{ matrix.language }}/integration-tests/${{ matrix.sample-app }}/${{ matrix.instrumentation-type }} | - tee --append $GITHUB_ENV - - uses: hashicorp/setup-terraform@v2 - - name: Initialize terraform - run: terraform init - working-directory: ${{ env.TERRAFORM_DIRECTORY }} - - name: Get terraform Lambda function name - run: | - echo TERRAFORM_LAMBDA_FUNCTION_NAME=lambda-${{ matrix.language }}-${{ matrix.sample-app }}-${{ matrix.instrumentation-type }}-${{ matrix.architecture }}-${{ github.run_id }} | - tee --append $GITHUB_ENV - - name: Apply terraform - run: terraform apply -auto-approve - working-directory: ${{ env.TERRAFORM_DIRECTORY }} - env: - TF_VAR_sdk_layer_name: opentelemetry-${{ matrix.language }}-${{ matrix.sample-app }}-${{ matrix.instrumentation-type }}-${{ matrix.architecture }}-${{ github.run_id }} - TF_VAR_function_name: ${{ env.TERRAFORM_LAMBDA_FUNCTION_NAME }} - TF_VAR_architecture: ${{ env.LAMBDA_FUNCTION_ARCH }} - - name: Extract endpoint - id: extract-endpoint - run: terraform output -raw api-gateway-url - working-directory: ${{ env.TERRAFORM_DIRECTORY }} - - name: Extract SDK layer arn - id: extract-sdk-layer-arn - if: ${{ matrix.language != 'dotnet' && matrix.language != 'go' }} - run: terraform output -raw sdk_layer_arn - working-directory: ${{ env.TERRAFORM_DIRECTORY }} - - name: Output annotations - if: ${{ matrix.language != 'dotnet' && matrix.language != 'go' }} - run: | - echo "::warning::Function: ${{ env.TERRAFORM_LAMBDA_FUNCTION_NAME }}" - echo "::warning::SDK Layer ARN: ${{ steps.extract-sdk-layer-arn.outputs.stdout }}" - - name: Send request to endpoint - run: curl -sS ${{ steps.extract-endpoint.outputs.stdout }} - - name: Checkout test framework - uses: actions/checkout@v4 - with: - repository: aws-observability/aws-otel-test-framework - path: test-framework - - name: validate trace sample - run: | - cp adot/utils/expected-templates/${{ matrix.language }}-${{ matrix.sample-app }}-${{ matrix.instrumentation-type }}.json \ - test-framework/validator/src/main/resources/expected-data-template/lambdaExpectedTrace.mustache - cd test-framework - ./gradlew :validator:run --args="-c default-lambda-validation.yml --endpoint ${{ steps.extract-endpoint.outputs.stdout }} --region $AWS_REGION" - - name: Destroy terraform - if: always() - run: terraform destroy -auto-approve - working-directory: ${{ env.TERRAFORM_DIRECTORY }} - env: - TF_VAR_architecture: ${{ env.LAMBDA_FUNCTION_ARCH }} - - python-runtime-test: - concurrency: - group: python-runtime-test - cancel-in-progress: false - uses: ./.github/workflows/main-build-python.yml - secrets: inherit - with: - caller-workflow-name: 'main-build' - - java-runtime-test: - concurrency: - group: java-runtime-test - cancel-in-progress: false - uses: ./.github/workflows/main-build-java.yml - secrets: inherit - with: - caller-workflow-name: 'main-build' - - nodejs-runtime-test: - concurrency: - group: nodejs-runtime-test - cancel-in-progress: false - uses: ./.github/workflows/main-build-nodejs.yml - secrets: inherit - with: - caller-workflow-name: 'main-build' - - publish-build-status: - needs: [integration-test, python-runtime-test, java-runtime-test, nodejs-runtime-test] - if: ${{ always() }} - uses: ./.github/workflows/publish-status.yml - with: - namespace: 'ADOT/GitHubActions' - repository: ${{ github.repository }} - branch: ${{ github.ref_name }} - workflow: main-build - success: ${{ needs.integration-test.result == 'success' && needs.python-runtime-test.result == 'success' && needs.java-runtime-test.result == 'success' && needs.nodejs-runtime-test.result == 'success' }} - region: us-west-2 - secrets: - roleArn: ${{ secrets.METRICS_ROLE_ARN }} diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml deleted file mode 100644 index 66fc404bf..000000000 --- a/.github/workflows/pr-build.yml +++ /dev/null @@ -1,86 +0,0 @@ -name: PR Build - -on: - pull_request: - branches: [ main ] - - -permissions: - id-token: write - contents: read - -jobs: - build-artifacts: - runs-on: ubuntu-22.04 - name: Build artifacts - ${{ matrix.language }} - strategy: - matrix: - language: [ java, nodejs, python, dotnet, go ] - build_command: [ ./build.sh ] - architecture: [ amd64, arm64 ] - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - uses: actions/setup-go@v5 - with: - go-version: '~1.24.3' - check-latest: true - - uses: actions/setup-java@v4 - if: ${{ matrix.language == 'java' }} - with: - distribution: corretto - java-version: '17' - - name: Cache (Java) - uses: actions/cache@v4 - if: ${{ matrix.language == 'java' }} - with: - path: | - ~/go/pkg/mod - ~/.gradle/caches - ~/.gradle/wrapper - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-gradle- - - uses: actions/setup-node@v4 - if: ${{ matrix.language == 'nodejs' }} - with: - node-version: '16' - - name: Cache (NodeJS) - uses: actions/cache@v4 - if: ${{ matrix.language == 'nodejs' }} - with: - path: | - ~/go/pkg/mod - ~/.npm - key: ${{ runner.os }}-node-${{ hashFiles('**/package.json') }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-node- - - uses: actions/setup-python@v5 - if: ${{ matrix.language == 'python' }} - with: - python-version: '3.x' - - name: Cache (Python) - uses: actions/cache@v4 - if: ${{ matrix.language == 'python' }} - with: - path: | - ~/go/pkg/mod - ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-pip- - - uses: actions/setup-dotnet@v4 - if: ${{ matrix.language == 'dotnet' }} - with: - dotnet-version: '6.0.405' - - name: Patch ADOT - run: ./patch-upstream.sh - - name: Build layers / sample functions - if: ${{ matrix.language != 'dotnet' && matrix.language != 'go' }} - run: GOARCH=${{ matrix.architecture }} ${{ matrix.build_command }} ${{ matrix.architecture }} - working-directory: ${{ matrix.language }} - - name: Build collector - if: ${{ matrix.language == 'dotnet' || matrix.language == 'go' }} - run: GOARCH=${{ matrix.architecture }} make package - working-directory: opentelemetry-lambda/collector diff --git a/.github/workflows/publish-collector-layer.yml b/.github/workflows/publish-collector-layer.yml new file mode 100644 index 000000000..e868698cb --- /dev/null +++ b/.github/workflows/publish-collector-layer.yml @@ -0,0 +1,250 @@ +name: Publish Collector Layer + +# Publishes the ADOT collector Lambda layer (with decouple processor) to AWS +# and makes it publicly accessible. +# +# Triggered automatically on v* tags, or manually via workflow_dispatch. +# +# Required GitHub secrets: +# AWS_ACCESS_KEY_ID - IAM user access key +# AWS_SECRET_ACCESS_KEY - IAM user secret key +# +# Required IAM permissions: +# lambda:PublishLayerVersion +# lambda:AddLayerVersionPermission +# lambda:ListLayerVersions + +on: + push: + tags: + - 'v*' + workflow_dispatch: + inputs: + version: + description: 'Layer version label (e.g. 0-138-0). Derived from tag automatically when triggered by a push.' + required: false + default: '0-138-0' + aws_regions: + description: 'JSON array of AWS regions to publish to' + required: false + default: '["us-east-1", "us-east-2", "us-west-2", "eu-west-1", "eu-central-1", "ap-southeast-1", "ap-northeast-1"]' + architecture: + description: 'JSON array of architectures to build' + required: false + default: '["amd64", "arm64"]' + +permissions: + id-token: write + contents: write + +env: + AWS_DEFAULT_REGION: us-east-1 + +jobs: + # Resolves inputs from either tag push or workflow_dispatch so all downstream + # jobs have a single consistent source of truth. + setup: + name: Resolve inputs + runs-on: ubuntu-22.04 + outputs: + version: ${{ steps.version.outputs.value }} + architecture: ${{ steps.arch.outputs.value }} + aws_regions: ${{ steps.regions.outputs.value }} + steps: + - name: Resolve version + id: version + run: | + if [ "${{ github.event_name }}" = "push" ]; then + # Convert tag v0.138.0 -> 0-138-0 + echo "value=$(echo '${{ github.ref_name }}' | sed 's/^v//' | tr '.' '-')" >> $GITHUB_OUTPUT + else + echo "value=${{ inputs.version || '0-138-0' }}" >> $GITHUB_OUTPUT + fi + + - name: Resolve architecture + id: arch + env: + VALUE: ${{ inputs.architecture || '["amd64", "arm64"]' }} + run: echo "value=$VALUE" >> $GITHUB_OUTPUT + + - name: Resolve regions + id: regions + env: + VALUE: ${{ inputs.aws_regions || '["us-east-1", "us-east-2", "us-west-2", "eu-west-1", "eu-central-1", "ap-southeast-1", "ap-northeast-1"]' }} + run: echo "value=$VALUE" >> $GITHUB_OUTPUT + + build: + name: Build - ${{ matrix.architecture }} + needs: setup + runs-on: ubuntu-22.04 + strategy: + matrix: + architecture: ${{ fromJson(needs.setup.outputs.architecture) }} + + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + + - uses: actions/setup-go@v5 + with: + go-version: '~1.24.3' + check-latest: true + + - name: Patch ADOT + run: ./patch-upstream.sh + + - name: Build collector layer + run: GOARCH=${{ matrix.architecture }} make package + working-directory: opentelemetry-lambda/collector + + - name: Upload layer zip + uses: actions/upload-artifact@v4 + with: + name: collector-layer-${{ matrix.architecture }} + path: opentelemetry-lambda/collector/build/opentelemetry-collector-layer-${{ matrix.architecture }}.zip + retention-days: 7 + + validate: + name: Validate layer - ${{ matrix.architecture }} + needs: [setup, build] + runs-on: ubuntu-22.04 + strategy: + matrix: + architecture: ${{ fromJson(needs.setup.outputs.architecture) }} + + steps: + - name: Download layer zip + uses: actions/download-artifact@v4 + with: + name: collector-layer-${{ matrix.architecture }} + + - name: Validate layer contents + run: | + mkdir layer-contents + unzip -q opentelemetry-collector-layer-${{ matrix.architecture }}.zip -d layer-contents + echo "Layer contents:" + find layer-contents -type f + + test -f layer-contents/extensions/collector \ + || (echo "ERROR: extensions/collector binary missing" && exit 1) + test -x layer-contents/extensions/collector \ + || (echo "ERROR: extensions/collector is not executable" && exit 1) + test -f layer-contents/collector-config/config.yaml \ + || (echo "ERROR: collector-config/config.yaml missing" && exit 1) + + echo "Validation passed ✓" + + publish: + name: Publish - ${{ matrix.aws_region }} / ${{ matrix.architecture }} + needs: [setup, validate] + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + aws_region: ${{ fromJson(needs.setup.outputs.aws_regions) }} + architecture: ${{ fromJson(needs.setup.outputs.architecture) }} + + steps: + - name: Download layer zip + uses: actions/download-artifact@v4 + with: + name: collector-layer-${{ matrix.architecture }} + + - uses: aws-actions/configure-aws-credentials@v4.0.2 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ matrix.aws_region }} + + - name: Publish layer version + id: publish + run: | + LAYER_ARN=$( + aws lambda publish-layer-version \ + --layer-name aws-otel-collector-${{ matrix.architecture }}-ver-${{ needs.setup.outputs.version }} \ + --description "ADOT Collector with decouple processor v${{ needs.setup.outputs.version }}" \ + --zip-file fileb://opentelemetry-collector-layer-${{ matrix.architecture }}.zip \ + --compatible-runtimes provided.al2 provided.al2023 \ + --query 'LayerVersionArn' \ + --output text + ) + echo "layer_arn=$LAYER_ARN" >> $GITHUB_OUTPUT + echo "::notice::${{ matrix.aws_region }} (${{ matrix.architecture }}): $LAYER_ARN" + + - name: Make layer public + run: | + VERSION=$( + aws lambda list-layer-versions \ + --layer-name aws-otel-collector-${{ matrix.architecture }}-ver-${{ needs.setup.outputs.version }} \ + --query 'max_by(LayerVersions, &Version).Version' + ) + aws lambda add-layer-version-permission \ + --layer-name aws-otel-collector-${{ matrix.architecture }}-ver-${{ needs.setup.outputs.version }} \ + --version-number $VERSION \ + --principal "*" \ + --statement-id public \ + --action lambda:GetLayerVersion + + - name: Save ARN to file + run: echo "${{ steps.publish.outputs.layer_arn }}" > arn.txt + + - name: Upload ARN artifact + uses: actions/upload-artifact@v4 + with: + name: layer-arn-${{ matrix.architecture }}-${{ matrix.aws_region }} + path: arn.txt + + release: + name: Create GitHub Release + needs: [setup, publish] + runs-on: ubuntu-22.04 + + steps: + - uses: actions/checkout@v4 + + - name: Download all ARN artifacts + uses: actions/download-artifact@v4 + with: + pattern: layer-arn-* + path: arns + merge-multiple: false + + - name: Download layer zips + uses: actions/download-artifact@v4 + with: + pattern: collector-layer-* + path: zips + merge-multiple: false + + - name: Generate release notes + run: | + echo "## ADOT Collector Lambda Layer with Decouple Processor" >> notes.md + echo "" >> notes.md + echo "Version: \`${{ needs.setup.outputs.version }}\`" >> notes.md + echo "" >> notes.md + for arch in amd64 arm64; do + echo "### \`$arch\`" >> notes.md + echo "" >> notes.md + echo "| Region | Layer ARN |" >> notes.md + echo "| ------ | --------- |" >> notes.md + for dir in arns/layer-arn-${arch}-*/; do + region=${dir#arns/layer-arn-${arch}-} + region=${region%/} + arn=$(cat ${dir}arn.txt) + echo "| $region | \`$arn\` |" >> notes.md + done + echo "" >> notes.md + done + cat notes.md + + - name: Create GitHub Release + uses: softprops/action-gh-release@v2 + with: + tag_name: ${{ github.ref_name }} + name: "ADOT Collector Layer v${{ needs.setup.outputs.version }} (with decouple processor)" + body_path: notes.md + make_latest: true + files: | + zips/collector-layer-amd64/opentelemetry-collector-layer-amd64.zip + zips/collector-layer-arm64/opentelemetry-collector-layer-arm64.zip diff --git a/.github/workflows/publish-status.yml b/.github/workflows/publish-status.yml deleted file mode 100644 index 5762e3cd5..000000000 --- a/.github/workflows/publish-status.yml +++ /dev/null @@ -1,59 +0,0 @@ -# Reusable workflow used to publish status of the caller github workflow to Cloudwatch metrics -name: Publish status -on: - workflow_call: - inputs: - # Cloudwatch Metrics namespace - namespace: - required: true - type: string - # Dimensions - repository: - required: true - type: string - branch: - required: true - type: string - workflow: - required: true - type: string - # Metric name - success: - required: true - type: boolean - # Region where the metric is published - region: - required: true - type: string - secrets: - roleArn: - required: true - -jobs: - publish-status: - runs-on: ubuntu-22.04 - permissions: - id-token: write - contents: read - steps: - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4.0.2 - with: - role-to-assume: ${{ secrets.roleArn }} - aws-region: ${{ inputs.region }} - role-duration-seconds: 21600 - - - name: Publish status success - if: ${{ inputs.success }} - run: | - aws cloudwatch put-metric-data --namespace '${{ inputs.namespace }}' \ - --metric-name Success \ - --dimensions repository=${{ inputs.repository }},branch=${{ inputs.branch }},workflow=${{ inputs.workflow }} \ - --value 1.0 - - name: Publish status failure - if: ${{ !inputs.success }} - run: | - aws cloudwatch put-metric-data --namespace '${{ inputs.namespace }}' \ - --metric-name Success \ - --dimensions repository=${{ inputs.repository }},branch=${{ inputs.branch }},workflow=${{ inputs.workflow }} \ - --value 0.0 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index 7385b0d9f..000000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,354 +0,0 @@ -name: Release public Lambda layer - -on: - workflow_dispatch: - inputs: - layer_arn_keyword: - description: 'Clone layer from keyword ARN by substituting architecture parameter (ie: arn:aws:lambda:us-east-1:012345678912:layer:aws-otel-nodejs-wrapper--d8b20954c3284abcd08f5fb8178986a89cff7160:1)' - required: true - layer_name_keyword: - description: 'Publish layer to keyword name by substituting architecture parameter (ie: aws-otel-python--ver-1-7-1)' - required: true - aws_region: - description: 'Deploy to aws region' - required: true - default: "[ \"us-east-1\", \"us-east-2\", \"us-west-1\", \"us-west-2\", \"ap-south-1\", \"ap-northeast-2\", \"ap-southeast-1\", \"ap-southeast-2\", \"ap-northeast-1\", \"ca-central-1\", \"eu-central-1\", \"eu-west-1\", \"eu-west-2\", \"eu-west-3\", \"eu-north-1\", \"sa-east-1\" ]" - architecture: - # NOTE: (NathanielRN) The operator should be extra cautious if they are - # trying to modify the release "architecture" input, because the only - # reason they would is if the Soak Test Layer ARN Keyword only worked - # for ONE architecture and FAILED for the other and at the point we - # should be really certain that we want to release such a layer. - description: 'Architecture-compatible distributions of the layer to release in each region' - required: true - default: "[ \"amd64\", \"arm64\" ]" - -env: - AMP_REGIONS: us-west-2,us-east-1,us-east-2,eu-central-1,eu-west-1 - -permissions: - id-token: write - contents: read - -jobs: - validate-inputs: - runs-on: ubuntu-22.04 - steps: - - name: Validate `layer_name_keyword` (${{ github.event.inputs.layer_name_keyword }}) - run: | - grep -Eq "aws-otel-(collector|java-agent|java-wrapper|nodejs|python)--ver-[0-9]+-[0-9]+-[0-9]+" <<< "${{ github.event.inputs.layer_name_keyword }}" - publish-prod: - runs-on: ubuntu-22.04 - needs: validate-inputs - strategy: - matrix: - architecture: ${{ fromJson(github.event.inputs.architecture) }} - aws_region: ${{ fromJson(github.event.inputs.aws_region) }} - steps: - - uses: aws-actions/configure-aws-credentials@v4.0.2 - with: - role-to-assume: ${{ secrets.INTEG_TEST_LAMBDA_ROLE_ARN }} - role-duration-seconds: 1200 - aws-region: us-east-1 - - name: Get layer ARN by substituting `${{ matrix.architecture }}` into Soak Test ARN keyword - run: | - echo LAYER_ARN=$(echo "${{ github.event.inputs.layer_arn_keyword }}" | sed 's//${{ matrix.architecture }}/') | tee --append $GITHUB_ENV - - name: Get layer name by substituting `${{ matrix.architecture }}` into Workflow Input name keyword - run: | - echo LAYER_NAME=$(echo "${{ github.event.inputs.layer_name_keyword }}" | sed 's//${{ matrix.architecture }}/') | tee --append $GITHUB_ENV - - name: Get layer kind by parsing `${{ github.event.inputs.layer_name_keyword }}` - run: | - echo LAYER_KIND=$(echo "${{ github.event.inputs.layer_name_keyword }}" | cut -d - -f 3) | tee --append $GITHUB_ENV - - name: download layer from source - run: | - URL=$(aws lambda get-layer-version-by-arn --arn ${{ env.LAYER_ARN }} --query Content.Location --output text) - curl $URL -o layer.zip - # switch to prod - - uses: aws-actions/configure-aws-credentials@v4.0.2 - with: - role-to-assume: ${{ secrets.PROD_LAMBDA_ROLE_ARN }} - role-duration-seconds: 1200 - aws-region: ${{ matrix.aws_region }} - mask-aws-account-id: false - - name: Get bucket name for release run - run: | - echo BUCKET_NAME=publish-${{ env.LAYER_KIND }}-${{ matrix.architecture }}-${{ github.run_id }}-${{ matrix.aws_region }} | tee --append $GITHUB_ENV - - name: publish - run: | - aws s3 mb s3://${{ env.BUCKET_NAME }} - aws s3 cp layer.zip s3://${{ env.BUCKET_NAME }} - layerARN=$( - aws lambda publish-layer-version \ - --layer-name ${{ env.LAYER_NAME }} \ - --content S3Bucket=${{ env.BUCKET_NAME }},S3Key=layer.zip \ - --query 'LayerVersionArn' \ - --output text - ) - echo $layerARN - mkdir ${{ env.LAYER_NAME }} - echo $layerARN > ${{ env.LAYER_NAME }}/${{ matrix.aws_region }} - cat ${{ env.LAYER_NAME }}/${{ matrix.aws_region }} - - name: public layer - run: | - layerVersion=$( - aws lambda list-layer-versions \ - --layer-name ${{ env.LAYER_NAME }} \ - --query 'max_by(LayerVersions, &Version).Version' - ) - aws lambda add-layer-version-permission \ - --layer-name ${{ env.LAYER_NAME }} \ - --version-number $layerVersion \ - --principal "*" \ - --statement-id publish \ - --action lambda:GetLayerVersion - - name: upload layer arn artifact - if: ${{ success() }} - uses: actions/upload-artifact@v4 - with: - name: ${{ env.LAYER_NAME }}-${{matrix.architecture}}-${{matrix.aws_region}} - path: ${{ env.LAYER_NAME }}/${{ matrix.aws_region }} - - name: clean s3 - if: always() - run: | - aws s3 rb --force s3://${{ env.BUCKET_NAME }} - generate-note: - runs-on: ubuntu-22.04 - needs: publish-prod - strategy: - matrix: - architecture: ${{ fromJson(github.event.inputs.architecture) }} - steps: - - uses: hashicorp/setup-terraform@v2 - - name: Get layer name by substituting `${{ matrix.architecture }}` into Workflow Input name keyword - run: | - echo LAYER_NAME=$(echo "${{ github.event.inputs.layer_name_keyword }}" | sed 's//${{ matrix.architecture }}/') | tee --append $GITHUB_ENV - - name: Get layer kind by parsing `${{ github.event.inputs.layer_name_keyword }}` - run: | - echo LAYER_KIND=$(echo "${{ github.event.inputs.layer_name_keyword }}" | cut -d - -f 3) | tee --append $GITHUB_ENV - - name: download layerARNs - uses: actions/download-artifact@v4 - with: - pattern: ${{ env.LAYER_NAME }}-* - path: ${{ env.LAYER_NAME }} - merge-multiple: true - - name: show layerARNs - run: | - for file in ${{ env.LAYER_NAME }}/* - do - echo $file - cat $file - done - - name: generate layer-note - working-directory: ${{ env.LAYER_NAME }} - run: | - echo "| Region | Layer ARN |" >> ../layer-note - echo "| ---- | ---- |" >> ../layer-note - for file in * - do - read arn < $file - echo "| " $file " | " $arn " |" >> ../layer-note - done - cd .. - cat layer-note - - name: generate tf layer - working-directory: ${{ env.LAYER_NAME }} - run: | - echo "locals {" >> ../layer.tf - if [ "${{ env.LAYER_KIND }}" != 'collector' ] - then - echo " sdk_layer_arns_${{ matrix.architecture }} = {" >> ../layer.tf - else - echo " collector_layer_arns_${{ matrix.architecture }} = {" >> ../layer.tf - fi - for file in * - do - read arn < $file - echo " \""$file"\" = \""$arn"\"" >> ../layer.tf - done - cd .. - echo " }" >> layer.tf - echo "}" >> layer.tf - terraform fmt layer.tf - cat layer.tf - - name: upload layer tf file - uses: actions/upload-artifact@v4 - with: - name: layer_${{ matrix.architecture }}.tf - path: layer.tf - smoke-test: - name: Smoke Test - (${{ matrix.aws_region }} - ${{ github.event.inputs.layer_name_keyword }} - ${{ matrix.architecture }}) - needs: generate-note - runs-on: ubuntu-22.04 - strategy: - matrix: - architecture: ${{ fromJson(github.event.inputs.architecture) }} - aws_region: ${{ fromJson(github.event.inputs.aws_region) }} - steps: - - name: Parse smoke test values from layer name - ${{ github.event.inputs.layer_name_keyword }} - # FIXME: (enowell) You can only Smoke Test 1 Sample App with this - # design. However, there are multiple Sample Apps that test the same - # Java Wrapper Lambda Layer. - # - # i.e. - # - `aws-sdk` and `okhttp` both test the `java-wrapper` layer but we - # can only test one (`aws-sdk`). - # - `go` and `dotnet` both test the `collector` layer but we can only - # test one (`go`). - run: | - LAYER_KIND=$(echo "${{ github.event.inputs.layer_name_keyword }}" | cut -d - -f 3) - if [ "$LAYER_KIND" = "python" ]; then - TEST_LANGUAGE=python - elif [ "$LAYER_KIND" = "collector" ]; then - TEST_LANGUAGE=go - else - TEST_LANGUAGE=$LAYER_KIND - fi - echo TEST_LANGUAGE=$TEST_LANGUAGE | tee --append $GITHUB_ENV - - LAYER_INSTR_TYPE=$(echo "${{ github.event.inputs.layer_name_keyword }}" | cut -d - -f 4) - if [ "$LAYER_INSTR_TYPE" = "agent" ] || [ "$LAYER_INSTR_TYPE" = "wrapper" ]; then - TEST_INSTR_TYPE=$LAYER_INSTR_TYPE - else - TEST_INSTR_TYPE="wrapper" - fi - echo TEST_INSTR_TYPE=$TEST_INSTR_TYPE | tee --append $GITHUB_ENV - - uses: actions/checkout@v4 - with: - submodules: recursive - - uses: actions/setup-java@v4 - if: ${{ env.TEST_LANGUAGE == 'java' }} - with: - distribution: corretto - java-version: '17' - - name: Cache (Java) - uses: actions/cache@v4 - if: ${{ env.TEST_LANGUAGE == 'java' }} - with: - path: | - ~/go/pkg/mod - ~/.gradle/caches - ~/.gradle/wrapper - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-gradle- - - uses: actions/setup-node@v4 - if: ${{ env.TEST_LANGUAGE == 'nodejs' }} - with: - node-version: '16' - - name: Cache (NodeJS) - uses: actions/cache@v4 - if: ${{ env.TEST_LANGUAGE == 'nodejs' }} - with: - path: | - ~/go/pkg/mod - ~/.npm - key: ${{ runner.os }}-node-${{ hashFiles('**/package.json') }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-node- - - uses: actions/setup-python@v5 - if: ${{ env.TEST_LANGUAGE == 'python' }} - with: - python-version: '3.x' - - name: Cache (Python) - uses: actions/cache@v4 - if: ${{ env.TEST_LANGUAGE == 'python' }} - with: - path: | - ~/go/pkg/mod - ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-pip- - # FIXME: (enowell) Same as above - .NET uses the collector layer, but we - # only test Go with the collector layer so comment out for now. - # - name: Use .NET Language - # uses: actions/setup-dotnet@v1 - # if: ${{ env.TEST_LANGUAGE == 'dotnet' }} - # with: - # dotnet-version: '3.1.x' - - name: Use Go Language - uses: actions/setup-go@v5 - # NOTE: (enowell) In case the languages below need to build the - # collector, and because building the collector requires go 1.18 and - # above, always setup go 1.18. - # if: ${{ env.TEST_LANGUAGE == 'go' }} - with: - go-version: '~1.24.3' - check-latest: true - - name: download layer tf file - uses: actions/download-artifact@v4 - with: - name: layer_${{ matrix.architecture }}.tf - - name: Get terraform directory - run: | - echo TERRAFORM_DIRECTORY=${{ env.TEST_LANGUAGE }}/sample-apps/aws-sdk/deploy/${{ env.TEST_INSTR_TYPE }} | - tee --append $GITHUB_ENV - - name: overwrite layer.tf - run: | - OLD_LAYER_TF_FILE=layer_${{ matrix.architecture }}.tf - cat ${{ env.TERRAFORM_DIRECTORY }}/$OLD_LAYER_TF_FILE - mv -f layer.tf ${{ env.TERRAFORM_DIRECTORY }}/$OLD_LAYER_TF_FILE - cat ${{ env.TERRAFORM_DIRECTORY }}/$OLD_LAYER_TF_FILE - # NOTE: (enowell) This builds BOTH the sample app functions AND the - # layer. But we only use the sample app build in the release workflow. - # Also, we NEED the architecture value because some apps (like .NET and - # Go) depend on this architecture value. - - name: Patch ADOT - run: ./patch-upstream.sh - - name: Build functions - run: GOARCH=${{ matrix.architecture }} ./build.sh ${{ matrix.architecture }} - working-directory: ${{ env.TEST_LANGUAGE }} - - uses: aws-actions/configure-aws-credentials@v4.0.2 - with: - role-to-assume: ${{ secrets.INTEG_TEST_LAMBDA_ROLE_ARN }} - role-duration-seconds: 1200 - aws-region: ${{ matrix.aws_region }} - - uses: hashicorp/setup-terraform@v2 - - name: Initialize terraform - run: terraform init - working-directory: ${{ env.TERRAFORM_DIRECTORY }} - - name: Get Lambda Layer `amd64` architecture value - if: ${{ matrix.architecture == 'amd64' }} - run: echo LAMBDA_FUNCTION_ARCH=x86_64 | tee --append $GITHUB_ENV - - name: Get Lambda Layer `arm64` architecture value - if: ${{ matrix.architecture == 'arm64' }} - run: echo LAMBDA_FUNCTION_ARCH=arm64 | tee --append $GITHUB_ENV - - name: Apply terraform - run: terraform apply -auto-approve - working-directory: ${{ env.TERRAFORM_DIRECTORY }} - env: - TF_VAR_function_name: lambda-${{ env.TEST_LANGUAGE }}-aws-sdk-${{ env.TEST_INSTR_TYPE }}-${{ matrix.architecture }}-${{ github.run_id }}-${{ matrix.aws_region }} - TF_VAR_architecture: ${{ env.LAMBDA_FUNCTION_ARCH }} - - name: Extract endpoint - id: extract-endpoint - run: terraform output -raw api-gateway-url - working-directory: ${{ env.TERRAFORM_DIRECTORY }} - - name: Extract AMP endpoint - if: ${{ env.TEST_LANGUAGE == 'java' && env.TEST_INSTR_TYPE == 'agent' && contains(env.AMP_REGIONS, matrix.aws_region) }} - id: extract-amp-endpoint - run: terraform output -raw amp_endpoint - working-directory: ${{ env.TERRAFORM_DIRECTORY }} - - name: Send request to endpoint - run: curl -sS ${{ steps.extract-endpoint.outputs.stdout }} - - name: Checkout test framework - uses: actions/checkout@v4 - with: - repository: aws-observability/aws-otel-test-framework - path: test-framework - - name: validate trace sample - run: | - cp adot/utils/expected-templates/${{ env.TEST_LANGUAGE }}-aws-sdk-${{ env.TEST_INSTR_TYPE }}.json test-framework/validator/src/main/resources/expected-data-template/lambdaExpectedTrace.mustache - cd test-framework - ./gradlew :validator:run --args="-c default-lambda-validation.yml --endpoint ${{ steps.extract-endpoint.outputs.stdout }} --region $AWS_REGION" - - name: validate java agent metric sample - if: ${{ env.TEST_LANGUAGE == 'java' && env.TEST_INSTR_TYPE == 'agent' && contains(env.AMP_REGIONS, matrix.aws_region) }} - run: | - cp adot/utils/expected-templates/${{ env.TEST_LANGUAGE }}-aws-sdk-${{ env.TEST_INSTR_TYPE }}-metric.json test-framework/validator/src/main/resources/expected-data-template/ampExpectedMetric.mustache - cd test-framework - ./gradlew :validator:run --args="-c prometheus-static-metric-validation.yml --cortex-instance-endpoint ${{ steps.extract-amp-endpoint.outputs.stdout }} --region $AWS_REGION" - - name: Destroy terraform - if: always() - run: terraform destroy -auto-approve - working-directory: ${{ env.TERRAFORM_DIRECTORY }} - env: - TF_VAR_architecture: ${{ env.LAMBDA_FUNCTION_ARCH }} diff --git a/.github/workflows/soaking.yml b/.github/workflows/soaking.yml deleted file mode 100644 index a17e6b662..000000000 --- a/.github/workflows/soaking.yml +++ /dev/null @@ -1,321 +0,0 @@ -name: Soak tests - -env: - AWS_DEFAULT_REGION: us-east-1 - -on: - schedule: - - cron: '0 14 * * 1,3,5' # Mon, Wed, Fri morning PST - workflow_dispatch: - inputs: - soak_config: - description: 'set memory/cpu threshold, soak time (s), emitter interval' - required: false - default: '-t 1800' - -permissions: - id-token: write - contents: read - -jobs: - soaking-test: - runs-on: ubuntu-22.04 - name: Soak Test - (${{ matrix.language }}, ${{ matrix.sample-app }}, ${{ matrix.instrumentation-type }}, ${{ matrix.architecture }}) - strategy: - fail-fast: false - matrix: - # FIXME: (enowell) Both .NET and Go Sample Apps want to Soak Test - # the same collector-only lambda layer. We count on Soaking Tests to - # test whether a layer is ready for release. However, the current - # workflow can only test one Sample App per Lambda Layer. We should - # create a separate workflow to soak-test Layers with multiple - # soak tests. - language: [ go, java, nodejs, python ] - sample-app: [ aws-sdk ] - instrumentation-type: [ wrapper ] - architecture: [ amd64, arm64 ] - include: - # FIXME: (enowell) Same problem as above, we cannot Soak Test the - # other java app (okhttp) because it will create its own Lambda Layer - # instead of soak test the same one as the `aws-sdk` sample app. - - language: java - sample-app: aws-sdk - instrumentation-type: agent - architecture: amd64 - - language: java - sample-app: aws-sdk - instrumentation-type: agent - architecture: arm64 - outputs: - go-wrapper-error: ${{ steps.set-layer-if-error-output.outputs.go-wrapper-error }} - nodejs-wrapper-error: ${{ steps.set-layer-if-error-output.outputs.nodejs-wrapper-error }} - python-wrapper-error: ${{ steps.set-layer-if-error-output.outputs.python-wrapper-error }} - java-agent-error: ${{ steps.set-layer-if-error-output.outputs.java-agent-error }} - java-wrapper-error: ${{ steps.set-layer-if-error-output.outputs.java-wrapper-error }} - - # NOTE: (enowell) When we release a Lambda Layer, we will ALWAYS release - # all the architectures TOGETHER. So all architectures will be at the same - # version. - go-wrapper-version: ${{ steps.set-collector-layer-version-output.outputs.go-wrapper-version }} - nodejs-wrapper-version: ${{ steps.set-sdk-layer-version-output.outputs.nodejs-wrapper-version }} - python-wrapper-version: ${{ steps.set-sdk-layer-version-output.outputs.python-wrapper-version }} - java-agent-version: ${{ steps.set-sdk-layer-version-output.outputs.java-agent-version }} - java-wrapper-version: ${{ steps.set-sdk-layer-version-output.outputs.java-wrapper-version }} - steps: - - uses: actions/checkout@v4 - with: - submodules: recursive - - uses: actions/setup-go@v5 - with: - go-version: '~1.24.3' - check-latest: true - - uses: actions/setup-java@v4 - if: ${{ matrix.language == 'java' }} - with: - distribution: corretto - java-version: '17' - - name: Cache (Java) - uses: actions/cache@v4 - if: ${{ matrix.language == 'java' }} - with: - path: | - ~/go/pkg/mod - ~/.gradle/caches - ~/.gradle/wrapper - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-gradle- - - name: Get default soaking test configuration - # CPU baseline was obtained empirically based on the max value observed for Go in a complete run of soak test - if: ${{ matrix.language != 'java' }} - run: | - echo SOAKING_TEST_CONFIG="-c 120 -m 70" | tee --append $GITHUB_ENV - - name: Get java soaking test configuration - # NOTE (enowell): Java's JVM is heavy and needs more memory than others. - if: ${{ matrix.language == 'java' }} - run: | - echo SOAKING_TEST_CONFIG="-c 200 -m 90" | tee --append $GITHUB_ENV - - uses: actions/setup-node@v4 - if: ${{ matrix.language == 'nodejs' }} - with: - node-version: '16' - - name: Cache (NodeJS) - uses: actions/cache@v4 - if: ${{ matrix.language == 'nodejs' }} - with: - path: | - ~/go/pkg/mod - ~/.npm - key: ${{ runner.os }}-node-${{ hashFiles('**/package.json') }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-node- - - uses: actions/setup-python@v5 - if: ${{ matrix.language == 'python' }} - with: - python-version: '3.x' - - name: Cache (Python) - uses: actions/cache@v4 - if: ${{ matrix.language == 'python' }} - with: - path: | - ~/go/pkg/mod - ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-pip- - - uses: actions/setup-dotnet@v4 - if: ${{ matrix.language == 'dotnet' }} - with: - dotnet-version: '6.0.405' - - uses: aws-actions/configure-aws-credentials@v4.0.2 - with: - role-to-assume: ${{ secrets.INTEG_TEST_LAMBDA_ROLE_ARN }} - mask-aws-account-id: false - aws-region: ${{ env.AWS_DEFAULT_REGION }} - # Default session duration is 1 hour with OIDC. - role-duration-seconds: 14400 # 4 hours - - name: Patch ADOT - run: ./patch-upstream.sh - # Login to ECR since may be needed for Python build image. - - name: Login to Public ECR - uses: docker/login-action@v3 - with: - registry: public.ecr.aws - - name: Build layers / functions - run: GOARCH=${{ matrix.architecture }} ./build.sh ${{ matrix.architecture }} - working-directory: ${{ matrix.language }} - - name: Get Lambda Layer `amd64` architecture value - if: ${{ matrix.architecture == 'amd64' }} - run: echo LAMBDA_FUNCTION_ARCH=x86_64 | tee --append $GITHUB_ENV - - name: Get Lambda Layer `arm64` architecture value - if: ${{ matrix.architecture == 'arm64' }} - run: echo LAMBDA_FUNCTION_ARCH=arm64 | tee --append $GITHUB_ENV - - name: Get terraform directory - run: | - echo TERRAFORM_DIRECTORY=${{ matrix.language }}/integration-tests/${{ matrix.sample-app }}/${{ matrix.instrumentation-type }} | - tee --append $GITHUB_ENV - - uses: hashicorp/setup-terraform@v2 - with: - terraform_version: 1.3.1 - - name: Initialize terraform - run: terraform init - working-directory: ${{ env.TERRAFORM_DIRECTORY }} - - name: Get terraform Lambda function name - run: | - echo TERRAFORM_LAMBDA_FUNCTION_NAME=lambda-${{ matrix.language }}-${{ matrix.sample-app }}-${{ matrix.instrumentation-type }}-${{ matrix.architecture }}-${{ github.run_id }} | - tee --append $GITHUB_ENV - # NOTE: (enowell) We don't need to include `sample-app` in the Lambda - # Layer name because different apps should be use the same layer, not - # create their own. However, if we ever Soak Test multiple apps, we need - # to BE CAREFUL about not creating duplicate layer with the same name. - - name: Get terraform Lambda layer name - run: | - echo TERRAFORM_LAMBDA_LAYER_NAME=aws-otel-${{ matrix.language }}-${{ matrix.instrumentation-type }}-${{ matrix.architecture }}-${{ github.sha }} | - tee --append $GITHUB_ENV - - name: Apply terraform - run: terraform apply -auto-approve - working-directory: ${{ env.TERRAFORM_DIRECTORY }} - env: - TF_VAR_sdk_layer_name: ${{ env.TERRAFORM_LAMBDA_LAYER_NAME }} - TF_VAR_collector_layer_name: ${{ env.TERRAFORM_LAMBDA_LAYER_NAME }} - TF_VAR_function_name: ${{ env.TERRAFORM_LAMBDA_FUNCTION_NAME }} - TF_VAR_architecture: ${{ env.LAMBDA_FUNCTION_ARCH }} - - name: Extract endpoint - id: extract-endpoint - run: terraform output -raw api-gateway-url - working-directory: ${{ env.TERRAFORM_DIRECTORY }} - - name: Extract AMP endpoint - if: ${{ matrix.language == 'java' && matrix.sample-app == 'aws-sdk' && matrix.instrumentation-type == 'agent' }} - id: extract-amp-endpoint - run: terraform output -raw amp_endpoint - working-directory: ${{ env.TERRAFORM_DIRECTORY }} - - name: Extract SDK layer arn - id: extract-sdk-layer-arn - if: ${{ matrix.language != 'dotnet' && matrix.language != 'go' }} - run: terraform output -raw sdk_layer_arn - working-directory: ${{ env.TERRAFORM_DIRECTORY }} - - name: Extract Collector layer arn - id: extract-collector-layer-arn - if: ${{ matrix.language == 'dotnet' || matrix.language == 'go' }} - run: terraform output -raw collector_layer_arn - working-directory: ${{ env.TERRAFORM_DIRECTORY }} - # NOTE: (enowell) `terraform output` outputs additional text we are - # not interested in because the `hashicorp/setup-terraform@v1` has a - # wrapper. We solve this by using separate steps, because this text - # doesn't show up when accessed in later steps. - # - # See more: https://github.com/hashicorp/setup-terraform/issues/20 - - name: Set SDK layer version output - id: set-sdk-layer-version-output - if: ${{ matrix.language != 'dotnet' && matrix.language != 'go' }} - run: | - version=$(echo "${{ steps.extract-sdk-layer-arn.outputs.stdout }}" | cut -d : -f 8) - echo "Found version number: $version" - echo "${{ matrix.language }}-${{ matrix.instrumentation-type }}-version=$version" >> $GITHUB_OUTPUT - working-directory: ${{ env.TERRAFORM_DIRECTORY }} - - name: Set Collector layer version output - id: set-collector-layer-version-output - if: ${{ matrix.language == 'dotnet' || matrix.language == 'go' }} - run: | - version=$(echo "${{ steps.extract-collector-layer-arn.outputs.stdout }}" | cut -d : -f 8) - echo "Found version number: $version" - echo "${{ matrix.language }}-${{ matrix.instrumentation-type }}-version=$version" >> $GITHUB_OUTPUT - working-directory: ${{ env.TERRAFORM_DIRECTORY }} - - name: Send request to endpoint - run: curl -sS ${{ steps.extract-endpoint.outputs.stdout }} - - name: Checkout test framework - uses: actions/checkout@v4 - with: - repository: aws-observability/aws-otel-test-framework - path: test-framework - - name: validate trace sample - run: | - cp adot/utils/expected-templates/${{ matrix.language }}-${{ matrix.sample-app }}-${{ matrix.instrumentation-type }}.json \ - test-framework/validator/src/main/resources/expected-data-template/lambdaExpectedTrace.mustache - cd test-framework - ./gradlew :validator:run --args="-c default-lambda-validation.yml --endpoint ${{ steps.extract-endpoint.outputs.stdout }} --region ${{ env.AWS_DEFAULT_REGION }}" - - name: validate java agent metric sample - if: ${{ matrix.language == 'java' && matrix.sample-app == 'aws-sdk' && matrix.instrumentation-type == 'agent' }} - run: | - cp adot/utils/expected-templates/${{ matrix.language }}-${{ matrix.sample-app }}-${{ matrix.instrumentation-type }}-metric.json \ - test-framework/validator/src/main/resources/expected-data-template/ampExpectedMetric.mustache - cd test-framework - ./gradlew :validator:run --args="-c prometheus-static-metric-validation.yml --cortex-instance-endpoint ${{ steps.extract-amp-endpoint.outputs.stdout }} --region ${{ env.AWS_DEFAULT_REGION }}" - - name: Run soak test - run: - >- - docker run - --rm - -e AWS_DEFAULT_REGION - -e AWS_ACCESS_KEY_ID - -e AWS_SECRET_ACCESS_KEY - -e AWS_SESSION_TOKEN - public.ecr.aws/aws-otel-test/lambda-soak:latest - -n ${{ env.TERRAFORM_LAMBDA_FUNCTION_NAME }} - -e ${{ steps.extract-endpoint.outputs.stdout }} - ${{ github.event.inputs.soak_config }} - ${{ env.SOAKING_TEST_CONFIG }} - -a ${{ matrix.architecture }} - - name: Set output if layer Soak Tests has error - id: set-layer-if-error-output - if: ${{ failure() }} - run: echo "${{ matrix.language }}-${{ matrix.instrumentation-type }}-error=FAILED" >> $GITHUB_OUTPUT - - name: Remove sdk layers from terraform management to prevent deletion. - if: ${{ matrix.language != 'go' }} - run: terraform state rm aws_lambda_layer_version.sdk_layer - working-directory: ${{ env.TERRAFORM_DIRECTORY }} - - name: Remove collector layers from terraform management to prevent deletion. - if: ${{ matrix.language == 'go' }} - run: terraform state rm aws_lambda_layer_version.collector_layer - working-directory: ${{ env.TERRAFORM_DIRECTORY }} - - name: Destroy terraform - if: always() - run: terraform destroy -auto-approve - working-directory: ${{ env.TERRAFORM_DIRECTORY }} - env: - TF_VAR_architecture: ${{ env.LAMBDA_FUNCTION_ARCH }} - output-keywords: - if: ${{ always() }} - name: Output (${{ matrix.language }}, ${{ matrix.instrumentation-type }}) Layer Keyword - runs-on: ubuntu-22.04 - needs: - - soaking-test - strategy: - fail-fast: false - matrix: - language: [ go, java, nodejs, python ] - instrumentation-type: [ wrapper ] - include: - - language: java - instrumentation-type: agent - steps: - - name: Confirm none of the architecture soak tests for the layer failed - run: | - AT_LEAST_ONE_LAYER_SOAK_TEST_FAILED=$( - echo '${{ toJSON(needs.soaking-test.outputs) }}' | - jq ' - ."${{ matrix.language }}-${{ matrix.instrumentation-type }}-error" == "FAILED" - ' || echo false - ) - [[ $AT_LEAST_ONE_LAYER_SOAK_TEST_FAILED == false ]] - - name: Output keyword for (${{ matrix.language }}, ${{ matrix.instrumentation-type }}) layer - run: | - VERSION=$( - echo '${{ toJSON(needs.soaking-test.outputs) }}' | - jq -r '."${{ matrix.language }}-${{ matrix.instrumentation-type }}-version"' - ) - echo "::warning::Layer ARN Keyword: arn:aws:lambda:${{ env.AWS_DEFAULT_REGION }}:611364707713:layer:aws-otel-${{ matrix.language }}-${{ matrix.instrumentation-type }}--${{ github.sha }}:$VERSION" - - publish-soaking-status: - needs: [soaking-test] - if: ${{ always() }} - uses: ./.github/workflows/publish-status.yml - with: - namespace: 'ADOT/GitHubActions' - repository: ${{ github.repository }} - branch: ${{ github.ref_name }} - workflow: soaking - success: ${{ needs.soaking-test.result == 'success' }} - region: us-west-2 - secrets: - roleArn: ${{ secrets.METRICS_ROLE_ARN }} diff --git a/CHANGELOG.md b/CHANGELOG.md index dbd4f1969..d073e7fb9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,14 @@ # Change Log All notable changes to this project will be documented in this file. +## Fork Release — 2026-04-07 (decouple-processor) + +- **Forked** from [aws-observability/aws-otel-lambda](https://github.com/aws-observability/aws-otel-lambda) to add decouple processor support to the ADOT collector Lambda layer. +- Added [`decoupleprocessor`](https://github.com/open-telemetry/opentelemetry-lambda/tree/main/collector/processor/decoupleprocessor) to the collector layer. This processor allows Lambda functions to complete their invocation before telemetry is fully exported, reducing response-time impact of tracing. +- Collector layer `aws-otel-collector--ver-0-138-0` based on ADOT Collector `v0.138.0` (OTel Collector `v1.44.0`). +- Published to 7 AWS regions: `us-east-1`, `us-east-2`, `us-west-2`, `eu-west-1`, `eu-central-1`, `ap-southeast-1`, `ap-northeast-1`. + + ## 2025-05-30 Release - Python layer [**aws-otel-python--ver-1-32-0**](https://aws-otel.github.io/docs/getting-started/lambda/lambda-python) contains OpenTelemetry Python `v1.32.0` and ADOT Collector for Lambda `v0.43.0` diff --git a/README.md b/README.md index 27e602cc9..e0531f5b6 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,111 @@ -# AWS managed OpenTelemetry Lambda Layers +# ADOT Collector Lambda Layer — with Decouple Processor + +> **This is a community fork of [aws-observability/aws-otel-lambda](https://github.com/aws-observability/aws-otel-lambda).** +> It publishes a custom **collector-only** Lambda layer that includes the [decouple processor](https://github.com/open-telemetry/opentelemetry-lambda/tree/main/collector/processor/decoupleprocessor), which was not accepted into the upstream project. +> All other layers (Python, Java, Node.js, .NET) are unmodified — use the [official AWS layers](https://aws-otel.github.io/docs/getting-started/lambda) for those. + +--- + +## Why this fork? + +The **decouple processor** allows a Lambda function to finish its invocation *before* the OTel collector has finished exporting telemetry. This significantly reduces the latency impact of tracing on your Lambda response times, at the cost of slightly higher billed duration. + +From the [decouple processor docs](https://github.com/open-telemetry/opentelemetry-lambda/tree/main/collector/processor/decoupleprocessor): + +> _This processor decouples the receiver and exporter ends of the pipeline. This allows the lambda function to finish before traces/metrics/logs have been exported by the collector._ + +**When combined with the batch processor**, this is the recommended pattern for minimising both latency and export costs. Use `decouple` as the **last processor** in your pipeline: + +```yaml +processors: + batch: + decouple: + +service: + pipelines: + traces: + receivers: [otlp] + processors: [batch, decouple] + exporters: [awsxray] +``` + +--- + +## Using the layer + +Find the current layer ARNs for each region on the [**GitHub Releases page**](../../releases/latest). + +Add the ARN for your region and architecture as a layer on your Lambda function: + +```bash +aws lambda update-function-configuration \ + --function-name my-function \ + --layers arn:aws:lambda:us-east-1:YOUR_ACCOUNT:layer:aws-otel-collector-amd64-ver-0-138-0:1 +``` + +Or in Terraform: + +```hcl +resource "aws_lambda_function" "example" { + # ... + layers = ["arn:aws:lambda:us-east-1:ACCOUNT:layer:aws-otel-collector-amd64-ver-0-138-0:1"] +} +``` + +The layer is compatible with `provided.al2` and `provided.al2023` runtimes (Go, .NET, Rust, or any custom runtime). + +--- + +## Supported components + +| Receivers | Processors | Exporters | Extensions | +|-----------|-----------|-----------|------------| +| `otlpreceiver` | `decoupleprocessor` ✨ | `awsemfexporter` | `sigv4authextension` | +| | | `awsxrayexporter` | | +| | | `prometheusremotewriteexporter` | | +| | | `debugexporter` | | +| | | `otlpexporter` | | +| | | `otlphttpexporter` | | + +✨ Added by this fork — not available in the official AWS layer. + +--- + +## Releases + +Releases are published automatically when a `v*` tag is pushed. Each release includes: +- Layer ARNs for all supported regions (both `amd64` and `arm64`) +- The layer zip files as downloadable artifacts (for self-hosting) + +See the [Releases page](../../releases) for all versions. + +--- + +## Releasing a new version (maintainer) + +```bash +git tag v0.138.0 +git push origin v0.138.0 +``` + +This triggers the [Publish Collector Layer](.github/workflows/publish-collector-layer.yml) workflow, which builds, tests, and publishes the layer to all configured AWS regions. + +--- + +## Keeping up with upstream + +A [weekly sync workflow](.github/workflows/sync-upstream.yml) runs every Monday and either fast-forwards `main` from [aws-observability/aws-otel-lambda](https://github.com/aws-observability/aws-otel-lambda) or opens a PR if there are conflicts. + +--- + +## Security + +See [CONTRIBUTING](CONTRIBUTING.md#security-issue-notifications) for information on reporting security issues. + +## License + +Apache-2.0 — see [LICENSE](LICENSE). + As a downstream Repo of [opentelemetry-lambda](https://github.com/open-telemetry/opentelemetry-lambda), ___aws-otel-lambda___ publishes AWS managed OpenTelemetry Lambda layers that are preconfigured for use with AWS services and bundle the reduced ADOT Collector. Users can onboard to OpenTelemetry in their existing Lambda functions by adding these ready-made layers directly. - Python layer [**aws-otel-python--ver-1-32-0**](https://aws-otel.github.io/docs/getting-started/lambda/lambda-python) contains OpenTelemetry Python `v1.32.0` and ADOT Collector for Lambda `v0.43.0` diff --git a/adot/collector/lambdacomponents/default.go b/adot/collector/lambdacomponents/default.go index 70de30e1b..8cd75ae49 100644 --- a/adot/collector/lambdacomponents/default.go +++ b/adot/collector/lambdacomponents/default.go @@ -22,10 +22,12 @@ import ( "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsxrayexporter" "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/prometheusremotewriteexporter" "github.com/open-telemetry/opentelemetry-collector-contrib/extension/sigv4authextension" + "github.com/open-telemetry/opentelemetry-lambda/collector/processor/decoupleprocessor" "go.opentelemetry.io/collector/exporter/debugexporter" "go.opentelemetry.io/collector/exporter/otlpexporter" "go.opentelemetry.io/collector/exporter/otlphttpexporter" "go.opentelemetry.io/collector/otelcol" + "go.opentelemetry.io/collector/processor" "go.opentelemetry.io/collector/receiver/otlpreceiver" "go.uber.org/multierr" ) @@ -64,10 +66,18 @@ func Components() ( errs = multierr.Append(errs, err) } + processors, err := otelcol.MakeFactoryMap[processor.Factory]( + decoupleprocessor.NewFactory(), + ) + if err != nil { + errs = multierr.Append(errs, err) + } + factories := otelcol.Factories{ Extensions: extensions, Receivers: receivers, Exporters: exporters, + Processors: processors, } return factories, errs diff --git a/adot/collector/lambdacomponents/default_test.go b/adot/collector/lambdacomponents/default_test.go index 55d69328f..9e0e26c5b 100644 --- a/adot/collector/lambdacomponents/default_test.go +++ b/adot/collector/lambdacomponents/default_test.go @@ -28,6 +28,7 @@ const ( exportersCount = 6 receiversCount = 1 extensionsCount = 1 + processorsCount = 1 ) func TestComponents(t *testing.T) { @@ -54,4 +55,9 @@ func TestComponents(t *testing.T) { assert.Len(t, extensions, extensionsCount) // aws extensions assert.NotNil(t, extensions[component.MustNewType("sigv4auth")]) + + processors := factories.Processors + assert.Len(t, processors, processorsCount) + // lambda processors + assert.NotNil(t, processors[component.MustNewType("decouple")]) } diff --git a/adot/collector/lambdacomponents/go.mod b/adot/collector/lambdacomponents/go.mod index 0d376d955..b3d947820 100644 --- a/adot/collector/lambdacomponents/go.mod +++ b/adot/collector/lambdacomponents/go.mod @@ -1,18 +1,27 @@ module github.com/aws-observability/aws-otel-lambda/adot/collector/lambdacomponents -go 1.24.0 +go 1.24.4 + +toolchain go1.24.13 + +replace ( + github.com/open-telemetry/opentelemetry-lambda/collector/lambdalifecycle => ../../../opentelemetry-lambda/collector/lambdalifecycle + github.com/open-telemetry/opentelemetry-lambda/collector/processor/decoupleprocessor => ../../../opentelemetry-lambda/collector/processor/decoupleprocessor +) require ( github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsemfexporter v0.138.0 github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.138.0 github.com/open-telemetry/opentelemetry-collector-contrib/exporter/prometheusremotewriteexporter v0.138.0 github.com/open-telemetry/opentelemetry-collector-contrib/extension/sigv4authextension v0.138.0 + github.com/open-telemetry/opentelemetry-lambda/collector/processor/decoupleprocessor v0.0.0-00010101000000-000000000000 github.com/stretchr/testify v1.11.1 go.opentelemetry.io/collector/component v1.44.0 go.opentelemetry.io/collector/exporter/debugexporter v0.138.0 go.opentelemetry.io/collector/exporter/otlpexporter v0.138.0 go.opentelemetry.io/collector/exporter/otlphttpexporter v0.138.0 go.opentelemetry.io/collector/otelcol v0.138.0 + go.opentelemetry.io/collector/processor v1.44.0 go.opentelemetry.io/collector/receiver/otlpreceiver v0.138.0 go.uber.org/multierr v1.11.0 ) @@ -90,6 +99,7 @@ require ( github.com/open-telemetry/opentelemetry-collector-contrib/pkg/resourcetotelemetry v0.138.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus v0.138.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite v0.138.0 // indirect + github.com/open-telemetry/opentelemetry-lambda/collector/lambdalifecycle v0.0.0-00010101000000-000000000000 // indirect github.com/pierrec/lz4/v4 v4.1.22 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect @@ -160,7 +170,7 @@ require ( go.opentelemetry.io/collector/pdata/xpdata v0.138.0 // indirect go.opentelemetry.io/collector/pipeline v1.44.0 // indirect go.opentelemetry.io/collector/pipeline/xpipeline v0.138.0 // indirect - go.opentelemetry.io/collector/processor v1.44.0 // indirect + go.opentelemetry.io/collector/processor/processorhelper v0.138.0 // indirect go.opentelemetry.io/collector/processor/processortest v0.138.0 // indirect go.opentelemetry.io/collector/processor/xprocessor v0.138.0 // indirect go.opentelemetry.io/collector/receiver v1.44.0 // indirect diff --git a/adot/collector/lambdacomponents/go.sum b/adot/collector/lambdacomponents/go.sum index 7e9d118d7..4af1a65ae 100644 --- a/adot/collector/lambdacomponents/go.sum +++ b/adot/collector/lambdacomponents/go.sum @@ -489,6 +489,8 @@ go.opentelemetry.io/collector/pipeline/xpipeline v0.138.0 h1:Y8blByFwDqhnEa4kOTA go.opentelemetry.io/collector/pipeline/xpipeline v0.138.0/go.mod h1:TOtck/PIWC89dI9+aYouX39boc7d+rGHP82SuH0xxN0= go.opentelemetry.io/collector/processor v1.44.0 h1:jB+vfkYSR9f7HJlGJrtncld9dmnPWndCoTHZ0Wz4nvg= go.opentelemetry.io/collector/processor v1.44.0/go.mod h1:BV0s5J7TH2YrVErfYAXvq3Z2ChJZdE84pY+sk1X55kw= +go.opentelemetry.io/collector/processor/processorhelper v0.138.0 h1:Affdz4mJdjE6iJMWO6IpLcorBr1E+HFbo3/ok194Qc4= +go.opentelemetry.io/collector/processor/processorhelper v0.138.0/go.mod h1:QS6FzV/0/4kN3VPIYA+FPMuKkJnXnxvGKdllz2Fuopw= go.opentelemetry.io/collector/processor/processortest v0.138.0 h1:WSHPESV1NqPHlt9ShzTlc9y7ZLf83223fyfC4wzJoZg= go.opentelemetry.io/collector/processor/processortest v0.138.0/go.mod h1:h+rFcy+svVipVVpAkellP5egcPYsHeOfL3o7lkFNsGs= go.opentelemetry.io/collector/processor/xprocessor v0.138.0 h1:V+zKVy2kstPhIDsGvEBIRUxq8HzAdG1zdJP/hAuwENQ=