From f691fab47afe791e935b7c7e3ae93607460ae902 Mon Sep 17 00:00:00 2001 From: Ahmad Faiyaz Date: Wed, 18 Sep 2024 11:08:21 +0100 Subject: [PATCH 1/3] Make secret value decoded for specific usecases --- .../services/secretsmanager/provider.py | 18 +++++++++++++++- .../secretsmanager/test_secretsmanager.py | 21 +++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/localstack-core/localstack/services/secretsmanager/provider.py b/localstack-core/localstack/services/secretsmanager/provider.py index 2bbc1fa62fcd1..80a10c531628a 100644 --- a/localstack-core/localstack/services/secretsmanager/provider.py +++ b/localstack-core/localstack/services/secretsmanager/provider.py @@ -1,10 +1,11 @@ from __future__ import annotations +import base64 import json import logging import re import time -from typing import Final, Optional, Union +from typing import Any, Dict, Final, Optional, Union import moto.secretsmanager.exceptions as moto_exception from botocore.utils import InvalidArnException @@ -246,6 +247,8 @@ def get_secret_value( self._raise_if_default_kms_key(secret_id, context, backend) try: response = backend.get_secret_value(secret_id, version_id, version_stage) + if should_decode_secret_binary(context): + response = decode_secret_binary_from_response(response) except moto_exception.SecretNotFoundException: raise ResourceNotFoundException( f"Secrets Manager can't find the specified secret value for staging label: {version_stage}" @@ -863,6 +866,19 @@ def get_resource_policy_response(self): return self.backend.get_resource_policy(secret_id=secret_id) +def should_decode_secret_binary(context: RequestContext): + headers = context.request.headers + user_agents = headers.get("User-Agent", "").split(" ") + return not any(ua_val.startswith(("boto3", "botocore", "aws-cli")) for ua_val in user_agents) + + +def decode_secret_binary_from_response(response: Dict[str, Any]): + if "SecretBinary" in response: + response["SecretBinary"] = base64.b64decode(response["SecretBinary"]) + + return response + + def delete_resource_policy_model(self, secret_id): if self._is_valid_identifier(secret_id): self.secrets[secret_id].policy = None diff --git a/tests/aws/services/secretsmanager/test_secretsmanager.py b/tests/aws/services/secretsmanager/test_secretsmanager.py index ab55402a39bb7..b272ca7db9052 100644 --- a/tests/aws/services/secretsmanager/test_secretsmanager.py +++ b/tests/aws/services/secretsmanager/test_secretsmanager.py @@ -1,3 +1,4 @@ +import base64 import json import logging import os @@ -2395,6 +2396,26 @@ def test_get_secret_value_errors(self, aws_client, create_secret, sm_snapshot): ) sm_snapshot.match("mismatch_version_id_and_stage", exc.value.response) + @markers.aws.unknown + def test_get_secret_value(self, aws_client, create_secret, sm_snapshot): + secret_name = short_uid() + secret_string = b"footest" + secret_string_b64_encoded = base64.b64encode(secret_string) + response = create_secret( + Name=secret_name, + SecretBinary=secret_string_b64_encoded, + ) + + sm_snapshot.add_transformers_list( + sm_snapshot.transform.secretsmanager_secret_id_arn(response, 0) + ) + + secret_arn = response["ARN"] + + value_response = aws_client.secretsmanager.get_secret_value(SecretId=secret_arn) + + assert value_response["SecretBinary"] == secret_string_b64_encoded + class TestSecretsManagerMultiAccounts: @markers.aws.validated From d729a9a78632410aa8f32da88cd545cdd9bd08ce Mon Sep 17 00:00:00 2001 From: Ahmad Faiyaz Date: Thu, 19 Sep 2024 10:36:27 +0100 Subject: [PATCH 2/3] Update snapshot related to secrets fetching --- .../secretsmanager/test_secretsmanager.py | 8 +++++--- .../test_secretsmanager.snapshot.json | 19 +++++++++++++++++++ .../test_secretsmanager.validation.json | 3 +++ 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/tests/aws/services/secretsmanager/test_secretsmanager.py b/tests/aws/services/secretsmanager/test_secretsmanager.py index b272ca7db9052..8307908144904 100644 --- a/tests/aws/services/secretsmanager/test_secretsmanager.py +++ b/tests/aws/services/secretsmanager/test_secretsmanager.py @@ -2396,7 +2396,7 @@ def test_get_secret_value_errors(self, aws_client, create_secret, sm_snapshot): ) sm_snapshot.match("mismatch_version_id_and_stage", exc.value.response) - @markers.aws.unknown + @markers.aws.validated def test_get_secret_value(self, aws_client, create_secret, sm_snapshot): secret_name = short_uid() secret_string = b"footest" @@ -2412,9 +2412,11 @@ def test_get_secret_value(self, aws_client, create_secret, sm_snapshot): secret_arn = response["ARN"] - value_response = aws_client.secretsmanager.get_secret_value(SecretId=secret_arn) + secret_value_response = aws_client.secretsmanager.get_secret_value(SecretId=secret_arn) + + sm_snapshot.match("secret_value_response", secret_value_response) - assert value_response["SecretBinary"] == secret_string_b64_encoded + assert secret_value_response["SecretBinary"] == secret_string_b64_encoded class TestSecretsManagerMultiAccounts: diff --git a/tests/aws/services/secretsmanager/test_secretsmanager.snapshot.json b/tests/aws/services/secretsmanager/test_secretsmanager.snapshot.json index 654835b37ad2f..44c04e4a26cb6 100644 --- a/tests/aws/services/secretsmanager/test_secretsmanager.snapshot.json +++ b/tests/aws/services/secretsmanager/test_secretsmanager.snapshot.json @@ -4509,5 +4509,24 @@ "get_secret_value_version_not_found_ex": "An error occurred (ResourceNotFoundException) when calling the GetSecretValue operation: Secrets Manager can't find the specified secret value for VersionId: ", "get_secret_value_stage_not_found_ex": "An error occurred (ResourceNotFoundException) when calling the GetSecretValue operation: Secrets Manager can't find the specified secret value for staging label: AWSPENDING" } + }, + "tests/aws/services/secretsmanager/test_secretsmanager.py::TestSecretsManager::test_get_secret_value": { + "recorded-date": "19-09-2024, 09:35:21", + "recorded-content": { + "secret_value_response": { + "ARN": "arn::secretsmanager::111111111111:secret:", + "CreatedDate": "datetime", + "Name": "", + "SecretBinary": "b'Zm9vdGVzdA=='", + "VersionId": "", + "VersionStages": [ + "AWSCURRENT" + ], + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + } + } } } diff --git a/tests/aws/services/secretsmanager/test_secretsmanager.validation.json b/tests/aws/services/secretsmanager/test_secretsmanager.validation.json index 36d3037db8efe..7abe1679921bd 100644 --- a/tests/aws/services/secretsmanager/test_secretsmanager.validation.json +++ b/tests/aws/services/secretsmanager/test_secretsmanager.validation.json @@ -44,6 +44,9 @@ "tests/aws/services/secretsmanager/test_secretsmanager.py::TestSecretsManager::test_get_random_exclude_characters_and_symbols": { "last_validated_date": "2024-03-15T08:12:01+00:00" }, + "tests/aws/services/secretsmanager/test_secretsmanager.py::TestSecretsManager::test_get_secret_value": { + "last_validated_date": "2024-09-19T09:35:21+00:00" + }, "tests/aws/services/secretsmanager/test_secretsmanager.py::TestSecretsManager::test_get_secret_value_errors": { "last_validated_date": "2024-04-11T05:37:47+00:00" }, From b7340571503d42e51963262e422559575862d87d Mon Sep 17 00:00:00 2001 From: Ahmad Faiyaz Date: Thu, 19 Sep 2024 21:22:44 +0100 Subject: [PATCH 3/3] Add snapshot testing for http request --- .../services/secretsmanager/provider.py | 9 +----- .../secretsmanager/test_secretsmanager.py | 30 +++++++++++++++---- .../test_secretsmanager.snapshot.json | 14 +++++++-- .../test_secretsmanager.validation.json | 2 +- 4 files changed, 39 insertions(+), 16 deletions(-) diff --git a/localstack-core/localstack/services/secretsmanager/provider.py b/localstack-core/localstack/services/secretsmanager/provider.py index 80a10c531628a..dd1c113dcd032 100644 --- a/localstack-core/localstack/services/secretsmanager/provider.py +++ b/localstack-core/localstack/services/secretsmanager/provider.py @@ -247,8 +247,7 @@ def get_secret_value( self._raise_if_default_kms_key(secret_id, context, backend) try: response = backend.get_secret_value(secret_id, version_id, version_stage) - if should_decode_secret_binary(context): - response = decode_secret_binary_from_response(response) + response = decode_secret_binary_from_response(response) except moto_exception.SecretNotFoundException: raise ResourceNotFoundException( f"Secrets Manager can't find the specified secret value for staging label: {version_stage}" @@ -866,12 +865,6 @@ def get_resource_policy_response(self): return self.backend.get_resource_policy(secret_id=secret_id) -def should_decode_secret_binary(context: RequestContext): - headers = context.request.headers - user_agents = headers.get("User-Agent", "").split(" ") - return not any(ua_val.startswith(("boto3", "botocore", "aws-cli")) for ua_val in user_agents) - - def decode_secret_binary_from_response(response: Dict[str, Any]): if "SecretBinary" in response: response["SecretBinary"] = base64.b64decode(response["SecretBinary"]) diff --git a/tests/aws/services/secretsmanager/test_secretsmanager.py b/tests/aws/services/secretsmanager/test_secretsmanager.py index 8307908144904..7ea2d4042a14e 100644 --- a/tests/aws/services/secretsmanager/test_secretsmanager.py +++ b/tests/aws/services/secretsmanager/test_secretsmanager.py @@ -2397,13 +2397,17 @@ def test_get_secret_value_errors(self, aws_client, create_secret, sm_snapshot): sm_snapshot.match("mismatch_version_id_and_stage", exc.value.response) @markers.aws.validated - def test_get_secret_value(self, aws_client, create_secret, sm_snapshot): + @markers.snapshot.skip_snapshot_verify(paths=["$..CreatedDate"]) + def test_get_secret_value( + self, aws_client, aws_http_client_factory, region_name, create_secret, sm_snapshot + ): secret_name = short_uid() - secret_string = b"footest" - secret_string_b64_encoded = base64.b64encode(secret_string) + secret_string = "footest" + secret_string_b64_encoded = base64.b64encode(secret_string.encode()) + response = create_secret( Name=secret_name, - SecretBinary=secret_string_b64_encoded, + SecretBinary=secret_string, ) sm_snapshot.add_transformers_list( @@ -2416,7 +2420,23 @@ def test_get_secret_value(self, aws_client, create_secret, sm_snapshot): sm_snapshot.match("secret_value_response", secret_value_response) - assert secret_value_response["SecretBinary"] == secret_string_b64_encoded + assert secret_value_response["SecretBinary"] == secret_string.encode() + + client = aws_http_client_factory( + "secretsmanager", region=region_name, signer_factory=SigV4Auth + ) + parameters = {"SecretId": secret_name} + + headers = { + "X-Amz-Target": "secretsmanager.GetSecretValue", + "Content-Type": "application/x-amz-json-1.1", + } + + response = client.post("/", data=json.dumps(parameters), headers=headers) + json_response = response.json() + sm_snapshot.match("secret_value_http_response", json_response) + + assert json_response["SecretBinary"] == str(secret_string_b64_encoded, encoding="utf-8") class TestSecretsManagerMultiAccounts: diff --git a/tests/aws/services/secretsmanager/test_secretsmanager.snapshot.json b/tests/aws/services/secretsmanager/test_secretsmanager.snapshot.json index 44c04e4a26cb6..5dae9de4b67e0 100644 --- a/tests/aws/services/secretsmanager/test_secretsmanager.snapshot.json +++ b/tests/aws/services/secretsmanager/test_secretsmanager.snapshot.json @@ -4511,13 +4511,13 @@ } }, "tests/aws/services/secretsmanager/test_secretsmanager.py::TestSecretsManager::test_get_secret_value": { - "recorded-date": "19-09-2024, 09:35:21", + "recorded-date": "19-09-2024, 20:19:26", "recorded-content": { "secret_value_response": { "ARN": "arn::secretsmanager::111111111111:secret:", "CreatedDate": "datetime", "Name": "", - "SecretBinary": "b'Zm9vdGVzdA=='", + "SecretBinary": "b'footest'", "VersionId": "", "VersionStages": [ "AWSCURRENT" @@ -4526,6 +4526,16 @@ "HTTPHeaders": {}, "HTTPStatusCode": 200 } + }, + "secret_value_http_response": { + "ARN": "arn::secretsmanager::111111111111:secret:", + "CreatedDate": 1726777166.068, + "Name": "", + "SecretBinary": "Zm9vdGVzdA==", + "VersionId": "", + "VersionStages": [ + "AWSCURRENT" + ] } } } diff --git a/tests/aws/services/secretsmanager/test_secretsmanager.validation.json b/tests/aws/services/secretsmanager/test_secretsmanager.validation.json index 7abe1679921bd..988c7809c65b1 100644 --- a/tests/aws/services/secretsmanager/test_secretsmanager.validation.json +++ b/tests/aws/services/secretsmanager/test_secretsmanager.validation.json @@ -45,7 +45,7 @@ "last_validated_date": "2024-03-15T08:12:01+00:00" }, "tests/aws/services/secretsmanager/test_secretsmanager.py::TestSecretsManager::test_get_secret_value": { - "last_validated_date": "2024-09-19T09:35:21+00:00" + "last_validated_date": "2024-09-19T20:19:26+00:00" }, "tests/aws/services/secretsmanager/test_secretsmanager.py::TestSecretsManager::test_get_secret_value_errors": { "last_validated_date": "2024-04-11T05:37:47+00:00"