From ed3aae01412753d0defd28adb6e92d7f9005910a Mon Sep 17 00:00:00 2001 From: Benjamin Simon Date: Fri, 24 Nov 2023 22:02:57 +0100 Subject: [PATCH 1/3] implement S3Control native provider --- localstack/aws/api/s3control/__init__.py | 2 + localstack/aws/protocol/serializer.py | 74 ++++++++- localstack/aws/spec-patches.json | 21 +++ localstack/config.py | 7 + localstack/services/providers.py | 8 + localstack/services/s3control/v2/__init__.py | 0 localstack/services/s3control/v2/models.py | 16 ++ localstack/services/s3control/v2/provider.py | 150 ++++++++++++++++++ .../aws/services/s3control/test_s3control.py | 96 ++++++++++- .../s3control/test_s3control.snapshot.json | 79 +++++++++ 10 files changed, 446 insertions(+), 7 deletions(-) create mode 100644 localstack/services/s3control/v2/__init__.py create mode 100644 localstack/services/s3control/v2/models.py create mode 100644 localstack/services/s3control/v2/provider.py create mode 100644 tests/aws/services/s3control/test_s3control.snapshot.json diff --git a/localstack/aws/api/s3control/__init__.py b/localstack/aws/api/s3control/__init__.py index 3728bbb4e64d6..5b1bbe9bfefd6 100644 --- a/localstack/aws/api/s3control/__init__.py +++ b/localstack/aws/api/s3control/__init__.py @@ -92,6 +92,7 @@ TagValueString = str TrafficDialPercentage = int VpcId = str +HostId = str class AsyncOperationName(str): @@ -430,6 +431,7 @@ class NoSuchPublicAccessBlockConfiguration(ServiceException): code: str = "NoSuchPublicAccessBlockConfiguration" sender_fault: bool = False status_code: int = 404 + AccountId: Optional[AccountId] class NotFoundException(ServiceException): diff --git a/localstack/aws/protocol/serializer.py b/localstack/aws/protocol/serializer.py index 471643bd20537..8e591eefbbcd8 100644 --- a/localstack/aws/protocol/serializer.py +++ b/localstack/aws/protocol/serializer.py @@ -619,7 +619,6 @@ def _serialize_error( request_id_element.text = request_id self._add_additional_error_tags(vars(error), root, shape, mime_type) - response.set_response(self._encode_payload(self._node_to_string(root, mime_type))) def _add_error_tags( @@ -627,6 +626,7 @@ def _add_error_tags( ) -> None: code_tag = ETree.SubElement(error_tag, "Code") code_tag.text = error.code + message = self._get_error_message(error) if message: self._default_serialize(error_tag, message, None, "Message", mime_type) @@ -1586,6 +1586,77 @@ def _timestamp_iso8601(value: datetime) -> str: return value.strftime("%Y-%m-%dT%H:%M:%S.000Z") +class S3ControlResponseSerializer(RestXMLResponseSerializer): + """ + The ``S3ResponseSerializer`` adds some minor logic to handle S3 specific peculiarities with the error response + serialization and the root node tag. + """ + + def _serialize_error( + self, + error: ServiceException, + response: HttpResponse, + shape: StructureShape, + operation_model: OperationModel, + mime_type: str, + request_id: str, + ) -> None: + # Check if we need to add a namespace + attr = ( + {"xmlns": operation_model.metadata.get("xmlNamespace")} + if "xmlNamespace" in operation_model.metadata + else {} + ) + root = ETree.Element("ErrorResponse", attr) + + error_tag = ETree.SubElement(root, "Error") + self._add_error_tags(error, error_tag, mime_type) + request_id_element = ETree.SubElement(root, "RequestId") + request_id_element.text = request_id + + host_id_element = ETree.SubElement(root, "HostId") + host_id_element.text = ( + "9Gjjt1m+cjU4OPvX9O9/8RuvnG41MRb/18Oux2o5H5MY7ISNTlXN+Dz9IG62/ILVxhAGI0qyPfg=" + ) + + self._add_additional_error_tags(vars(error), root, shape, mime_type) + p = self._encode_payload(self._node_to_string(root, mime_type)) + print(f"{p=}") + response.set_response(self._encode_payload(self._node_to_string(root, mime_type))) + + def _add_error_tags( + self, error: ServiceException, error_tag: ETree.Element, mime_type: str + ) -> None: + super()._add_error_tags(error, error_tag, mime_type) + + if hasattr(error, "AccountId"): + account_id_tag = ETree.SubElement(error_tag, "AccountId") + account_id_tag.text = error.AccountId + + def _add_additional_error_tags( + self, parameters: dict, node: ETree, shape: StructureShape, mime_type: str + ): + if shape: + params = {} + # TODO add a possibility to serialize simple non-modelled errors (like S3 NoSuchBucket#BucketName) + for member in shape.members: + # XML protocols do not add modeled default fields to the root node + # (tested for cloudfront, route53, cloudwatch, iam) + if member.lower() not in ["code", "message", "accountid"] and member in parameters: + params[member] = parameters[member] + + # If there is an error shape with members which should be set, they need to be added to the node + if params: + # Serialize the remaining params + root_name = shape.serialization.get("name", shape.name) + pseudo_root = ETree.Element("") + self._serialize(shape, params, pseudo_root, root_name, mime_type) + real_root = list(pseudo_root)[0] + # Add the child elements to the already created root error element + for child in list(real_root): + node.append(child) + + class SqsQueryResponseSerializer(QueryResponseSerializer): """ Unfortunately, SQS uses a rare interpretation of the XML protocol: It uses HTML entities within XML tag text nodes. @@ -1761,6 +1832,7 @@ def create_serializer(service: ServiceModel) -> ResponseSerializer: "sqs-query": SqsQueryResponseSerializer, "sqs": SqsResponseSerializer, "s3": S3ResponseSerializer, + "s3control": S3ControlResponseSerializer, } protocol_specific_serializers = { "query": QueryResponseSerializer, diff --git a/localstack/aws/spec-patches.json b/localstack/aws/spec-patches.json index a327eebd71c77..f5e26e0b5c512 100644 --- a/localstack/aws/spec-patches.json +++ b/localstack/aws/spec-patches.json @@ -1153,5 +1153,26 @@ "exception": true } } + ], + "s3control/2018-08-20/service-2": [ + { + "op": "add", + "path": "/operations/DeletePublicAccessBlock/http/responseCode", + "value": 204 + }, + { + "op": "add", + "path": "/shapes/HostId", + "value": { + "type": "string" + } + }, + { + "op": "add", + "path": "/shapes/NoSuchPublicAccessBlockConfiguration/members/AccountId", + "value": { + "shape": "AccountId" + } + } ] } diff --git a/localstack/config.py b/localstack/config.py index 922142db7c311..e072c808db29c 100644 --- a/localstack/config.py +++ b/localstack/config.py @@ -423,6 +423,13 @@ def in_docker(): # whether the S3 legacy V2/ASF provider is enabled LEGACY_V2_S3_PROVIDER = os.environ.get("PROVIDER_OVERRIDE_S3", "") in ("v2", "legacy_v2", "asf") +# force the native provider for tests +if not os.environ.get("PROVIDER_OVERRIDE_S3CONTROL"): + os.environ["PROVIDER_OVERRIDE_S3CONTROL"] = "v2" + +# whether the S3 Control native provider is enabled +NATIVE_S3_CONTROL_PROVIDER = os.environ.get("PROVIDER_OVERRIDE_S3CONTROL", "") == "v2" + # Whether to report internal failures as 500 or 501 errors. FAIL_FAST = is_env_true("FAIL_FAST") diff --git a/localstack/services/providers.py b/localstack/services/providers.py index f37d2a7a0f52c..221332db90c79 100644 --- a/localstack/services/providers.py +++ b/localstack/services/providers.py @@ -262,6 +262,14 @@ def s3control(): return Service.for_provider(provider, dispatch_table_factory=MotoFallbackDispatcher) +@aws_provider(api="s3control", name="v2") +def s3control_v2(): + from localstack.services.s3control.v2.provider import S3ControlProvider + + provider = S3ControlProvider() + return Service.for_provider(provider) + + @aws_provider() def scheduler(): from localstack.services.moto import MotoFallbackDispatcher diff --git a/localstack/services/s3control/v2/__init__.py b/localstack/services/s3control/v2/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/localstack/services/s3control/v2/models.py b/localstack/services/s3control/v2/models.py new file mode 100644 index 0000000000000..6320b2f9e1535 --- /dev/null +++ b/localstack/services/s3control/v2/models.py @@ -0,0 +1,16 @@ +from localstack.aws.api.s3control import PublicAccessBlockConfiguration +from localstack.services.stores import ( + AccountRegionBundle, + BaseStore, + CrossRegionAttribute, +) + + +class S3ControlStore(BaseStore): + # buckets: dict[BucketName, S3Bucket] = CrossRegionAttribute(default=dict) + public_access_block: PublicAccessBlockConfiguration = CrossRegionAttribute(default=dict) + # access_point_alias: dict[Alias, BucketName] = CrossRegionAttribute(default=dict) + # global_bucket_map: dict[BucketName, AccountId] = CrossAccountAttribute(default=dict) + + +s3control_stores = AccountRegionBundle[S3ControlStore]("s3control", S3ControlStore) diff --git a/localstack/services/s3control/v2/provider.py b/localstack/services/s3control/v2/provider.py new file mode 100644 index 0000000000000..e51c007667518 --- /dev/null +++ b/localstack/services/s3control/v2/provider.py @@ -0,0 +1,150 @@ +from localstack.aws.api import CommonServiceException, RequestContext +from localstack.aws.api.s3control import ( + AccessPointName, + AccountId, + BucketName, + CreateAccessPointResult, + GetAccessPointPolicyResult, + GetAccessPointPolicyStatusResult, + GetAccessPointResult, + GetPublicAccessBlockOutput, + ListAccessPointsResult, + MaxResults, + NonEmptyMaxLength1024String, + NoSuchPublicAccessBlockConfiguration, + Policy, + PublicAccessBlockConfiguration, + S3ControlApi, + VpcConfiguration, +) +from localstack.services.s3.utils import validate_dict_fields +from localstack.services.s3control.v2.models import S3ControlStore, s3control_stores + + +class MalformedXML(CommonServiceException): + def __init__(self, message=None): + if not message: + message = "The XML you provided was not well-formed or did not validate against our published schema" + super().__init__("MalformedXML", status_code=400, message=message) + + +class InvalidRequest(CommonServiceException): + def __init__(self, message=None): + super().__init__("InvalidRequest", status_code=400, message=message) + + +FAKE_HOST_ID = "9Gjjt1m+cjU4OPvX9O9/8RuvnG41MRb/18Oux2o5H5MY7ISNTlXN+Dz9IG62/ILVxhAGI0qyPfg=" + + +class S3ControlProvider(S3ControlApi): + """ + Lots of S3 Control API methods are related to S3 Outposts (S3 in your own datacenter) + These are not implemented in this provider + """ + + @staticmethod + def get_store(account_id: str, region_name: str) -> S3ControlStore: + return s3control_stores[account_id][region_name] + + def put_public_access_block( + self, + context: RequestContext, + public_access_block_configuration: PublicAccessBlockConfiguration, + account_id: AccountId, + ) -> None: + # TODO: do some check between passed account_id and context.account_id, but this is IAM realm + # the region does not matter, everything is global + store = self.get_store(account_id, context.region) + + public_access_block_fields = { + "BlockPublicAcls", + "BlockPublicPolicy", + "IgnorePublicAcls", + "RestrictPublicBuckets", + } + + if not validate_dict_fields( + public_access_block_configuration, + required_fields=set(), + optional_fields=public_access_block_fields, + ): + raise MalformedXML() + + if not public_access_block_configuration: + raise InvalidRequest( + "Must specify at least one configuration.", + ) + + for field in public_access_block_fields: + if public_access_block_configuration.get(field) is None: + public_access_block_configuration[field] = False + + store.public_access_block = public_access_block_configuration + + def get_public_access_block( + self, context: RequestContext, account_id: AccountId + ) -> GetPublicAccessBlockOutput: + store = self.get_store(context.account_id, context.region) + if not store.public_access_block: + raise NoSuchPublicAccessBlockConfiguration( + "The public access block configuration was not found", + AccountId=account_id, + ) + + return GetPublicAccessBlockOutput(PublicAccessBlockConfiguration=store.public_access_block) + + def delete_public_access_block(self, context: RequestContext, account_id: AccountId) -> None: + store = self.get_store(context.account_id, context.region) + store.public_access_block = None + + def create_access_point( + self, + context: RequestContext, + account_id: AccountId, + name: AccessPointName, + bucket: BucketName, + vpc_configuration: VpcConfiguration = None, + public_access_block_configuration: PublicAccessBlockConfiguration = None, + bucket_account_id: AccountId = None, + ) -> CreateAccessPointResult: + pass + + def get_access_point( + self, context: RequestContext, account_id: AccountId, name: AccessPointName + ) -> GetAccessPointResult: + pass + + def list_access_points( + self, + context: RequestContext, + account_id: AccountId, + bucket: BucketName = None, + next_token: NonEmptyMaxLength1024String = None, + max_results: MaxResults = None, + ) -> ListAccessPointsResult: + pass + + def delete_access_point( + self, context: RequestContext, account_id: AccountId, name: AccessPointName + ) -> None: + pass + + def put_access_point_policy( + self, context: RequestContext, account_id: AccountId, name: AccessPointName, policy: Policy + ) -> None: + pass + + def get_access_point_policy( + self, context: RequestContext, account_id: AccountId, name: AccessPointName + ) -> GetAccessPointPolicyResult: + pass + + def delete_access_point_policy( + self, context: RequestContext, account_id: AccountId, name: AccessPointName + ) -> None: + pass + + def get_access_point_policy_status( + self, context: RequestContext, account_id: AccountId, name: AccessPointName + ) -> GetAccessPointPolicyStatusResult: + pass diff --git a/tests/aws/services/s3control/test_s3control.py b/tests/aws/services/s3control/test_s3control.py index 1543a9a1c65a9..db3cde25e466a 100644 --- a/tests/aws/services/s3control/test_s3control.py +++ b/tests/aws/services/s3control/test_s3control.py @@ -1,4 +1,5 @@ import pytest +from botocore.client import Config from botocore.exceptions import ClientError from localstack import config @@ -7,18 +8,35 @@ TEST_AWS_ACCOUNT_ID, TEST_AWS_SECRET_ACCESS_KEY, ) +from localstack.testing.aws.util import is_aws_cloud from localstack.testing.pytest import markers remote_endpoint = config.external_service_url(protocol="https") +s3_control_endpoint = f"http://s3-control.{remote_endpoint.split('://')[1]}" + + +@pytest.fixture(autouse=True) +def s3control_snapshot(snapshot): + snapshot.add_transformers_list( + [snapshot.transform.key_value("HostId", reference_replacement=False)] + ) @pytest.fixture -def s3control_client(aws_client_factory): - return aws_client_factory( - aws_access_key_id=TEST_AWS_ACCESS_KEY_ID, - aws_secret_access_key=TEST_AWS_SECRET_ACCESS_KEY, - endpoint_url=remote_endpoint, - ).s3control +def s3control_client(aws_client_factory, aws_client): + """ + The endpoint for S3 Control looks like `http(s)://.s3-control./v20180820/configuration/ + We need to manually set it to something else than `localhost` so that it is resolvable, as boto will prefix the host + with the account-id + """ + if not is_aws_cloud(): + return aws_client_factory( + aws_access_key_id=TEST_AWS_ACCESS_KEY_ID, + aws_secret_access_key=TEST_AWS_SECRET_ACCESS_KEY, + endpoint_url=s3_control_endpoint, + ).s3control + else: + return aws_client.s3control @markers.aws.unknown @@ -64,3 +82,69 @@ def test_public_access_block_validations(s3control_client): AccountId=TEST_AWS_ACCOUNT_ID, PublicAccessBlockConfiguration={} ) assert error.value.response["Error"]["Code"] == "InvalidRequest" + + +class TestS3ControlPublicAccessBlock: + @markers.aws.validated + # @pytest.mark.xfail( + # condition=config.LEGACY_V2_S3_PROVIDER, + # reason="Moto implementation does not have default public access block", + # ) + def test_crud_public_access_block(self, s3control_client, account_id, snapshot): + with pytest.raises(ClientError) as e: + s3control_client.get_public_access_block(AccountId=account_id) + snapshot.match("get-default-public-access-block", e.value.response) + + put_public_access_block = s3control_client.put_public_access_block( + AccountId=account_id, + PublicAccessBlockConfiguration={ + "BlockPublicAcls": False, + "IgnorePublicAcls": False, + "BlockPublicPolicy": False, + }, + ) + snapshot.match("put-public-access-block", put_public_access_block) + + get_public_access_block = s3control_client.get_public_access_block(AccountId=account_id) + snapshot.match("get-public-access-block", get_public_access_block) + + delete_public_access_block = s3control_client.delete_public_access_block( + AccountId=account_id + ) + snapshot.match("delete-public-access-block", delete_public_access_block) + + with pytest.raises(ClientError) as e: + s3control_client.get_public_access_block(AccountId=account_id) + snapshot.match("get-public-access-block-after-delete", e.value.response) + + delete_public_access_block = s3control_client.delete_public_access_block( + AccountId=account_id + ) + snapshot.match("idempotent-delete-public-access-block", delete_public_access_block) + + @markers.aws.validated + def test_empty_public_access_block(self, aws_client_factory, account_id, snapshot): + # we need to disable validation for this test + if not is_aws_cloud(): + s3control_client = aws_client_factory( + config=Config(parameter_validation=False), + aws_access_key_id=TEST_AWS_ACCESS_KEY_ID, + aws_secret_access_key=TEST_AWS_SECRET_ACCESS_KEY, + endpoint_url=s3_control_endpoint, + ).s3control + else: + s3control_client = aws_client_factory( + config=Config(parameter_validation=False) + ).s3control + + with pytest.raises(ClientError) as e: + s3control_client.put_public_access_block( + AccountId=account_id, + PublicAccessBlockConfiguration={}, + ) + snapshot.match("put-public-access-block-empty", e.value.response) + # Wanted to try it with a wrong key in the PublicAccessBlockConfiguration but boto is unable to serialize + + +class TestS3ControlAccessPoint: + pass diff --git a/tests/aws/services/s3control/test_s3control.snapshot.json b/tests/aws/services/s3control/test_s3control.snapshot.json new file mode 100644 index 0000000000000..9f55b7a16ff4b --- /dev/null +++ b/tests/aws/services/s3control/test_s3control.snapshot.json @@ -0,0 +1,79 @@ +{ + "tests/aws/services/s3control/test_s3control.py::TestS3ControlPublicAccessBlock::test_crud_public_access_block": { + "recorded-date": "24-11-2023, 21:36:06", + "recorded-content": { + "get-default-public-access-block": { + "Error": { + "AccountId": "111111111111", + "Code": "NoSuchPublicAccessBlockConfiguration", + "Message": "The public access block configuration was not found" + }, + "HostId": "host-id", + "Message": "The public access block configuration was not found", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 404 + } + }, + "put-public-access-block": { + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "get-public-access-block": { + "PublicAccessBlockConfiguration": { + "BlockPublicAcls": false, + "BlockPublicPolicy": false, + "IgnorePublicAcls": false, + "RestrictPublicBuckets": false + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "delete-public-access-block": { + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 204 + } + }, + "get-public-access-block-after-delete": { + "Error": { + "AccountId": "111111111111", + "Code": "NoSuchPublicAccessBlockConfiguration", + "Message": "The public access block configuration was not found" + }, + "HostId": "host-id", + "Message": "The public access block configuration was not found", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 404 + } + }, + "idempotent-delete-public-access-block": { + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 204 + } + } + } + }, + "tests/aws/services/s3control/test_s3control.py::TestS3ControlPublicAccessBlock::test_empty_public_access_block": { + "recorded-date": "24-11-2023, 22:02:08", + "recorded-content": { + "put-public-access-block-empty": { + "Error": { + "Code": "InvalidRequest", + "Message": "Must specify at least one configuration." + }, + "HostId": "host-id", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 400 + } + } + } + } +} From bd5bfc1abd7ecb47a2bf1de80ebd4f6710aeb9a2 Mon Sep 17 00:00:00 2001 From: Benjamin Simon Date: Fri, 24 Nov 2023 23:29:26 +0100 Subject: [PATCH 2/3] add AccessPoint to S3Control --- localstack/aws/api/s3control/__init__.py | 7 + localstack/aws/protocol/serializer.py | 8 +- localstack/aws/spec-patches.json | 22 +++ localstack/services/s3control/v2/models.py | 10 +- localstack/services/s3control/v2/provider.py | 107 +++++++++++-- .../aws/services/s3control/test_s3control.py | 144 +++++++++++++----- .../s3control/test_s3control.snapshot.json | 86 +++++++++++ 7 files changed, 322 insertions(+), 62 deletions(-) diff --git a/localstack/aws/api/s3control/__init__.py b/localstack/aws/api/s3control/__init__.py index 5b1bbe9bfefd6..dab5d9de00cb1 100644 --- a/localstack/aws/api/s3control/__init__.py +++ b/localstack/aws/api/s3control/__init__.py @@ -452,6 +452,13 @@ class TooManyTagsException(ServiceException): status_code: int = 400 +class NoSuchAccessPoint(ServiceException): + code: str = "NoSuchAccessPoint" + sender_fault: bool = False + status_code: int = 404 + AccessPointName: Optional[AccessPointName] + + class AbortIncompleteMultipartUpload(TypedDict, total=False): DaysAfterInitiation: Optional[DaysAfterInitiation] diff --git a/localstack/aws/protocol/serializer.py b/localstack/aws/protocol/serializer.py index 8e591eefbbcd8..8e385d285c258 100644 --- a/localstack/aws/protocol/serializer.py +++ b/localstack/aws/protocol/serializer.py @@ -1619,7 +1619,7 @@ def _serialize_error( "9Gjjt1m+cjU4OPvX9O9/8RuvnG41MRb/18Oux2o5H5MY7ISNTlXN+Dz9IG62/ILVxhAGI0qyPfg=" ) - self._add_additional_error_tags(vars(error), root, shape, mime_type) + self._add_additional_error_tags(vars(error), error_tag, shape, mime_type) p = self._encode_payload(self._node_to_string(root, mime_type)) print(f"{p=}") response.set_response(self._encode_payload(self._node_to_string(root, mime_type))) @@ -1629,10 +1629,6 @@ def _add_error_tags( ) -> None: super()._add_error_tags(error, error_tag, mime_type) - if hasattr(error, "AccountId"): - account_id_tag = ETree.SubElement(error_tag, "AccountId") - account_id_tag.text = error.AccountId - def _add_additional_error_tags( self, parameters: dict, node: ETree, shape: StructureShape, mime_type: str ): @@ -1642,7 +1638,7 @@ def _add_additional_error_tags( for member in shape.members: # XML protocols do not add modeled default fields to the root node # (tested for cloudfront, route53, cloudwatch, iam) - if member.lower() not in ["code", "message", "accountid"] and member in parameters: + if member.lower() not in ["code", "message"] and member in parameters: params[member] = parameters[member] # If there is an error shape with members which should be set, they need to be added to the node diff --git a/localstack/aws/spec-patches.json b/localstack/aws/spec-patches.json index f5e26e0b5c512..bbbd4e1d21082 100644 --- a/localstack/aws/spec-patches.json +++ b/localstack/aws/spec-patches.json @@ -1173,6 +1173,28 @@ "value": { "shape": "AccountId" } + }, + { + "op": "add", + "path": "/shapes/NoSuchAccessPoint", + "value": { + "type": "structure", + "members": { + "AccessPointName": { + "shape": "AccessPointName" + } + }, + "error": { + "httpStatusCode": 404 + }, + "documentation": "

The specified accesspoint does not exist

", + "exception": true + } + }, + { + "op": "add", + "path": "/operations/DeleteAccessPoint/http/responseCode", + "value": 204 } ] } diff --git a/localstack/services/s3control/v2/models.py b/localstack/services/s3control/v2/models.py index 6320b2f9e1535..eca7e4d756575 100644 --- a/localstack/services/s3control/v2/models.py +++ b/localstack/services/s3control/v2/models.py @@ -1,14 +1,22 @@ -from localstack.aws.api.s3control import PublicAccessBlockConfiguration +from localstack.aws.api.s3control import ( + AccessPointName, + GetAccessPointResult, + PublicAccessBlockConfiguration, +) from localstack.services.stores import ( AccountRegionBundle, BaseStore, CrossRegionAttribute, + LocalAttribute, ) class S3ControlStore(BaseStore): # buckets: dict[BucketName, S3Bucket] = CrossRegionAttribute(default=dict) public_access_block: PublicAccessBlockConfiguration = CrossRegionAttribute(default=dict) + access_points: dict[AccessPointName, GetAccessPointResult] = LocalAttribute( + default=dict + ) # TODO: check locality # access_point_alias: dict[Alias, BucketName] = CrossRegionAttribute(default=dict) # global_bucket_map: dict[BucketName, AccountId] = CrossAccountAttribute(default=dict) diff --git a/localstack/services/s3control/v2/provider.py b/localstack/services/s3control/v2/provider.py index e51c007667518..f4cf55e1c4755 100644 --- a/localstack/services/s3control/v2/provider.py +++ b/localstack/services/s3control/v2/provider.py @@ -1,5 +1,8 @@ +import datetime + from localstack.aws.api import CommonServiceException, RequestContext from localstack.aws.api.s3control import ( + AccessPoint, AccessPointName, AccountId, BucketName, @@ -10,7 +13,9 @@ GetPublicAccessBlockOutput, ListAccessPointsResult, MaxResults, + NetworkOrigin, NonEmptyMaxLength1024String, + NoSuchAccessPoint, NoSuchPublicAccessBlockConfiguration, Policy, PublicAccessBlockConfiguration, @@ -19,6 +24,9 @@ ) from localstack.services.s3.utils import validate_dict_fields from localstack.services.s3control.v2.models import S3ControlStore, s3control_stores +from localstack.utils.collections import select_from_typed_dict +from localstack.utils.strings import short_uid +from localstack.utils.urls import localstack_host class MalformedXML(CommonServiceException): @@ -34,6 +42,18 @@ def __init__(self, message=None): FAKE_HOST_ID = "9Gjjt1m+cjU4OPvX9O9/8RuvnG41MRb/18Oux2o5H5MY7ISNTlXN+Dz9IG62/ILVxhAGI0qyPfg=" +PUBLIC_ACCESS_BLOCK_FIELDS = { + "BlockPublicAcls", + "BlockPublicPolicy", + "IgnorePublicAcls", + "RestrictPublicBuckets", +} +DEFAULT_ENDPOINTS = { + "dualstack": f"s3-accesspoint.dualstack..{localstack_host()}", + "fips": f"s3-accesspoint-fips..{localstack_host()}", + "fips_dualstack": f"s3-accesspoint-fips.dualstack..{localstack_host()}", + "ipv4": f"s3-accesspoint..{localstack_host()}", +} class S3ControlProvider(S3ControlApi): @@ -56,17 +76,10 @@ def put_public_access_block( # the region does not matter, everything is global store = self.get_store(account_id, context.region) - public_access_block_fields = { - "BlockPublicAcls", - "BlockPublicPolicy", - "IgnorePublicAcls", - "RestrictPublicBuckets", - } - if not validate_dict_fields( public_access_block_configuration, required_fields=set(), - optional_fields=public_access_block_fields, + optional_fields=PUBLIC_ACCESS_BLOCK_FIELDS, ): raise MalformedXML() @@ -75,7 +88,7 @@ def put_public_access_block( "Must specify at least one configuration.", ) - for field in public_access_block_fields: + for field in PUBLIC_ACCESS_BLOCK_FIELDS: if public_access_block_configuration.get(field) is None: public_access_block_configuration[field] = False @@ -84,7 +97,7 @@ def put_public_access_block( def get_public_access_block( self, context: RequestContext, account_id: AccountId ) -> GetPublicAccessBlockOutput: - store = self.get_store(context.account_id, context.region) + store = self.get_store(account_id, context.region) if not store.public_access_block: raise NoSuchPublicAccessBlockConfiguration( "The public access block configuration was not found", @@ -94,7 +107,7 @@ def get_public_access_block( return GetPublicAccessBlockOutput(PublicAccessBlockConfiguration=store.public_access_block) def delete_public_access_block(self, context: RequestContext, account_id: AccountId) -> None: - store = self.get_store(context.account_id, context.region) + store = self.get_store(account_id, context.region) store.public_access_block = None def create_access_point( @@ -107,12 +120,63 @@ def create_access_point( public_access_block_configuration: PublicAccessBlockConfiguration = None, bucket_account_id: AccountId = None, ) -> CreateAccessPointResult: - pass + # TODO: support VpcConfiguration + # TODO: support PublicAccessBlockConfiguration + # TODO: check bucket_account_id + # TODO: check if Bucket exists? + # TODO: validate name, same as bucket with some more validation? + # TODO: check if endpoint name already exists + + # TODO: access point might be region only?? test it + store = self.get_store(account_id, context.region) + + # needs to be 32 long? maybe? + # TODO: add to map? + alias = f"{name}-{short_uid()}-s3alias" + + public_access_block_configuration = public_access_block_configuration or {} + for field in PUBLIC_ACCESS_BLOCK_FIELDS: + if public_access_block_configuration.get(field) is None: + public_access_block_configuration[field] = True + + regional_endpoints = { + t: endpoint.replace("", context.region) + for t, endpoint in DEFAULT_ENDPOINTS.items() + } + access_point_arn = f"arn:aws:s3:{context.region}:{account_id}:accesspoint/{name}" + + access_point = GetAccessPointResult( + Name=name, + Bucket=bucket, + NetworkOrigin=NetworkOrigin.VPC if vpc_configuration else NetworkOrigin.Internet, + PublicAccessBlockConfiguration=public_access_block_configuration, + CreationDate=datetime.datetime.now(tz=datetime.UTC), + Alias=alias, + AccessPointArn=access_point_arn, + Endpoints=regional_endpoints, + BucketAccountId=bucket_account_id or account_id, # TODO + ) + if vpc_configuration: + access_point["VpcConfiguration"] = vpc_configuration + + store.access_points[name] = access_point + + return CreateAccessPointResult( + AccessPointArn=access_point_arn, + Alias=alias, + ) def get_access_point( self, context: RequestContext, account_id: AccountId, name: AccessPointName ) -> GetAccessPointResult: - pass + store = self.get_store(account_id, context.region) + if not (access_point := store.access_points.get(name)): + raise NoSuchAccessPoint( + "The specified accesspoint does not exist", + AccessPointName=name, + ) + + return access_point def list_access_points( self, @@ -122,12 +186,25 @@ def list_access_points( next_token: NonEmptyMaxLength1024String = None, max_results: MaxResults = None, ) -> ListAccessPointsResult: - pass + # TODO: implement pagination + # TODO: implement filter with Bucket name + store = self.get_store(account_id, context.region) + + result = [] + for full_access_point in store.access_points.values(): + access_point: AccessPoint = select_from_typed_dict(AccessPoint, full_access_point) + result.append(access_point) + + return ListAccessPointsResult( + AccessPointList=result, + ) def delete_access_point( self, context: RequestContext, account_id: AccountId, name: AccessPointName ) -> None: - pass + store = self.get_store(account_id, context.region) + if not store.access_points.pop(name, None): + pass def put_access_point_policy( self, context: RequestContext, account_id: AccountId, name: AccessPointName, policy: Policy diff --git a/tests/aws/services/s3control/test_s3control.py b/tests/aws/services/s3control/test_s3control.py index db3cde25e466a..7ce1492ab8057 100644 --- a/tests/aws/services/s3control/test_s3control.py +++ b/tests/aws/services/s3control/test_s3control.py @@ -1,3 +1,5 @@ +import contextlib + import pytest from botocore.client import Config from botocore.exceptions import ClientError @@ -10,8 +12,10 @@ ) from localstack.testing.aws.util import is_aws_cloud from localstack.testing.pytest import markers +from localstack.utils.strings import short_uid +from localstack.utils.urls import localstack_host -remote_endpoint = config.external_service_url(protocol="https") +remote_endpoint = config.external_service_url(protocol="http") s3_control_endpoint = f"http://s3-control.{remote_endpoint.split('://')[1]}" @@ -39,57 +43,70 @@ def s3control_client(aws_client_factory, aws_client): return aws_client.s3control -@markers.aws.unknown -def test_lifecycle_public_access_block(s3control_client): - with pytest.raises(ClientError) as ce: - s3control_client.get_public_access_block(AccountId=TEST_AWS_ACCOUNT_ID) - assert ce.value.response["Error"]["Code"] == "NoSuchPublicAccessBlockConfiguration" - - access_block_config = { - "BlockPublicAcls": True, - "IgnorePublicAcls": True, - "BlockPublicPolicy": True, - "RestrictPublicBuckets": True, - } +@pytest.fixture +def s3control_create_access_point(s3control_client): + access_points = [] - put_response = s3control_client.put_public_access_block( - AccountId=TEST_AWS_ACCOUNT_ID, PublicAccessBlockConfiguration=access_block_config - ) + def _create_access_point(**kwargs): + resp = s3control_client.create_access_point(**kwargs) + access_points.append((kwargs["Name"], kwargs["AccountId"])) + return resp - assert put_response["ResponseMetadata"]["HTTPStatusCode"] == 200 + yield _create_access_point - get_response = s3control_client.get_public_access_block(AccountId=TEST_AWS_ACCOUNT_ID) - assert access_block_config == get_response["PublicAccessBlockConfiguration"] + for access_point_name, account_id in access_points: + with contextlib.suppress(ClientError): + s3control_client.delete_access_point(AccountId=account_id, Name=access_point_name) - s3control_client.delete_public_access_block(AccountId=TEST_AWS_ACCOUNT_ID) +class TestLegacyS3Control: + @markers.aws.unknown + def test_lifecycle_public_access_block(self, s3control_client): + with pytest.raises(ClientError) as ce: + s3control_client.get_public_access_block(AccountId=TEST_AWS_ACCOUNT_ID) + assert ce.value.response["Error"]["Code"] == "NoSuchPublicAccessBlockConfiguration" -@markers.aws.unknown -def test_public_access_block_validations(s3control_client): - with pytest.raises(ClientError) as error: - s3control_client.get_public_access_block(AccountId="111111111111") - assert error.value.response["Error"]["Code"] == "AccessDenied" + access_block_config = { + "BlockPublicAcls": True, + "IgnorePublicAcls": True, + "BlockPublicPolicy": True, + "RestrictPublicBuckets": True, + } - with pytest.raises(ClientError) as error: - s3control_client.put_public_access_block( - AccountId="111111111111", - PublicAccessBlockConfiguration={"BlockPublicAcls": True}, + put_response = s3control_client.put_public_access_block( + AccountId=TEST_AWS_ACCOUNT_ID, PublicAccessBlockConfiguration=access_block_config ) - assert error.value.response["Error"]["Code"] == "AccessDenied" - with pytest.raises(ClientError) as error: - s3control_client.put_public_access_block( - AccountId=TEST_AWS_ACCOUNT_ID, PublicAccessBlockConfiguration={} - ) - assert error.value.response["Error"]["Code"] == "InvalidRequest" + assert put_response["ResponseMetadata"]["HTTPStatusCode"] == 200 + + get_response = s3control_client.get_public_access_block(AccountId=TEST_AWS_ACCOUNT_ID) + assert access_block_config == get_response["PublicAccessBlockConfiguration"] + + s3control_client.delete_public_access_block(AccountId=TEST_AWS_ACCOUNT_ID) + + @markers.aws.unknown + @pytest.mark.skip(reason="Moto forces IAM use with the account id even when not enabled") + def test_public_access_block_validations(self, s3control_client): + with pytest.raises(ClientError) as error: + s3control_client.get_public_access_block(AccountId="111111111111") + assert error.value.response["Error"]["Code"] == "AccessDenied" + + with pytest.raises(ClientError) as error: + s3control_client.put_public_access_block( + AccountId="111111111111", + PublicAccessBlockConfiguration={"BlockPublicAcls": True}, + ) + assert error.value.response["Error"]["Code"] == "AccessDenied" + + with pytest.raises(ClientError) as error: + s3control_client.put_public_access_block( + AccountId=TEST_AWS_ACCOUNT_ID, PublicAccessBlockConfiguration={} + ) + assert error.value.response["Error"]["Code"] == "InvalidRequest" class TestS3ControlPublicAccessBlock: @markers.aws.validated - # @pytest.mark.xfail( - # condition=config.LEGACY_V2_S3_PROVIDER, - # reason="Moto implementation does not have default public access block", - # ) def test_crud_public_access_block(self, s3control_client, account_id, snapshot): with pytest.raises(ClientError) as e: s3control_client.get_public_access_block(AccountId=account_id) @@ -147,4 +164,51 @@ def test_empty_public_access_block(self, aws_client_factory, account_id, snapsho class TestS3ControlAccessPoint: - pass + @markers.aws.validated + def test_access_point_lifecycle( + self, s3control_client, s3control_create_access_point, account_id, s3_bucket, snapshot + ): + snapshot.add_transformers_list( + [ + snapshot.transform.key_value("Name"), + snapshot.transform.key_value("Bucket"), + snapshot.transform.key_value("Bucket"), + snapshot.transform.regex("amazonaws.com", ""), + snapshot.transform.regex(localstack_host().host_and_port(), ""), + ] + ) + + list_access_points = s3control_client.list_access_points(AccountId=account_id) + snapshot.match("list-access-points-start", list_access_points) + + ap_name = short_uid() + create_access_point = s3control_create_access_point( + AccountId=account_id, Name=ap_name, Bucket=s3_bucket + ) + + alias_random_part = create_access_point["Alias"].split("-")[1] + # TODO: assert length + snapshot.add_transformer( + snapshot.transform.regex(alias_random_part, replacement="") + ) + + snapshot.match("create-access-point", create_access_point) + + get_access_point = s3control_client.get_access_point(AccountId=account_id, Name=ap_name) + snapshot.match("get-access-point", get_access_point) + # todo: assert CreationDate format + + list_access_points = s3control_client.list_access_points(AccountId=account_id) + snapshot.match("list-access-points-after-create", list_access_points) + + delete_access_point = s3control_client.delete_access_point( + AccountId=account_id, Name=ap_name + ) + snapshot.match("delete-access-point", delete_access_point) + + list_access_points = s3control_client.list_access_points(AccountId=account_id) + snapshot.match("list-access-points-after-delete", list_access_points) + + with pytest.raises(ClientError) as e: + s3control_client.get_access_point(AccountId=account_id, Name=ap_name) + snapshot.match("get-delete-access-point", e.value.response) diff --git a/tests/aws/services/s3control/test_s3control.snapshot.json b/tests/aws/services/s3control/test_s3control.snapshot.json index 9f55b7a16ff4b..efc60b18a6786 100644 --- a/tests/aws/services/s3control/test_s3control.snapshot.json +++ b/tests/aws/services/s3control/test_s3control.snapshot.json @@ -75,5 +75,91 @@ } } } + }, + "tests/aws/services/s3control/test_s3control.py::TestS3ControlAccessPoint::test_access_point_lifecycle": { + "recorded-date": "24-11-2023, 22:23:01", + "recorded-content": { + "list-access-points-start": { + "AccessPointList": [], + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "create-access-point": { + "AccessPointArn": "arn:aws:s3::111111111111:accesspoint/", + "Alias": "--s3alias", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "get-access-point": { + "AccessPointArn": "arn:aws:s3::111111111111:accesspoint/", + "Alias": "--s3alias", + "Bucket": "", + "BucketAccountId": "111111111111", + "CreationDate": "datetime", + "Endpoints": { + "dualstack": "s3-accesspoint.dualstack..", + "fips": "s3-accesspoint-fips..", + "fips_dualstack": "s3-accesspoint-fips.dualstack..", + "ipv4": "s3-accesspoint.." + }, + "Name": "", + "NetworkOrigin": "Internet", + "PublicAccessBlockConfiguration": { + "BlockPublicAcls": true, + "BlockPublicPolicy": true, + "IgnorePublicAcls": true, + "RestrictPublicBuckets": true + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "list-access-points-after-create": { + "AccessPointList": [ + { + "AccessPointArn": "arn:aws:s3::111111111111:accesspoint/", + "Alias": "--s3alias", + "Bucket": "", + "BucketAccountId": "111111111111", + "Name": "", + "NetworkOrigin": "Internet" + } + ], + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "delete-access-point": { + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 204 + } + }, + "list-access-points-after-delete": { + "AccessPointList": [], + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "get-delete-access-point": { + "Error": { + "AccessPointName": "", + "Code": "NoSuchAccessPoint", + "Message": "The specified accesspoint does not exist" + }, + "HostId": "host-id", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 404 + } + } + } } } From f6e2fa0e0bfdfe405abf51bdccd7162a134c28c6 Mon Sep 17 00:00:00 2001 From: Benjamin Simon Date: Tue, 28 Nov 2023 23:19:49 +0200 Subject: [PATCH 3/3] add validation for AccessPoints --- localstack/aws/api/s3control/__init__.py | 8 + localstack/aws/protocol/serializer.py | 40 +-- localstack/aws/spec-patches.json | 21 ++ localstack/services/s3control/v2/models.py | 6 +- localstack/services/s3control/v2/provider.py | 67 ++++- .../aws/services/s3control/test_s3control.py | 197 ++++++++++++-- .../s3control/test_s3control.snapshot.json | 251 +++++++++++++++++- 7 files changed, 525 insertions(+), 65 deletions(-) diff --git a/localstack/aws/api/s3control/__init__.py b/localstack/aws/api/s3control/__init__.py index dab5d9de00cb1..6e6c876feb774 100644 --- a/localstack/aws/api/s3control/__init__.py +++ b/localstack/aws/api/s3control/__init__.py @@ -93,6 +93,7 @@ TrafficDialPercentage = int VpcId = str HostId = str +URI = str class AsyncOperationName(str): @@ -459,6 +460,13 @@ class NoSuchAccessPoint(ServiceException): AccessPointName: Optional[AccessPointName] +class InvalidURI(ServiceException): + code: str = "InvalidURI" + sender_fault: bool = False + status_code: int = 400 + URI: Optional[URI] + + class AbortIncompleteMultipartUpload(TypedDict, total=False): DaysAfterInitiation: Optional[DaysAfterInitiation] diff --git a/localstack/aws/protocol/serializer.py b/localstack/aws/protocol/serializer.py index 8e385d285c258..0291df5dace75 100644 --- a/localstack/aws/protocol/serializer.py +++ b/localstack/aws/protocol/serializer.py @@ -619,6 +619,7 @@ def _serialize_error( request_id_element.text = request_id self._add_additional_error_tags(vars(error), root, shape, mime_type) + response.set_response(self._encode_payload(self._node_to_string(root, mime_type))) def _add_error_tags( @@ -626,7 +627,6 @@ def _add_error_tags( ) -> None: code_tag = ETree.SubElement(error_tag, "Code") code_tag.text = error.code - message = self._get_error_message(error) if message: self._default_serialize(error_tag, message, None, "Message", mime_type) @@ -1610,6 +1610,8 @@ def _serialize_error( root = ETree.Element("ErrorResponse", attr) error_tag = ETree.SubElement(root, "Error") + # the difference for S3Control is here: it adds additional error tags inside the Error tags, unlike other + # rest-xml services self._add_error_tags(error, error_tag, mime_type) request_id_element = ETree.SubElement(root, "RequestId") request_id_element.text = request_id @@ -1620,37 +1622,15 @@ def _serialize_error( ) self._add_additional_error_tags(vars(error), error_tag, shape, mime_type) - p = self._encode_payload(self._node_to_string(root, mime_type)) - print(f"{p=}") response.set_response(self._encode_payload(self._node_to_string(root, mime_type))) - def _add_error_tags( - self, error: ServiceException, error_tag: ETree.Element, mime_type: str - ) -> None: - super()._add_error_tags(error, error_tag, mime_type) - - def _add_additional_error_tags( - self, parameters: dict, node: ETree, shape: StructureShape, mime_type: str - ): - if shape: - params = {} - # TODO add a possibility to serialize simple non-modelled errors (like S3 NoSuchBucket#BucketName) - for member in shape.members: - # XML protocols do not add modeled default fields to the root node - # (tested for cloudfront, route53, cloudwatch, iam) - if member.lower() not in ["code", "message"] and member in parameters: - params[member] = parameters[member] - - # If there is an error shape with members which should be set, they need to be added to the node - if params: - # Serialize the remaining params - root_name = shape.serialization.get("name", shape.name) - pseudo_root = ETree.Element("") - self._serialize(shape, params, pseudo_root, root_name, mime_type) - real_root = list(pseudo_root)[0] - # Add the child elements to the already created root error element - for child in list(real_root): - node.append(child) + @staticmethod + def _timestamp_iso8601(value: datetime) -> str: + """ + This is very specific to S3, S3 returns an ISO8601 timestamp but with milliseconds always set to 000 + Some SDKs are very picky about the length + """ + return value.strftime("%Y-%m-%dT%H:%M:%S.000Z") class SqsQueryResponseSerializer(QueryResponseSerializer): diff --git a/localstack/aws/spec-patches.json b/localstack/aws/spec-patches.json index bbbd4e1d21082..241c97cec3c33 100644 --- a/localstack/aws/spec-patches.json +++ b/localstack/aws/spec-patches.json @@ -1195,6 +1195,27 @@ "op": "add", "path": "/operations/DeleteAccessPoint/http/responseCode", "value": 204 + }, + { + "op": "add", + "path": "/shapes/URI", + "value": { + "type": "string" + } + }, + { + "op": "add", + "path": "/shapes/InvalidURI", + "value": { + "type": "structure", + "members": { + "URI": { + "shape": "URI" + } + }, + "documentation": "

Couldn't parse the specified URI.

", + "exception": true + } } ] } diff --git a/localstack/services/s3control/v2/models.py b/localstack/services/s3control/v2/models.py index eca7e4d756575..7bf0b7978e24d 100644 --- a/localstack/services/s3control/v2/models.py +++ b/localstack/services/s3control/v2/models.py @@ -1,11 +1,14 @@ from localstack.aws.api.s3control import ( AccessPointName, + Alias, + BucketName, GetAccessPointResult, PublicAccessBlockConfiguration, ) from localstack.services.stores import ( AccountRegionBundle, BaseStore, + CrossAccountAttribute, CrossRegionAttribute, LocalAttribute, ) @@ -17,7 +20,8 @@ class S3ControlStore(BaseStore): access_points: dict[AccessPointName, GetAccessPointResult] = LocalAttribute( default=dict ) # TODO: check locality - # access_point_alias: dict[Alias, BucketName] = CrossRegionAttribute(default=dict) + # TODO: check for accross-region accesses + access_point_alias: dict[Alias, BucketName] = CrossAccountAttribute(default=dict) # global_bucket_map: dict[BucketName, AccountId] = CrossAccountAttribute(default=dict) diff --git a/localstack/services/s3control/v2/provider.py b/localstack/services/s3control/v2/provider.py index f4cf55e1c4755..59c702ff7f5c4 100644 --- a/localstack/services/s3control/v2/provider.py +++ b/localstack/services/s3control/v2/provider.py @@ -1,4 +1,9 @@ import datetime +import re +from random import choices +from string import ascii_lowercase, digits + +from botocore.exceptions import ClientError from localstack.aws.api import CommonServiceException, RequestContext from localstack.aws.api.s3control import ( @@ -11,6 +16,7 @@ GetAccessPointPolicyStatusResult, GetAccessPointResult, GetPublicAccessBlockOutput, + InvalidURI, ListAccessPointsResult, MaxResults, NetworkOrigin, @@ -22,10 +28,10 @@ S3ControlApi, VpcConfiguration, ) +from localstack.aws.connect import connect_to from localstack.services.s3.utils import validate_dict_fields from localstack.services.s3control.v2.models import S3ControlStore, s3control_stores from localstack.utils.collections import select_from_typed_dict -from localstack.utils.strings import short_uid from localstack.utils.urls import localstack_host @@ -55,11 +61,15 @@ def __init__(self, message=None): "ipv4": f"s3-accesspoint..{localstack_host()}", } +ACCESS_POINT_REGEX = re.compile(r"^((?!xn--)(?!.*-s3alias$)[a-z0-9][a-z0-9-]{1,48}[a-z0-9])$") + class S3ControlProvider(S3ControlApi): """ Lots of S3 Control API methods are related to S3 Outposts (S3 in your own datacenter) These are not implemented in this provider + Access Points limitations: + - https://docs.aws.amazon.com/AmazonS3/latest/userguide/access-points-restrictions-limitations.html """ @staticmethod @@ -120,24 +130,52 @@ def create_access_point( public_access_block_configuration: PublicAccessBlockConfiguration = None, bucket_account_id: AccountId = None, ) -> CreateAccessPointResult: + # Access Point naming rules, see: + # https://docs.aws.amazon.com/AmazonS3/latest/userguide/creating-access-points.html#access-points-names + # TODO: support VpcConfiguration # TODO: support PublicAccessBlockConfiguration # TODO: check bucket_account_id - # TODO: check if Bucket exists? - # TODO: validate name, same as bucket with some more validation? - # TODO: check if endpoint name already exists # TODO: access point might be region only?? test it store = self.get_store(account_id, context.region) + if not ACCESS_POINT_REGEX.match(name): + if len(name) < 3 or len(name) > 50 or "_" in name or name.isupper(): + raise InvalidURI( + "Couldn't parse the specified URI.", + URI=f"accesspoint/{name}", + ) + + raise InvalidRequest("Your Amazon S3 AccessPoint name is invalid") + + if name in store.access_points: + # TODO: implement additional checks if the account id is different than the access point + raise CommonServiceException( + "AccessPointAlreadyOwnedByYou", + "Your previous request to create the named accesspoint succeeded and you already own it.", + status_code=409, + ) - # needs to be 32 long? maybe? - # TODO: add to map? - alias = f"{name}-{short_uid()}-s3alias" - + # TODO: what are the permissions to needed to create an AccessPoint to a bucket? + try: + connect_to(region_name=context.region).s3.head_bucket(Bucket=bucket) + except ClientError as e: + if e.response.get("Error", {}).get("Code") == "404": + raise InvalidRequest( + "Amazon S3 AccessPoint can only be created for existing bucket", + ) + # TODO: find AccessDenied exception? + raise + + alias = create_random_alias(name) + + # if the PublicAccessBlockConfiguration is not set, then every field default to True + # else, it's set to False + is_pabc_none = public_access_block_configuration is None public_access_block_configuration = public_access_block_configuration or {} for field in PUBLIC_ACCESS_BLOCK_FIELDS: if public_access_block_configuration.get(field) is None: - public_access_block_configuration[field] = True + public_access_block_configuration[field] = is_pabc_none regional_endpoints = { t: endpoint.replace("", context.region) @@ -160,6 +198,7 @@ def create_access_point( access_point["VpcConfiguration"] = vpc_configuration store.access_points[name] = access_point + store.access_point_alias[alias] = bucket return CreateAccessPointResult( AccessPointArn=access_point_arn, @@ -188,6 +227,7 @@ def list_access_points( ) -> ListAccessPointsResult: # TODO: implement pagination # TODO: implement filter with Bucket name + # TODO: implement ordering store = self.get_store(account_id, context.region) result = [] @@ -204,7 +244,10 @@ def delete_access_point( ) -> None: store = self.get_store(account_id, context.region) if not store.access_points.pop(name, None): - pass + raise NoSuchAccessPoint( + "The specified accesspoint does not exist", + AccessPointName=name, + ) def put_access_point_policy( self, context: RequestContext, account_id: AccountId, name: AccessPointName, policy: Policy @@ -225,3 +268,7 @@ def get_access_point_policy_status( self, context: RequestContext, account_id: AccountId, name: AccessPointName ) -> GetAccessPointPolicyStatusResult: pass + + +def create_random_alias(name: str) -> str: + return f"{name}-{''.join(choices(ascii_lowercase + digits, k=34))}-s3alias" diff --git a/tests/aws/services/s3control/test_s3control.py b/tests/aws/services/s3control/test_s3control.py index 7ce1492ab8057..21c4ee478aa64 100644 --- a/tests/aws/services/s3control/test_s3control.py +++ b/tests/aws/services/s3control/test_s3control.py @@ -4,7 +4,6 @@ from botocore.client import Config from botocore.exceptions import ClientError -from localstack import config from localstack.constants import ( TEST_AWS_ACCESS_KEY_ID, TEST_AWS_ACCOUNT_ID, @@ -15,14 +14,22 @@ from localstack.utils.strings import short_uid from localstack.utils.urls import localstack_host -remote_endpoint = config.external_service_url(protocol="http") -s3_control_endpoint = f"http://s3-control.{remote_endpoint.split('://')[1]}" +s3_control_endpoint = f"http://s3-control.{localstack_host()}" @pytest.fixture(autouse=True) def s3control_snapshot(snapshot): snapshot.add_transformers_list( - [snapshot.transform.key_value("HostId", reference_replacement=False)] + [ + snapshot.transform.key_value("HostId", reference_replacement=False), + snapshot.transform.key_value("Name"), + snapshot.transform.key_value("Bucket"), + snapshot.transform.regex("amazonaws.com", ""), + snapshot.transform.regex(localstack_host().host_and_port(), ""), + snapshot.transform.regex( + '([a-z0-9]{34})(?=.*-s3alias")', replacement="" + ), + ] ) @@ -43,6 +50,21 @@ def s3control_client(aws_client_factory, aws_client): return aws_client.s3control +@pytest.fixture +def s3control_client_no_validation(aws_client_factory): + if not is_aws_cloud(): + s3control_client = aws_client_factory( + config=Config(parameter_validation=False), + aws_access_key_id=TEST_AWS_ACCESS_KEY_ID, + aws_secret_access_key=TEST_AWS_SECRET_ACCESS_KEY, + endpoint_url=s3_control_endpoint, + ).s3control + else: + s3control_client = aws_client_factory(config=Config(parameter_validation=False)).s3control + + return s3control_client + + @pytest.fixture def s3control_create_access_point(s3control_client): access_points = [] @@ -140,22 +162,11 @@ def test_crud_public_access_block(self, s3control_client, account_id, snapshot): snapshot.match("idempotent-delete-public-access-block", delete_public_access_block) @markers.aws.validated - def test_empty_public_access_block(self, aws_client_factory, account_id, snapshot): + def test_empty_public_access_block(self, s3control_client_no_validation, account_id, snapshot): # we need to disable validation for this test - if not is_aws_cloud(): - s3control_client = aws_client_factory( - config=Config(parameter_validation=False), - aws_access_key_id=TEST_AWS_ACCESS_KEY_ID, - aws_secret_access_key=TEST_AWS_SECRET_ACCESS_KEY, - endpoint_url=s3_control_endpoint, - ).s3control - else: - s3control_client = aws_client_factory( - config=Config(parameter_validation=False) - ).s3control with pytest.raises(ClientError) as e: - s3control_client.put_public_access_block( + s3control_client_no_validation.put_public_access_block( AccountId=account_id, PublicAccessBlockConfiguration={}, ) @@ -172,7 +183,6 @@ def test_access_point_lifecycle( [ snapshot.transform.key_value("Name"), snapshot.transform.key_value("Bucket"), - snapshot.transform.key_value("Bucket"), snapshot.transform.regex("amazonaws.com", ""), snapshot.transform.regex(localstack_host().host_and_port(), ""), ] @@ -187,16 +197,12 @@ def test_access_point_lifecycle( ) alias_random_part = create_access_point["Alias"].split("-")[1] - # TODO: assert length - snapshot.add_transformer( - snapshot.transform.regex(alias_random_part, replacement="") - ) + assert len(alias_random_part) == 34 snapshot.match("create-access-point", create_access_point) get_access_point = s3control_client.get_access_point(AccountId=account_id, Name=ap_name) snapshot.match("get-access-point", get_access_point) - # todo: assert CreationDate format list_access_points = s3control_client.list_access_points(AccountId=account_id) snapshot.match("list-access-points-after-create", list_access_points) @@ -212,3 +218,148 @@ def test_access_point_lifecycle( with pytest.raises(ClientError) as e: s3control_client.get_access_point(AccountId=account_id, Name=ap_name) snapshot.match("get-delete-access-point", e.value.response) + + with pytest.raises(ClientError) as e: + s3control_client.delete_access_point(AccountId=account_id, Name=ap_name) + snapshot.match("delete-already-deleted-access-point", e.value.response) + + @markers.aws.validated + def test_access_point_bucket_not_exists( + self, s3control_create_access_point, account_id, snapshot + ): + ap_name = short_uid() + with pytest.raises(ClientError) as e: + s3control_create_access_point( + AccountId=account_id, + Name=ap_name, + Bucket=f"fake-bucket-{short_uid()}-{short_uid()}", + ) + snapshot.match("access-point-bucket-not-exists", e.value.response) + + @markers.aws.validated + def test_access_point_name_validation( + self, s3control_client_no_validation, account_id, snapshot, s3_bucket + ): + # not using parametrization because that would be a lot of snapshot. + # only validate the first one + wrong_name = "xn--test-alias" + wrong_names = [ + "-hyphen-start", + "cannot-end-s3alias", + "cannot-have.dot", + ] + + with pytest.raises(ClientError) as e: + s3control_client_no_validation.create_access_point( + AccountId=account_id, + Name=wrong_name, + Bucket=s3_bucket, + ) + snapshot.match("access-point-wrong-naming", e.value.response) + + for name in wrong_names: + with pytest.raises(ClientError) as e: + s3control_client_no_validation.create_access_point( + AccountId=account_id, + Name=name, + Bucket=s3_bucket, + ) + assert e.match("Your Amazon S3 AccessPoint name is invalid"), (name, e.value.response) + + # error is different for too short of a name + with pytest.raises(ClientError) as e: + s3control_client_no_validation.create_access_point( + AccountId=account_id, + Name="sa", + Bucket=s3_bucket, + ) + snapshot.match("access-point-name-too-short", e.value.response) + + uri_error_names = [ + "a" * 51, + "WRONG-casing", + "cannot-have_underscore", + ] + for name in uri_error_names: + with pytest.raises(ClientError) as e: + s3control_client_no_validation.create_access_point( + AccountId=account_id, + Name="a" * 51, + Bucket=s3_bucket, + ) + assert e.match("InvalidURI"), (name, e.value.response) + + @markers.aws.validated + def test_access_point_already_exists( + self, s3control_create_access_point, s3_bucket, account_id, snapshot + ): + ap_name = short_uid() + s3control_create_access_point(AccountId=account_id, Name=ap_name, Bucket=s3_bucket) + with pytest.raises(ClientError) as e: + s3control_create_access_point(AccountId=account_id, Name=ap_name, Bucket=s3_bucket) + snapshot.match("access-point-already-exists", e.value.response) + + @markers.aws.validated + def test_access_point_vpc_config( + self, s3control_create_access_point, s3control_client, account_id, snapshot, s3_bucket + ): + pass + + @markers.aws.validated + def test_access_point_public_access_block_configuration( + self, s3control_client, s3control_create_access_point, account_id, snapshot, s3_bucket + ): + # set a letter in the name for ordering + ap_name_1 = f"a{short_uid()}" + response = s3control_create_access_point( + AccountId=account_id, + Name=ap_name_1, + Bucket=s3_bucket, + PublicAccessBlockConfiguration={}, + ) + snapshot.match("put-ap-empty-pabc", response) + get_ap = s3control_client.get_access_point(AccountId=account_id, Name=ap_name_1) + snapshot.match("get-ap-empty-pabc", get_ap) + + ap_name_2 = f"b{short_uid()}" + response = s3control_create_access_point( + AccountId=account_id, + Name=ap_name_2, + Bucket=s3_bucket, + PublicAccessBlockConfiguration={"BlockPublicAcls": False}, + ) + snapshot.match("put-ap-partial-pabc", response) + get_ap = s3control_client.get_access_point(AccountId=account_id, Name=ap_name_2) + snapshot.match("get-ap-partial-pabc", get_ap) + + ap_name_3 = f"c{short_uid()}" + response = s3control_create_access_point( + AccountId=account_id, + Name=ap_name_3, + Bucket=s3_bucket, + PublicAccessBlockConfiguration={"BlockPublicAcls": True}, + ) + snapshot.match("put-ap-partial-true-pabc", response) + get_ap = s3control_client.get_access_point(AccountId=account_id, Name=ap_name_3) + snapshot.match("get-ap-partial-true-pabc", get_ap) + + ap_name_4 = f"d{short_uid()}" + response = s3control_create_access_point( + AccountId=account_id, + Name=ap_name_4, + Bucket=s3_bucket, + ) + snapshot.match("put-ap-pabc-not-set", response) + get_ap = s3control_client.get_access_point(AccountId=account_id, Name=ap_name_4) + snapshot.match("get-ap-pabc-not-set", get_ap) + + list_access_points = s3control_client.list_access_points(AccountId=account_id) + snapshot.match("list-access-points", list_access_points) + + @markers.aws.validated + def test_access_point_regions(self): + pass + + @markers.aws.validated + def test_access_point_pagination(self): + pass diff --git a/tests/aws/services/s3control/test_s3control.snapshot.json b/tests/aws/services/s3control/test_s3control.snapshot.json index efc60b18a6786..b64b4038fc072 100644 --- a/tests/aws/services/s3control/test_s3control.snapshot.json +++ b/tests/aws/services/s3control/test_s3control.snapshot.json @@ -77,7 +77,7 @@ } }, "tests/aws/services/s3control/test_s3control.py::TestS3ControlAccessPoint::test_access_point_lifecycle": { - "recorded-date": "24-11-2023, 22:23:01", + "recorded-date": "28-11-2023, 22:45:37", "recorded-content": { "list-access-points-start": { "AccessPointList": [], @@ -159,6 +159,255 @@ "HTTPHeaders": {}, "HTTPStatusCode": 404 } + }, + "delete-already-deleted-access-point": { + "Error": { + "AccessPointName": "", + "Code": "NoSuchAccessPoint", + "Message": "The specified accesspoint does not exist" + }, + "HostId": "host-id", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 404 + } + } + } + }, + "tests/aws/services/s3control/test_s3control.py::TestS3ControlAccessPoint::test_access_point_bucket_not_exists": { + "recorded-date": "28-11-2023, 21:39:20", + "recorded-content": { + "access-point-bucket-not-exists": { + "Error": { + "Code": "InvalidRequest", + "Message": "Amazon S3 AccessPoint can only be created for existing bucket" + }, + "HostId": "host-id", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 400 + } + } + } + }, + "tests/aws/services/s3control/test_s3control.py::TestS3ControlAccessPoint::test_access_point_name_validation": { + "recorded-date": "28-11-2023, 22:16:51", + "recorded-content": { + "access-point-wrong-naming": { + "Error": { + "Code": "InvalidRequest", + "Message": "Your Amazon S3 AccessPoint name is invalid" + }, + "HostId": "host-id", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 400 + } + }, + "access-point-name-too-short": { + "Error": { + "Code": "InvalidURI", + "Message": "Couldn't parse the specified URI.", + "URI": "accesspoint/sa" + }, + "HostId": "host-id", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 400 + } + } + } + }, + "tests/aws/services/s3control/test_s3control.py::TestS3ControlAccessPoint::test_access_point_already_exists": { + "recorded-date": "28-11-2023, 22:31:38", + "recorded-content": { + "access-point-already-exists": { + "Error": { + "Code": "AccessPointAlreadyOwnedByYou", + "Message": "Your previous request to create the named accesspoint succeeded and you already own it." + }, + "HostId": "host-id", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 409 + } + } + } + }, + "tests/aws/services/s3control/test_s3control.py::TestS3ControlAccessPoint::test_access_point_public_access_block_configuration": { + "recorded-date": "28-11-2023, 23:15:08", + "recorded-content": { + "put-ap-empty-pabc": { + "AccessPointArn": "arn:aws:s3::111111111111:accesspoint/", + "Alias": "--s3alias", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "get-ap-empty-pabc": { + "AccessPointArn": "arn:aws:s3::111111111111:accesspoint/", + "Alias": "--s3alias", + "Bucket": "", + "BucketAccountId": "111111111111", + "CreationDate": "datetime", + "Endpoints": { + "dualstack": "s3-accesspoint.dualstack..", + "fips": "s3-accesspoint-fips..", + "fips_dualstack": "s3-accesspoint-fips.dualstack..", + "ipv4": "s3-accesspoint.." + }, + "Name": "", + "NetworkOrigin": "Internet", + "PublicAccessBlockConfiguration": { + "BlockPublicAcls": false, + "BlockPublicPolicy": false, + "IgnorePublicAcls": false, + "RestrictPublicBuckets": false + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "put-ap-partial-pabc": { + "AccessPointArn": "arn:aws:s3::111111111111:accesspoint/", + "Alias": "--s3alias", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "get-ap-partial-pabc": { + "AccessPointArn": "arn:aws:s3::111111111111:accesspoint/", + "Alias": "--s3alias", + "Bucket": "", + "BucketAccountId": "111111111111", + "CreationDate": "datetime", + "Endpoints": { + "dualstack": "s3-accesspoint.dualstack..", + "fips": "s3-accesspoint-fips..", + "fips_dualstack": "s3-accesspoint-fips.dualstack..", + "ipv4": "s3-accesspoint.." + }, + "Name": "", + "NetworkOrigin": "Internet", + "PublicAccessBlockConfiguration": { + "BlockPublicAcls": false, + "BlockPublicPolicy": false, + "IgnorePublicAcls": false, + "RestrictPublicBuckets": false + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "put-ap-partial-true-pabc": { + "AccessPointArn": "arn:aws:s3::111111111111:accesspoint/", + "Alias": "--s3alias", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "get-ap-partial-true-pabc": { + "AccessPointArn": "arn:aws:s3::111111111111:accesspoint/", + "Alias": "--s3alias", + "Bucket": "", + "BucketAccountId": "111111111111", + "CreationDate": "datetime", + "Endpoints": { + "dualstack": "s3-accesspoint.dualstack..", + "fips": "s3-accesspoint-fips..", + "fips_dualstack": "s3-accesspoint-fips.dualstack..", + "ipv4": "s3-accesspoint.." + }, + "Name": "", + "NetworkOrigin": "Internet", + "PublicAccessBlockConfiguration": { + "BlockPublicAcls": true, + "BlockPublicPolicy": false, + "IgnorePublicAcls": false, + "RestrictPublicBuckets": false + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "put-ap-pabc-not-set": { + "AccessPointArn": "arn:aws:s3::111111111111:accesspoint/", + "Alias": "--s3alias", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "get-ap-pabc-not-set": { + "AccessPointArn": "arn:aws:s3::111111111111:accesspoint/", + "Alias": "--s3alias", + "Bucket": "", + "BucketAccountId": "111111111111", + "CreationDate": "datetime", + "Endpoints": { + "dualstack": "s3-accesspoint.dualstack..", + "fips": "s3-accesspoint-fips..", + "fips_dualstack": "s3-accesspoint-fips.dualstack..", + "ipv4": "s3-accesspoint.." + }, + "Name": "", + "NetworkOrigin": "Internet", + "PublicAccessBlockConfiguration": { + "BlockPublicAcls": true, + "BlockPublicPolicy": true, + "IgnorePublicAcls": true, + "RestrictPublicBuckets": true + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "list-access-points": { + "AccessPointList": [ + { + "AccessPointArn": "arn:aws:s3::111111111111:accesspoint/", + "Alias": "--s3alias", + "Bucket": "", + "BucketAccountId": "111111111111", + "Name": "", + "NetworkOrigin": "Internet" + }, + { + "AccessPointArn": "arn:aws:s3::111111111111:accesspoint/", + "Alias": "--s3alias", + "Bucket": "", + "BucketAccountId": "111111111111", + "Name": "", + "NetworkOrigin": "Internet" + }, + { + "AccessPointArn": "arn:aws:s3::111111111111:accesspoint/", + "Alias": "--s3alias", + "Bucket": "", + "BucketAccountId": "111111111111", + "Name": "", + "NetworkOrigin": "Internet" + }, + { + "AccessPointArn": "arn:aws:s3::111111111111:accesspoint/", + "Alias": "--s3alias", + "Bucket": "", + "BucketAccountId": "111111111111", + "Name": "", + "NetworkOrigin": "Internet" + } + ], + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } } } }