From 09dd63bc568d1529811e16dd80d2a16bd9b70607 Mon Sep 17 00:00:00 2001 From: Pedro Serrano Date: Tue, 26 May 2026 15:44:50 +0200 Subject: [PATCH 1/3] add connection files to recognited files --- pkg/s3.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/s3.go b/pkg/s3.go index 3e6ba54b..920a0c04 100644 --- a/pkg/s3.go +++ b/pkg/s3.go @@ -62,6 +62,10 @@ var ( // source: https://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-access-logs.html#access-log-file-format // format: bucket[/prefix]/AWSLogs/aws-account-id/elasticloadbalancing/region/yyyy/mm/dd/aws-account-id_elasticloadbalancing_region_app.load-balancer-id_end-time_ip-address_random-string.log.gz // example: my-bucket/AWSLogs/123456789012/elasticloadbalancing/us-east-1/2022/01/24/123456789012_elasticloadbalancing_us-east-1_app.my-loadbalancer.b13ea9d19f16d015_20220124T0000Z_0.0.0.0_2et2e1mx.log.gz + // AWS Application Load Balancers Connection Logs + // source: https://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-connection-logs.html#connection-log-entry-format + // format: bucket[/prefix]/AWSLogs/aws-account-id/elasticloadbalancing/region/yyyy/mm/dd/conn_log_aws-account-id_elasticloadbalancing_region_app.load-balancer-id_end-time_ip-address_random-string.log.gz + // example: my-bucket/AWSLogs/123456789012/elasticloadbalancing/us-east-1/2022/01/24/conn_log_123456789012_elasticloadbalancing_us-east-1_app.my-loadbalancer_20220124T0000Z_0.0.0.0_2et2e1mx.log.gz // AWS Network Load Balancers // source: https://docs.aws.amazon.com/elasticloadbalancing/latest/network/load-balancer-access-logs.html#access-log-file-format // format: bucket[/prefix]/AWSLogs/aws-account-id/elasticloadbalancing/region/yyyy/mm/dd/aws-account-id_elasticloadbalancing_region_net.load-balancer-id_end-time_random-string.log.gz @@ -92,7 +96,7 @@ var ( // source: https://docs.aws.amazon.com/AmazonS3/latest/userguide/ServerLogs.html // format: aws-account-id/region/bucket-name/year/month/day/timestamp-hash // example: 123456789012/us-west-2/amzn-s3-demo-source-bucket/2023/03/01/2023-03-01-21-32-16-E568B2907131C0C0 - defaultFilenameRegex = regexp.MustCompile(`AWSLogs\/(?P\d+)\/(?P[a-zA-Z0-9_\-]+)\/(?P[\w-]+)\/(?P\d+)\/(?P\d+)\/(?P\d+)\/(?:health_check_log_)?\d+\_(?:elasticloadbalancing|vpcflowlogs)_(?:\w+-\w+-(?:\w+-)?\d)_(?:(?Papp|net)\.*?)?(?P[a-zA-Z0-9\-]+)`) + defaultFilenameRegex = regexp.MustCompile(`AWSLogs\/(?:conn_log_)?(?P\d+)\/(?P[a-zA-Z0-9_\-]+)\/(?P[\w-]+)\/(?P\d+)\/(?P\d+)\/(?P\d+)\/(?:health_check_log_)?(?:conn_log_)?\d+\_(?:elasticloadbalancing|vpcflowlogs)_(?:\w+-\w+-(?:\w+-)?\d)_(?:(?Papp|net)\.*?)?(?P[a-zA-Z0-9\-]+)`) defaultTimestampRegex = regexp.MustCompile(`(?P\d+-\d+-\d+T\d+:\d+:\d+(?:\.\d+Z)?)`) cloudtrailFilenameRegex = regexp.MustCompile(`AWSLogs\/(?Po-[a-z0-9]{10,32})?\/?(?P\d+)\/(?P[a-zA-Z0-9_\-]+)\/(?P[\w-]+)\/(?P\d+)\/(?P\d+)\/(?P\d+)\/\d+\_(?:CloudTrail|CloudTrail-Digest)_(?:\w+-\w+-(?:\w+-)?\d)_(?:(?:app|nlb|net)\.*?)?.+_(?P[a-zA-Z0-9\-]+)`) cloudfrontFilenameRegex = regexp.MustCompile(`(?P.*)\/(?P[A-Z0-9]+)\.(?P\d+)-(?P\d+)-(?P\d+)-(.+)`) From 75733170fcd917618649773b406530baa87c8811 Mon Sep 17 00:00:00 2001 From: Pedro Serrano Date: Wed, 27 May 2026 07:26:11 +0200 Subject: [PATCH 2/3] add test file --- pkg/s3_test.go | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/pkg/s3_test.go b/pkg/s3_test.go index ddcf80bb..26bde7b6 100644 --- a/pkg/s3_test.go +++ b/pkg/s3_test.go @@ -26,7 +26,40 @@ func Test_getLabels(t *testing.T) { wantErr bool }{ { - name: "s3_alb", + name: "s3_alb_conn_test", + args: args{ + record: events.S3EventRecord{ + AWSRegion: "us-east-1", + S3: events.S3Entity{ + Bucket: events.S3Bucket{ + Name: "elb_conn_logs_test", + OwnerIdentity: events.S3UserIdentity{ + PrincipalID: "test", + }, + }, + Object: events.S3Object{ + Key: "AWSLogs/123456789012/elasticloadbalancing/us-east-1/2023/10/04/conn_log_123456789012_elasticloadbalancing_us-east-1_app.my-connect-alb_20231004T1700Z_0.0.0.0_abcdef12.log.gz", + }, + }, + }, + }, + want: map[string]string{ + "account_id": "123456789012", + "bucket": "elb_conn_logs_test", + "bucket_owner": "test", + "bucket_region": "us-east-1", + "day": "04", + "key": "AWSLogs/123456789012/elasticloadbalancing/us-east-1/2023/10/04/conn_log_123456789012_elasticloadbalancing_us-east-1_app.my-connect-alb_20231004T1700Z_0.0.0.0_abcdef12.log.gz", + "month": "10", + "region": "us-east-1", + "lb_type": LbAlbType, + "src": "my-connect-alb", + "type": LbLogType, + "year": "2023", + }, + wantErr: false, + }, + { name: "s3_alb", args: args{ record: events.S3EventRecord{ AWSRegion: "us-east-1", From 06bc58a68aaf22ca89ce3ae987a2036fc12c308e Mon Sep 17 00:00:00 2001 From: Pedro Serrano Date: Tue, 30 Jun 2026 23:01:57 +0200 Subject: [PATCH 3/3] fix format --- pkg/s3_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/s3_test.go b/pkg/s3_test.go index 26bde7b6..c75f2a21 100644 --- a/pkg/s3_test.go +++ b/pkg/s3_test.go @@ -59,7 +59,7 @@ func Test_getLabels(t *testing.T) { }, wantErr: false, }, - { name: "s3_alb", + {name: "s3_alb", args: args{ record: events.S3EventRecord{ AWSRegion: "us-east-1",