From 5863d46344fdf9dbdc1adb58d115752e9a41b3df Mon Sep 17 00:00:00 2001 From: Martin Hutchinson Date: Wed, 27 Aug 2025 13:47:51 +0000 Subject: [PATCH] VIndex for Static CT logs This outputs each DNSName from each cert into the VIndex, traversing up to the eTLD plus one. Confirmed this works on the Google test log coachandhorses2026h1. This was picked because it's a smaller log (only 174 million entries). Changed general client to support static CT Input Logs verifier keys. It doesn't support static CT enough to dereference the input log pointers though. We'll need a custom client for that. --- go.mod | 22 +- go.sum | 58 ++--- vindex/cmd/client/client.go | 62 ++++-- vindex/cmd/ct/README.md | 77 +++++++ vindex/cmd/ct/main.go | 416 ++++++++++++++++++++++++++++++++++++ vindex/cmd/ct/main_test.go | 148 +++++++++++++ vindex/cmd/ct/web.go | 78 +++++++ vindex/outputlog_test.go | 2 +- 8 files changed, 811 insertions(+), 52 deletions(-) create mode 100644 vindex/cmd/ct/README.md create mode 100644 vindex/cmd/ct/main.go create mode 100644 vindex/cmd/ct/main_test.go create mode 100644 vindex/cmd/ct/web.go diff --git a/go.mod b/go.mod index 552fe99..cc92c0e 100644 --- a/go.mod +++ b/go.mod @@ -3,26 +3,31 @@ module github.com/transparency-dev/incubator go 1.25.0 require ( + filippo.io/sunlight v0.8.0 + filippo.io/torchwood v0.9.0 github.com/cockroachdb/pebble v1.1.5 github.com/go-git/go-git/v5 v5.19.1 github.com/google/go-cmp v0.7.0 github.com/gorilla/mux v1.8.1 github.com/prometheus/client_golang v1.23.2 - github.com/transparency-dev/formats v0.1.0 + github.com/transparency-dev/formats v0.1.1 github.com/transparency-dev/merkle v0.0.2 github.com/transparency-dev/tessera v1.0.3-0.20260318145621-a1e0ccb4adf4 go.opentelemetry.io/otel v1.44.0 go.opentelemetry.io/otel/exporters/prometheus v0.66.0 go.opentelemetry.io/otel/metric v1.44.0 go.opentelemetry.io/otel/sdk/metric v1.44.0 - golang.org/x/mod v0.36.0 - golang.org/x/sync v0.20.0 + golang.org/x/crypto v0.53.0 + golang.org/x/mod v0.37.0 + golang.org/x/net v0.55.0 + golang.org/x/sync v0.21.0 k8s.io/klog/v2 v2.140.0 rsc.io/tmp/mpt v0.2.0 ) require ( dario.cat/mergo v1.0.0 // indirect + filippo.io/mldsa v0.0.0-20260215214346-43d0283efc3e // indirect github.com/DataDog/zstd v1.4.5 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/ProtonMail/go-crypto v1.1.6 // indirect @@ -45,6 +50,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect github.com/golang/snappy v0.0.4 // indirect + github.com/google/certificate-transparency-go v1.3.2 // indirect github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect @@ -60,7 +66,7 @@ require ( github.com/prometheus/common v0.67.5 // indirect github.com/prometheus/otlptranslator v1.0.0 // indirect github.com/prometheus/procfs v0.20.1 // indirect - github.com/rogpeppe/go-internal v1.14.1 // indirect + github.com/rogpeppe/go-internal v1.15.0 // indirect github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect github.com/skeema/knownhosts v1.3.1 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect @@ -68,11 +74,9 @@ require ( go.opentelemetry.io/otel/sdk v1.44.0 // indirect go.opentelemetry.io/otel/trace v1.44.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect - golang.org/x/crypto v0.50.0 // indirect - golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect - golang.org/x/net v0.53.0 // indirect - golang.org/x/sys v0.45.0 // indirect - golang.org/x/text v0.36.0 // indirect + golang.org/x/exp v0.0.0-20260603202125-055de637280b // indirect + golang.org/x/sys v0.46.0 // indirect + golang.org/x/text v0.38.0 // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect ) diff --git a/go.sum b/go.sum index d0c20eb..da860c7 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,11 @@ dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +filippo.io/mldsa v0.0.0-20260215214346-43d0283efc3e h1:VsUbObBMxXlc23Eb9VeeJYE4jvTs87qa5RqSN2U5FJU= +filippo.io/mldsa v0.0.0-20260215214346-43d0283efc3e/go.mod h1:32qQ5yj3R24Eu03iWFWchdC3OB653wPvoepWejkefbY= +filippo.io/sunlight v0.8.0 h1:7ytoUj2KmU5k4ogDSLwEtCoEjjrTZsh+g++UIfTGpM4= +filippo.io/sunlight v0.8.0/go.mod h1:gJ1qFtjHWqj9j4f5M2fnaER6ZFPUkTrRz4/pTamneDg= +filippo.io/torchwood v0.9.0 h1:2W156cI7K3MyxEyNTuS1C9lYEW7y1u7PoHLmvgNsiZc= +filippo.io/torchwood v0.9.0/go.mod h1:zOJguxdmaODUQScAvC80bV6N0SOA9U+bFZG1DwJU6N8= github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= @@ -37,8 +43,9 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/cyphar/filepath-securejoin v0.6.1 h1:5CeZ1jPXEiYt3+Z6zqprSAgSWiggmpVyciv8syjIpVE= github.com/cyphar/filepath-securejoin v0.6.1/go.mod h1:A8hd4EnAeyujCJRrICiOWqjS1AX0a9kM5XL+NwKoYSc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o= github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= @@ -68,6 +75,8 @@ github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8J github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/certificate-transparency-go v1.3.2 h1:9ahSNZF2o7SYMaKaXhAumVEzXB2QaayzII9C8rv7v+A= +github.com/google/certificate-transparency-go v1.3.2/go.mod h1:H5FpMUaGa5Ab2+KCYsxg6sELw3Flkl7pGZzWdBoYLXs= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -106,8 +115,9 @@ github.com/pjbgf/sha1cd v0.6.0/go.mod h1:lhpGlyHLpQZoxMv8HcgXvZEhcGs0PG/vsZnEJ7H github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= @@ -119,8 +129,8 @@ github.com/prometheus/otlptranslator v1.0.0/go.mod h1:vRYWnXvI6aWGpsdY/mOT/cbeVR github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc= github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= -github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/rogpeppe/go-internal v1.15.0 h1:D0RCU5rMAp+SpgkiNdrjfJ+LX4J1M32V2NeCY7EJ6hc= +github.com/rogpeppe/go-internal v1.15.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= @@ -131,8 +141,8 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -github.com/transparency-dev/formats v0.1.0 h1:oL0zUFuYUjg8AbtjPMnIRDmjbaHo5jCjEWU5yaNuz0g= -github.com/transparency-dev/formats v0.1.0/go.mod h1:d2FibUOHfCMdCe/+/rbKt1IPLBbPTDfwj46kt541/mU= +github.com/transparency-dev/formats v0.1.1 h1:4bVHJc+KdBgpA1OJD1yjI+g0i5Z1graCppTMH8lWKJI= +github.com/transparency-dev/formats v0.1.1/go.mod h1:qtZ8goRuJ8FTBG9c9+Bj0rn2rUG7eG/AUTkr+Aw3jFw= github.com/transparency-dev/merkle v0.0.2 h1:Q9nBoQcZcgPamMkGn7ghV8XiTZ/kRxn1yCG81+twTK4= github.com/transparency-dev/merkle v0.0.2/go.mod h1:pqSy+OXefQ1EDUVmAJ8MUhHB9TXGuzVAT58PqBoHz1A= github.com/transparency-dev/tessera v1.0.3-0.20260318145621-a1e0ccb4adf4 h1:6Xj0ZKwzfBFuNALfJ2Ys6PtWdGqSoOR6gxxyG0XS4qo= @@ -165,26 +175,26 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= -golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= -golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= -golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80= +golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= +golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= +golang.org/x/exp v0.0.0-20260603202125-055de637280b h1:v1uXiEBHo8QA0LiGCo7UgHMzHT4Kdfpl2zmtH5vaP1Q= +golang.org/x/exp v0.0.0-20260603202125-055de637280b/go.mod h1:d2fgXJLVs4dYDHUk5lwMIfzRzSrWCfGZb0ZqeLa/Vcw= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= -golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= +golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= +golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= -golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= +golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= +golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= -golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -194,22 +204,22 @@ golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= -golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY= -golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY= +golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= +golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= -golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= +golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= +golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= -golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= +golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= +golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/vindex/cmd/client/client.go b/vindex/cmd/client/client.go index a0b32ec..8def047 100644 --- a/vindex/cmd/client/client.go +++ b/vindex/cmd/client/client.go @@ -19,25 +19,29 @@ package main import ( "context" + "crypto/x509" + "encoding/base64" "errors" "flag" "fmt" "net/http" "slices" + fnote "github.com/transparency-dev/formats/note" "github.com/transparency-dev/incubator/vindex/client" "golang.org/x/mod/sumdb/note" "k8s.io/klog/v2" ) var ( - vindexBaseURL = flag.String("vindex_base_url", "", "The base URL of the vindex server.") - inLogBaseURL = flag.String("in_log_base_url", "", "The base URL of the input log.") - lookup = flag.String("lookup", "", "The key to look up in the vindex.") - outLogPubKey = flag.String("out_log_pub_key", "", "The public key to use to verify the output log checkpoint.") - inLogPubKey = flag.String("in_log_pub_key", "", "The public key to use to verify the input log checkpoint.") - inLogOrigin = flag.String("in_log_origin", "", "Optional: allows the Input Log Origin string to be configured to something other than the public key name.") - minIdx = flag.Uint64("min_idx", 0, "The minimum index to look up in the input log.") + vindexBaseURL = flag.String("vindex_base_url", "", "The base URL of the vindex server.") + inLogBaseURL = flag.String("in_log_base_url", "", "The base URL of the input log.") + lookup = flag.String("lookup", "", "The key to look up in the vindex.") + outLogPubKey = flag.String("out_log_pub_key", "", "The public key to use to verify the output log checkpoint. Required.") + inLogPubKey = flag.String("in_log_pub_key", "", "The public key to use to verify the input log checkpoint. Required.") + inLogPubKeyDER = flag.String("in_log_pub_key_der", "", "For CT logs. The public key to use to verify the input log checkpoint. Required, along with in_log_origin.") + inLogOrigin = flag.String("in_log_origin", "", "Required if in_log_pub_key_der is used. Otherwise, allows the Input Log Origin string to be configured to something other than the public key name.") + minIdx = flag.Uint64("min_idx", 0, "The minimum index to look up in the input log.") ) func main() { @@ -104,10 +108,7 @@ func newVIndexClientFromFlags() *client.VIndexClient { if *outLogPubKey == "" { klog.Exitf("out_log_pub_key must be provided") } - inV, err := note.NewVerifier(*inLogPubKey) - if err != nil { - klog.Exitf("failed to construct input log verifier: %v", err) - } + inV := inputLogVerifierFromFlags() outV, err := note.NewVerifier(*outLogPubKey) if err != nil { klog.Exitf("failed to construct output log verifier: %v", err) @@ -123,13 +124,7 @@ func newInputLogClientFromFlags() *client.InputLogClient { if *inLogBaseURL == "" { klog.Exit("in_log_base_url flag must be provided") } - if *inLogPubKey == "" { - klog.Exitf("in_log_pub_key must be provided") - } - v, err := note.NewVerifier(*inLogPubKey) - if err != nil { - klog.Exitf("failed to construct input log verifier: %v", err) - } + v := inputLogVerifierFromFlags() origin := *inLogOrigin if len(origin) == 0 { origin = v.Name() @@ -140,3 +135,34 @@ func newInputLogClientFromFlags() *client.InputLogClient { } return c } + +func inputLogVerifierFromFlags() note.Verifier { + if *inLogPubKey == "" && *inLogPubKeyDER == "" { + klog.Exitf("Must provide exactly one --in_log_pub_key* flag") + } + if *inLogPubKey != "" { + v, err := note.NewVerifier(*inLogPubKey) + if err != nil { + klog.Exitf("failed to construct input log verifier: %v", err) + } + return v + } + derBytes, err := base64.StdEncoding.DecodeString(*inLogPubKeyDER) + if err != nil { + klog.Exitf("Error decoding public key: %s", err) + } + pub, err := x509.ParsePKIXPublicKey(derBytes) + if err != nil { + klog.Exitf("Error parsing public key: %v", err) + } + + verifierKey, err := fnote.RFC6962VerifierString(*inLogOrigin, pub) + if err != nil { + klog.Exitf("Error creating RFC6962 verifier string: %v", err) + } + v, err := fnote.NewVerifier(verifierKey) + if err != nil { + klog.Exitf("Error creating verifier: %v", err) + } + return v +} diff --git a/vindex/cmd/ct/README.md b/vindex/cmd/ct/README.md new file mode 100644 index 0000000..da63d19 --- /dev/null +++ b/vindex/cmd/ct/README.md @@ -0,0 +1,77 @@ +## Verifiable Index: CT + +This is a demo of pulling the contents of a tile-based CT log into a [Verifiable Index](../../README.md). + +[tlog-tiles]: https://c2sp.org/tlog-tiles +[Tessera]: https://github.com/transparency-dev/tessera + +The CT Input Log is processed, with each entry being indexed on all common names defined in the cert. +This allows the owner of a domain to look up all certs for their domain, in a way that is fully verified. + +## Running + +The static CT Input Log is expected to be available for reading at a URL provided by the `--monitoring_url` flag. +This is the base directory that should contain the checkpoint file. +The Verifiable Index and Output Log are constructed locally, persisted to local disk (in the `--storage_dir` directory), and hosted via a web server. + +```shell +OUTPUT_LOG_PRIVATE_KEY=PRIVATE+KEY+example.com/outputlog+07392c46+ATPJ4crkyUbPeaRffN/4NUof3KV0pQznVIPGOQm3SDEJ \ +go run ./vindex/cmd/ct \ + --storage_dir ~/vindex-ct/ \ + --origin="coachandhorses2026h1.staging.certificate.transparency.goog" \ + --public_key="MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAECHOhXfvYgTcu+Fnl7M7niFj3FgqWlQpXUSWUDw2KAaJXvhGxdJTtmyciN5rWTiDtpeNENVmsUTHFS4XQgeRE0g==" \ + --monitoring_url="https://storage.googleapis.com/coachandhorses2026h1.staging.certificate.transparency.goog" +``` + +### Resource Requirements + +Running this demo has the following estimated system requirements (based on indexing ~180M certs from a staging log): +* **Architecture:** **64-bit OS** (Linux/Unix). The MPT library reserves a large virtual address space (16 TB) and cannot run on 32-bit systems. +* **RAM:** **32 GB+** (observed ~31 GB physical RAM usage). The current prototype stores the key-value index in a raw Go map in memory. +* **CPU:** **8+ cores** (actively utilizes ~6 cores during ingestion). +* **Disk:** **100 GB+ SSD** (uses ~45 GB for WAL and MPT files, fast I/O is required). + +> [!NOTE] +> The high memory usage is a limitation of the current prototype's in-memory key-value store. The planned [v1 architecture](../../docs/v1/IMPLEMENTATION.md) will move this store to a disk-backed Pebble database, which is expected to drastically reduce RAM requirements to approximately 6-8 GB. + +Running the above will run a web server hosting the following URLs: + - `/vindex/lookup` - the provisional [vindex lookup API](./api/api.go) + - `/outputlog/` - the [tlog-tiles][] base URL for the output log + +To inspect the log, you can use the woodpecker tool (using the corresponding public key to the private key used above): + +```shell +# To inspect the Output Log +go run github.com/mhutchinson/woodpecker@main --custom_log_type=tiles --custom_log_url=http://localhost:8088/outputlog/ --custom_log_vkey=example.com/outputlog+07392c46+AWyS8y8ZsRmQnTr6Fr2knaa8+t6CPYFh5Ho3wJEr14B8 +``` + +Use left/right cursor to browse, and `q` to quit. + +A domain indexed by the verifiable map can be looked up using the following command: + +```shell +go run ./vindex/cmd/client \ + --vindex_base_url http://localhost:8088/vindex/ \ + --out_log_pub_key=example.com/outputlog+07392c46+AWyS8y8ZsRmQnTr6Fr2knaa8+t6CPYFh5Ho3wJEr14B8 \ + --in_log_pub_key_der=MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAECHOhXfvYgTcu+Fnl7M7niFj3FgqWlQpXUSWUDw2KAaJXvhGxdJTtmyciN5rWTiDtpeNENVmsUTHFS4XQgeRE0g== \ + --in_log_origin=coachandhorses2026h1.staging.certificate.transparency.goog \ + --lookup=google.com + +I0610 15:02:17.112527 87150 client.go:83] in_log_base_url not provided, so cannot dereference pointers +148000245 +151898263 +152014951 +152015244 +152015262 +152015307 +... +154306826 +154307178 +154307232 +154307321 +154368790 +154368845 +``` + +To view the certs at the index, [woodpecker](https://github.com/mhutchinson/woodpecker) can be used. + diff --git a/vindex/cmd/ct/main.go b/vindex/cmd/ct/main.go new file mode 100644 index 0000000..e7f45a6 --- /dev/null +++ b/vindex/cmd/ct/main.go @@ -0,0 +1,416 @@ +// Copyright 2025 Google LLC. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// logandmap is a binary that serves as a demo of how to run a log and a map in the +// same process. +// The log is a Tessera POSIX log, and the map is an in-memory verifiable index. +// A web server is hosted that allows lookups in the map to be performed. +// The log is updated periodically with entries of type LogEntry, and the map keys +// each of the module names from that struct to each of the indices in the log where +// an entry for that module is stored. +package main + +import ( + "context" + "crypto/sha256" + "crypto/x509" + "encoding/base64" + "errors" + "flag" + "fmt" + "iter" + "net/http" + "os" + "os/signal" + "path" + "strings" + "syscall" + "time" + + "golang.org/x/net/publicsuffix" + + "filippo.io/sunlight" + "filippo.io/torchwood" + "github.com/gorilla/mux" + "github.com/transparency-dev/formats/log" + fnote "github.com/transparency-dev/formats/note" + "github.com/transparency-dev/incubator/vindex" + "go.opentelemetry.io/otel/exporters/prometheus" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "golang.org/x/crypto/cryptobyte" + "golang.org/x/mod/sumdb/note" + "golang.org/x/mod/sumdb/tlog" + "k8s.io/klog/v2" +) + +var ( + inputLogUrl = flag.String("monitoring_url", "", "Base URL of the static CT log to index") + origin = flag.String("origin", "", "Origin of the log to check") + pubKey = flag.String("public_key", "", "The log's public key in base64 encoded DER format") + userAgentInfo = flag.String("user_agent_info", "", "Optional string to append to the user agent (e.g. email address for Sunlight logs)") + persistentCacheDir = flag.String("persistent_cache_dir", "", "Optional location of a directory to cache Input Log tiles") + persistIndex = flag.Bool("persist_index", true, "Set to false to use a memory-based implementation of the verifiable index.") + + outputLogPrivKeyFile = flag.String("output_log_private_key", "", "Location of private key file. If unset, uses the contents of the OUTPUT_LOG_PRIVATE_KEY environment variable.") + storageDir = flag.String("storage_dir", "", "Root directory in which to store the data for the demo. This will create subdirectories for the Input Log, Output Log, and allocate space to store the verifiable map persistence.") + listen = flag.String("listen", ":8088", "Address to set up HTTP server listening on") +) + +const ( + userAgent = "TrustFabric VerifiableIndex" +) + +func main() { + klog.InitFlags(nil) + flag.Parse() + + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer cancel() + + if err := run(ctx); err != nil { + klog.Exitf("Run failed: %v", err) + } +} + +func run(ctx context.Context) error { + // Set up storage for the input log, index, and output log. + if *storageDir == "" { + return errors.New("storage_dir must be set") + } + outputLogDir := path.Join(*storageDir, "outputlog") + mapRoot := path.Join(*storageDir, "vindex") + + if err := os.MkdirAll(outputLogDir, 0o755); err != nil { + return fmt.Errorf("failed to create output log directory: %v", err) + } + if err := os.MkdirAll(mapRoot, 0o755); err != nil { + return fmt.Errorf("failed to create vindex directory: %v", err) + } + + exporter, err := prometheus.New() + if err != nil { + return fmt.Errorf("failed to create prometheus exporter: %v", err) + } + provider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(exporter)) + defer func() { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := provider.Shutdown(shutdownCtx); err != nil { + klog.Errorf("failed to shutdown meter provider: %v", err) + } + }() + + outputLog, outputCloser := outputLogOrDie(ctx, outputLogDir) + defer outputCloser(ctx) + + inputLog := newStaticCTInputLogFromFlags() + + vi, err := vindex.NewVerifiableIndex(ctx, inputLog, mapFn, outputLog, mapRoot, vindex.Options{ + PersistIndex: *persistIndex, + MeterProvider: provider, + }) + if err != nil { + return fmt.Errorf("failed to create vindex: %v", err) + } + klog.Info("Created verifiable index") + + // Keeps the map synced with the latest published input log state. + go maintainMap(ctx, vi) + + // Run a web server to serve the input log, index, and output log. + go runWebServer(vi, outputLogDir) + <-ctx.Done() + return nil +} + +func cutEntry(tile []byte) (entry []byte, rh tlog.Hash, rest []byte, err error) { + // This implementation is terribly inefficient, parsing the whole entry just + // to re-serialize and throw it away. If this function shows up in profiles, + // let me know and I'll improve it. + e, rest, err := sunlight.ReadTileLeaf(tile) + if err != nil { + return nil, tlog.Hash{}, nil, err + } + rh = tlog.RecordHash(e.MerkleTreeLeaf()) + entry = tile[:len(tile)-len(rest)] + return entry, rh, rest, nil +} + +func newStaticCTInputLogFromFlags() *staticCTInputLog { + ua := userAgent + if *userAgentInfo != "" { + ua = fmt.Sprintf("%s (%s)", userAgent, *userAgentInfo) + } + fetcher, err := torchwood.NewTileFetcher(*inputLogUrl, + torchwood.WithTilePath(sunlight.TilePath), + torchwood.WithUserAgent(ua)) + if err != nil { + klog.Exitf("failed to create client: %v", err) + } + var tileReader torchwood.TileReader = fetcher + if *persistentCacheDir != "" { + tileReader, err = torchwood.NewPermanentCache(fetcher, *persistentCacheDir) + if err != nil { + klog.Exitf("failed to create permanent cache: %v", err) + } + } + client, err := torchwood.NewClient(tileReader, torchwood.WithCutEntry(cutEntry)) + if err != nil { + klog.Exitf("failed to create client: %v", err) + } + return &staticCTInputLog{ + c: client, + f: fetcher, + v: verifierFromFlags(), + } +} + +type staticCTInputLog struct { + c *torchwood.Client + f *torchwood.TileFetcher + v note.Verifier + + lastCheckpoint log.Checkpoint +} + +func (l *staticCTInputLog) Checkpoint(ctx context.Context) (checkpoint []byte, err error) { + return l.f.ReadEndpoint(ctx, "checkpoint") +} + +// Parse unmarshals and verifies a checkpoint obtained from GetCheckpoint. +func (l *staticCTInputLog) Parse(checkpoint []byte) (*log.Checkpoint, error) { + cp, _, _, err := log.ParseCheckpoint(checkpoint, l.v.Name(), l.v) + if err != nil { + return nil, err + } + l.lastCheckpoint = *cp + return cp, err +} + +// Leaves returns all the leaves in the range [start, end), outputting them via +// the returned iterator. +func (l *staticCTInputLog) Leaves(ctx context.Context, start, end uint64) iter.Seq2[[]byte, error] { + tree := tlog.Tree{ + N: int64(end), + Hash: tlog.Hash(l.lastCheckpoint.Hash), + } + return func(yield func([]byte, error) bool) { + for _, entry := range l.c.Entries(ctx, tree, int64(start)) { + e, _, err := sunlight.ReadTileLeaf(entry) + if err != nil { + if !yield(nil, err) { + return + } + } + if !yield(e.MerkleTreeLeaf(), nil) { + return + } + } + if err := l.c.Err(); err != nil { + yield(nil, l.c.Err()) + } + } +} + +// outputLogOrDie returns an output log using a POSIX log in the given directory. +func outputLogOrDie(ctx context.Context, outputLogDir string) (log vindex.OutputLog, closer func(context.Context)) { + s, v := getOutputLogSignerVerifierOrDie() + + l, c, err := vindex.NewOutputLog(ctx, outputLogDir, s, v, vindex.OutputLogOpts{}) + if err != nil { + klog.Exit(err) + } + return l, c +} + +func verifierFromFlags() note.Verifier { + if *origin == "" { + klog.Exitf("Must provide the --origin flag") + } + if *pubKey == "" { + klog.Exitf("Must provide the --pub_key flag") + } + derBytes, err := base64.StdEncoding.DecodeString(*pubKey) + if err != nil { + klog.Exitf("Error decoding public key: %s", err) + } + pub, err := x509.ParsePKIXPublicKey(derBytes) + if err != nil { + klog.Exitf("Error parsing public key: %v", err) + } + + verifierKey, err := fnote.RFC6962VerifierString(*origin, pub) + if err != nil { + klog.Exitf("Error creating RFC6962 verifier string: %v", err) + } + logSigV, err := fnote.NewVerifier(verifierKey) + if err != nil { + klog.Exitf("Error creating verifier: %v", err) + } + + klog.Infof("Using verifier string: %v", verifierKey) + + return logSigV +} + +// maintainMap reads entries from the log and sync them to the vindex. +func maintainMap(ctx context.Context, vi *vindex.VerifiableIndex) { + ticker := time.NewTicker(10 * time.Second) + defer ticker.Stop() + + for { + if err := vi.Update(ctx); err != nil { + klog.Warning(err) + } + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } +} + +func runWebServer(vi *vindex.VerifiableIndex, outLogDir string) { + web := NewServer(vi.Lookup) + + olfs := http.FileServer(http.Dir(outLogDir)) + r := mux.NewRouter() + r.PathPrefix("/outputlog/").Handler(http.StripPrefix("/outputlog/", olfs)) + web.registerHandlers(r) + hServer := &http.Server{ + Addr: *listen, + Handler: r, + } + go func() { + _ = hServer.ListenAndServe() + }() + klog.Infof("Started HTTP server listening on %s", *listen) +} + +// Read output log private key from file or environment variable and generate the +// note Signer and Verifier pair for it. +func getOutputLogSignerVerifierOrDie() (note.Signer, note.Verifier) { + var privKey string + var err error + if len(*outputLogPrivKeyFile) > 0 { + privKey, err = getKeyFile(*outputLogPrivKeyFile) + if err != nil { + klog.Exitf("Unable to get private key: %v", err) + } + } else { + privKey = os.Getenv("OUTPUT_LOG_PRIVATE_KEY") + if len(privKey) == 0 { + klog.Exit("Supply private key file path using --output_log_private_key or set OUTPUT_LOG_PRIVATE_KEY environment variable") + } + } + s, v, err := fnote.NewEd25519SignerVerifier(privKey) + if err != nil { + klog.Exitf("Failed to get signer/verifier: %v", err) + } + return s, v +} + +func getKeyFile(path string) (string, error) { + k, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("failed to read key file: %w", err) + } + return string(k), nil +} + +func mapFn(data []byte) [][sha256.Size]byte { + s := cryptobyte.String(data) + + var version, leafType uint8 + var timestamp uint64 + var certType uint16 + if !s.ReadUint8(&version) || !s.ReadUint8(&leafType) || !s.ReadUint64(×tamp) || !s.ReadUint16(&certType) { + klog.Warningf("Failed to unmarshal headers") + // This should return a sentinel value (e.g. all zero hash) so unprocessable entries can be found + return nil + } + var isPreCert bool + var cert cryptobyte.String + switch certType { + case 0: + // x509 + isPreCert = false + s.ReadUint24LengthPrefixed(&cert) + case 1: + if true { + // Need to support parsing TBS certs + return nil + } + // precert + isPreCert = true + var ikh []byte + s.ReadBytes(&ikh, sha256.Size) + s.ReadUint24LengthPrefixed(&cert) + default: + panic("unknown cert type") + } + + parsedCert, err := x509.ParseCertificate(cert) + if err != nil { + klog.Warningf("failed to parse x509 cert (preCert=%t): %v", isPreCert, err) + // This could return a sentinel value (e.g. all zero hash) so unprocessable entries can be found + return nil + } + if klog.V(2).Enabled() { + klog.V(2).Info(parsedCert.DNSNames) + } + uniqueNames := make(map[string]bool) + for _, cn := range parsedCert.DNSNames { + cn = strings.ToLower(cn) + if strings.HasPrefix(cn, "*.") { + cn = cn[2:] + } else if strings.HasPrefix(cn, "*") { + cn = cn[1:] + } + if cn == "" { + continue + } + uniqueNames[cn] = true + + // TODO(mhutchinson): This is the most controversial part - the eTLD evolves. + // We can lock this down with WASM, but what to do when the ecosystem needs the new changes? + etld1, err := publicsuffix.EffectiveTLDPlusOne(cn) + if err != nil { + continue + } + if cn == etld1 { + continue + } + curr := cn + for { + idx := strings.Index(curr, ".") + if idx == -1 { + break + } + curr = curr[idx+1:] + if len(curr) < len(etld1) { + break + } + uniqueNames[curr] = true + if curr == etld1 { + break + } + } + } + hashes := make([][sha256.Size]byte, 0, len(uniqueNames)) + for name := range uniqueNames { + hashes = append(hashes, sha256.Sum256([]byte(name))) + } + return hashes +} diff --git a/vindex/cmd/ct/main_test.go b/vindex/cmd/ct/main_test.go new file mode 100644 index 0000000..df6c638 --- /dev/null +++ b/vindex/cmd/ct/main_test.go @@ -0,0 +1,148 @@ +// Copyright 2026 Google LLC. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "crypto/rand" + "crypto/rsa" + "crypto/sha256" + "crypto/x509" + "crypto/x509/pkix" + "math/big" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/gorilla/mux" + "github.com/transparency-dev/incubator/vindex/api" + "golang.org/x/crypto/cryptobyte" +) + +func createTestCertBytes(dnsNames []string) ([]byte, error) { + priv, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return nil, err + } + template := x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{ + Organization: []string{"Test Org"}, + }, + NotBefore: time.Now().Add(-1 * time.Hour), + NotAfter: time.Now().Add(1 * time.Hour), + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + DNSNames: dnsNames, + } + certDER, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv) + if err != nil { + return nil, err + } + + // Now marshal using cryptobyte into the form expected by mapFn + // version = 0, leafType = 0, timestamp = 12345678, certType = 0 (x509) + var b cryptobyte.Builder + b.AddUint8(0) // version + b.AddUint8(0) // leafType + b.AddUint64(12345678) // timestamp + b.AddUint16(0) // certType (x509) + b.AddUint24LengthPrefixed(func(b *cryptobyte.Builder) { + b.AddBytes(certDER) + }) + return b.Bytes() +} + +func TestMapFn(t *testing.T) { + testCases := []struct { + desc string + dnsNames []string + wantKeys []string + }{ + { + desc: "standard domains and wildcards", + dnsNames: []string{"*.google.com", "google.com", "maps.google.com"}, + wantKeys: []string{"google.com", "maps.google.com"}, + }, + { + desc: "deeper subdomain", + dnsNames: []string{"deep.maps.google.co.uk"}, + wantKeys: []string{"deep.maps.google.co.uk", "maps.google.co.uk", "google.co.uk"}, + }, + { + desc: "mixed case", + dnsNames: []string{"MAPS.GOOGLE.COM"}, + wantKeys: []string{"maps.google.com", "google.com"}, + }, + { + desc: "invalid or TLD", + dnsNames: []string{"localhost", "*.co.uk"}, + wantKeys: []string{"localhost", "co.uk"}, + }, + } + + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + data, err := createTestCertBytes(tc.dnsNames) + if err != nil { + t.Fatalf("failed to create test cert: %v", err) + } + hashes := mapFn(data) + + gotKeys := make(map[string]bool) + for _, h := range hashes { + found := false + for _, wk := range tc.wantKeys { + if sha256.Sum256([]byte(wk)) == h { + gotKeys[wk] = true + found = true + break + } + } + if !found { + t.Errorf("got unexpected hash for a key not in wantKeys") + } + } + + for _, wk := range tc.wantKeys { + if !gotKeys[wk] { + t.Errorf("missing expected key: %s", wk) + } + } + if len(gotKeys) != len(tc.wantKeys) { + t.Errorf("got %d unique keys, want %d", len(gotKeys), len(tc.wantKeys)) + } + }) + } +} + +func TestMetricsEndpoint(t *testing.T) { + s := NewServer(func(ctx context.Context, h [sha256.Size]byte) (api.LookupResponse, error) { + return api.LookupResponse{}, nil + }) + r := mux.NewRouter() + s.registerHandlers(r) + + req := httptest.NewRequest("GET", "/metrics", nil) + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Errorf("GET /metrics: got status %d, want %d", w.Code, http.StatusOK) + } +} + diff --git a/vindex/cmd/ct/web.go b/vindex/cmd/ct/web.go new file mode 100644 index 0000000..7fc62c2 --- /dev/null +++ b/vindex/cmd/ct/web.go @@ -0,0 +1,78 @@ +// Copyright 2025 Google LLC. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "crypto/sha256" + _ "embed" + "encoding/hex" + "encoding/json" + "fmt" + "net/http" + + "github.com/gorilla/mux" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/transparency-dev/incubator/vindex/api" + "k8s.io/klog/v2" +) + +func NewServer(lookup func(context.Context, [sha256.Size]byte) (api.LookupResponse, error)) Server { + return Server{ + lookup: lookup, + } +} + +type Server struct { + lookup func(context.Context, [sha256.Size]byte) (api.LookupResponse, error) +} + +// handleLookup handles GET requests for looking up map entries. +func (s Server) handleLookup(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + hashStr, ok := vars["hash"] + if !ok { + http.Error(w, "hash parameter not found", http.StatusBadRequest) + return + } + + h, err := hex.DecodeString(hashStr) + if err != nil { + http.Error(w, fmt.Sprintf("invalid hex hash: %v", err), http.StatusBadRequest) + return + } + if l := len(h); l != sha256.Size { + http.Error(w, fmt.Sprintf("hash wrong length (decoded %d bytes)", l), http.StatusBadRequest) + return + } + + klog.V(2).Infof("Received hash from request: '%s'", h) + + resp, err := s.lookup(r.Context(), [sha256.Size]byte(h)) + if err != nil { + http.Error(w, fmt.Sprintf("lookup failed: %v", err), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + if err := json.NewEncoder(w).Encode(resp); err != nil { + klog.Warningf("failed to encode response: %v", err) + } +} + +func (s Server) registerHandlers(r *mux.Router) { + r.HandleFunc("/vindex/lookup/{hash}", s.handleLookup).Methods("GET") + r.Handle("/metrics", promhttp.Handler()) +} diff --git a/vindex/outputlog_test.go b/vindex/outputlog_test.go index d55e67f..d002767 100644 --- a/vindex/outputlog_test.go +++ b/vindex/outputlog_test.go @@ -80,7 +80,7 @@ func TestOutputLog_Lookup(t *testing.T) { t.Fatal(err) } defer func() { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) defer cancel() closer(ctx) }()