diff --git a/go.mod b/go.mod index 552fe99..cc92c0e 100644 --- a/go.mod +++ b/go.mod @@ -3,26 +3,31 @@ module github.com/transparency-dev/incubator go 1.25.0 require ( + filippo.io/sunlight v0.8.0 + filippo.io/torchwood v0.9.0 github.com/cockroachdb/pebble v1.1.5 github.com/go-git/go-git/v5 v5.19.1 github.com/google/go-cmp v0.7.0 github.com/gorilla/mux v1.8.1 github.com/prometheus/client_golang v1.23.2 - github.com/transparency-dev/formats v0.1.0 + github.com/transparency-dev/formats v0.1.1 github.com/transparency-dev/merkle v0.0.2 github.com/transparency-dev/tessera v1.0.3-0.20260318145621-a1e0ccb4adf4 go.opentelemetry.io/otel v1.44.0 go.opentelemetry.io/otel/exporters/prometheus v0.66.0 go.opentelemetry.io/otel/metric v1.44.0 go.opentelemetry.io/otel/sdk/metric v1.44.0 - golang.org/x/mod v0.36.0 - golang.org/x/sync v0.20.0 + golang.org/x/crypto v0.53.0 + golang.org/x/mod v0.37.0 + golang.org/x/net v0.55.0 + golang.org/x/sync v0.21.0 k8s.io/klog/v2 v2.140.0 rsc.io/tmp/mpt v0.2.0 ) require ( dario.cat/mergo v1.0.0 // indirect + filippo.io/mldsa v0.0.0-20260215214346-43d0283efc3e // indirect github.com/DataDog/zstd v1.4.5 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/ProtonMail/go-crypto v1.1.6 // indirect @@ -45,6 +50,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect github.com/golang/snappy v0.0.4 // indirect + github.com/google/certificate-transparency-go v1.3.2 // indirect github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect @@ -60,7 +66,7 @@ require ( github.com/prometheus/common v0.67.5 // indirect github.com/prometheus/otlptranslator v1.0.0 // indirect github.com/prometheus/procfs v0.20.1 // indirect - github.com/rogpeppe/go-internal v1.14.1 // indirect + github.com/rogpeppe/go-internal v1.15.0 // indirect github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect github.com/skeema/knownhosts v1.3.1 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect @@ -68,11 +74,9 @@ require ( go.opentelemetry.io/otel/sdk v1.44.0 // indirect go.opentelemetry.io/otel/trace v1.44.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect - golang.org/x/crypto v0.50.0 // indirect - golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect - golang.org/x/net v0.53.0 // indirect - golang.org/x/sys v0.45.0 // indirect - golang.org/x/text v0.36.0 // indirect + golang.org/x/exp v0.0.0-20260603202125-055de637280b // indirect + golang.org/x/sys v0.46.0 // indirect + golang.org/x/text v0.38.0 // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect ) diff --git a/go.sum b/go.sum index d0c20eb..da860c7 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,11 @@ dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +filippo.io/mldsa v0.0.0-20260215214346-43d0283efc3e h1:VsUbObBMxXlc23Eb9VeeJYE4jvTs87qa5RqSN2U5FJU= +filippo.io/mldsa v0.0.0-20260215214346-43d0283efc3e/go.mod h1:32qQ5yj3R24Eu03iWFWchdC3OB653wPvoepWejkefbY= +filippo.io/sunlight v0.8.0 h1:7ytoUj2KmU5k4ogDSLwEtCoEjjrTZsh+g++UIfTGpM4= +filippo.io/sunlight v0.8.0/go.mod h1:gJ1qFtjHWqj9j4f5M2fnaER6ZFPUkTrRz4/pTamneDg= +filippo.io/torchwood v0.9.0 h1:2W156cI7K3MyxEyNTuS1C9lYEW7y1u7PoHLmvgNsiZc= +filippo.io/torchwood v0.9.0/go.mod h1:zOJguxdmaODUQScAvC80bV6N0SOA9U+bFZG1DwJU6N8= github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= @@ -37,8 +43,9 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/cyphar/filepath-securejoin v0.6.1 h1:5CeZ1jPXEiYt3+Z6zqprSAgSWiggmpVyciv8syjIpVE= github.com/cyphar/filepath-securejoin v0.6.1/go.mod h1:A8hd4EnAeyujCJRrICiOWqjS1AX0a9kM5XL+NwKoYSc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o= github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= @@ -68,6 +75,8 @@ github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8J github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/certificate-transparency-go v1.3.2 h1:9ahSNZF2o7SYMaKaXhAumVEzXB2QaayzII9C8rv7v+A= +github.com/google/certificate-transparency-go v1.3.2/go.mod h1:H5FpMUaGa5Ab2+KCYsxg6sELw3Flkl7pGZzWdBoYLXs= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -106,8 +115,9 @@ github.com/pjbgf/sha1cd v0.6.0/go.mod h1:lhpGlyHLpQZoxMv8HcgXvZEhcGs0PG/vsZnEJ7H github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= @@ -119,8 +129,8 @@ github.com/prometheus/otlptranslator v1.0.0/go.mod h1:vRYWnXvI6aWGpsdY/mOT/cbeVR github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc= github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= -github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/rogpeppe/go-internal v1.15.0 h1:D0RCU5rMAp+SpgkiNdrjfJ+LX4J1M32V2NeCY7EJ6hc= +github.com/rogpeppe/go-internal v1.15.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= @@ -131,8 +141,8 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -github.com/transparency-dev/formats v0.1.0 h1:oL0zUFuYUjg8AbtjPMnIRDmjbaHo5jCjEWU5yaNuz0g= -github.com/transparency-dev/formats v0.1.0/go.mod h1:d2FibUOHfCMdCe/+/rbKt1IPLBbPTDfwj46kt541/mU= +github.com/transparency-dev/formats v0.1.1 h1:4bVHJc+KdBgpA1OJD1yjI+g0i5Z1graCppTMH8lWKJI= +github.com/transparency-dev/formats v0.1.1/go.mod h1:qtZ8goRuJ8FTBG9c9+Bj0rn2rUG7eG/AUTkr+Aw3jFw= github.com/transparency-dev/merkle v0.0.2 h1:Q9nBoQcZcgPamMkGn7ghV8XiTZ/kRxn1yCG81+twTK4= github.com/transparency-dev/merkle v0.0.2/go.mod h1:pqSy+OXefQ1EDUVmAJ8MUhHB9TXGuzVAT58PqBoHz1A= github.com/transparency-dev/tessera v1.0.3-0.20260318145621-a1e0ccb4adf4 h1:6Xj0ZKwzfBFuNALfJ2Ys6PtWdGqSoOR6gxxyG0XS4qo= @@ -165,26 +175,26 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= -golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= -golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= -golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80= +golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= +golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= +golang.org/x/exp v0.0.0-20260603202125-055de637280b h1:v1uXiEBHo8QA0LiGCo7UgHMzHT4Kdfpl2zmtH5vaP1Q= +golang.org/x/exp v0.0.0-20260603202125-055de637280b/go.mod h1:d2fgXJLVs4dYDHUk5lwMIfzRzSrWCfGZb0ZqeLa/Vcw= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= -golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= +golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= +golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= -golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= +golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= +golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= -golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -194,22 +204,22 @@ golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= -golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY= -golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY= +golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= +golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= -golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= +golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= +golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= -golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= +golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= +golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/vindex/cmd/client/client.go b/vindex/cmd/client/client.go index a0b32ec..8def047 100644 --- a/vindex/cmd/client/client.go +++ b/vindex/cmd/client/client.go @@ -19,25 +19,29 @@ package main import ( "context" + "crypto/x509" + "encoding/base64" "errors" "flag" "fmt" "net/http" "slices" + fnote "github.com/transparency-dev/formats/note" "github.com/transparency-dev/incubator/vindex/client" "golang.org/x/mod/sumdb/note" "k8s.io/klog/v2" ) var ( - vindexBaseURL = flag.String("vindex_base_url", "", "The base URL of the vindex server.") - inLogBaseURL = flag.String("in_log_base_url", "", "The base URL of the input log.") - lookup = flag.String("lookup", "", "The key to look up in the vindex.") - outLogPubKey = flag.String("out_log_pub_key", "", "The public key to use to verify the output log checkpoint.") - inLogPubKey = flag.String("in_log_pub_key", "", "The public key to use to verify the input log checkpoint.") - inLogOrigin = flag.String("in_log_origin", "", "Optional: allows the Input Log Origin string to be configured to something other than the public key name.") - minIdx = flag.Uint64("min_idx", 0, "The minimum index to look up in the input log.") + vindexBaseURL = flag.String("vindex_base_url", "", "The base URL of the vindex server.") + inLogBaseURL = flag.String("in_log_base_url", "", "The base URL of the input log.") + lookup = flag.String("lookup", "", "The key to look up in the vindex.") + outLogPubKey = flag.String("out_log_pub_key", "", "The public key to use to verify the output log checkpoint. Required.") + inLogPubKey = flag.String("in_log_pub_key", "", "The public key to use to verify the input log checkpoint. Required.") + inLogPubKeyDER = flag.String("in_log_pub_key_der", "", "For CT logs. The public key to use to verify the input log checkpoint. Required, along with in_log_origin.") + inLogOrigin = flag.String("in_log_origin", "", "Required if in_log_pub_key_der is used. Otherwise, allows the Input Log Origin string to be configured to something other than the public key name.") + minIdx = flag.Uint64("min_idx", 0, "The minimum index to look up in the input log.") ) func main() { @@ -104,10 +108,7 @@ func newVIndexClientFromFlags() *client.VIndexClient { if *outLogPubKey == "" { klog.Exitf("out_log_pub_key must be provided") } - inV, err := note.NewVerifier(*inLogPubKey) - if err != nil { - klog.Exitf("failed to construct input log verifier: %v", err) - } + inV := inputLogVerifierFromFlags() outV, err := note.NewVerifier(*outLogPubKey) if err != nil { klog.Exitf("failed to construct output log verifier: %v", err) @@ -123,13 +124,7 @@ func newInputLogClientFromFlags() *client.InputLogClient { if *inLogBaseURL == "" { klog.Exit("in_log_base_url flag must be provided") } - if *inLogPubKey == "" { - klog.Exitf("in_log_pub_key must be provided") - } - v, err := note.NewVerifier(*inLogPubKey) - if err != nil { - klog.Exitf("failed to construct input log verifier: %v", err) - } + v := inputLogVerifierFromFlags() origin := *inLogOrigin if len(origin) == 0 { origin = v.Name() @@ -140,3 +135,34 @@ func newInputLogClientFromFlags() *client.InputLogClient { } return c } + +func inputLogVerifierFromFlags() note.Verifier { + if *inLogPubKey == "" && *inLogPubKeyDER == "" { + klog.Exitf("Must provide exactly one --in_log_pub_key* flag") + } + if *inLogPubKey != "" { + v, err := note.NewVerifier(*inLogPubKey) + if err != nil { + klog.Exitf("failed to construct input log verifier: %v", err) + } + return v + } + derBytes, err := base64.StdEncoding.DecodeString(*inLogPubKeyDER) + if err != nil { + klog.Exitf("Error decoding public key: %s", err) + } + pub, err := x509.ParsePKIXPublicKey(derBytes) + if err != nil { + klog.Exitf("Error parsing public key: %v", err) + } + + verifierKey, err := fnote.RFC6962VerifierString(*inLogOrigin, pub) + if err != nil { + klog.Exitf("Error creating RFC6962 verifier string: %v", err) + } + v, err := fnote.NewVerifier(verifierKey) + if err != nil { + klog.Exitf("Error creating verifier: %v", err) + } + return v +} diff --git a/vindex/cmd/ct/README.md b/vindex/cmd/ct/README.md new file mode 100644 index 0000000..da63d19 --- /dev/null +++ b/vindex/cmd/ct/README.md @@ -0,0 +1,77 @@ +## Verifiable Index: CT + +This is a demo of pulling the contents of a tile-based CT log into a [Verifiable Index](../../README.md). + +[tlog-tiles]: https://c2sp.org/tlog-tiles +[Tessera]: https://github.com/transparency-dev/tessera + +The CT Input Log is processed, with each entry being indexed on all common names defined in the cert. +This allows the owner of a domain to look up all certs for their domain, in a way that is fully verified. + +## Running + +The static CT Input Log is expected to be available for reading at a URL provided by the `--monitoring_url` flag. +This is the base directory that should contain the checkpoint file. +The Verifiable Index and Output Log are constructed locally, persisted to local disk (in the `--storage_dir` directory), and hosted via a web server. + +```shell +OUTPUT_LOG_PRIVATE_KEY=PRIVATE+KEY+example.com/outputlog+07392c46+ATPJ4crkyUbPeaRffN/4NUof3KV0pQznVIPGOQm3SDEJ \ +go run ./vindex/cmd/ct \ + --storage_dir ~/vindex-ct/ \ + --origin="coachandhorses2026h1.staging.certificate.transparency.goog" \ + --public_key="MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAECHOhXfvYgTcu+Fnl7M7niFj3FgqWlQpXUSWUDw2KAaJXvhGxdJTtmyciN5rWTiDtpeNENVmsUTHFS4XQgeRE0g==" \ + --monitoring_url="https://storage.googleapis.com/coachandhorses2026h1.staging.certificate.transparency.goog" +``` + +### Resource Requirements + +Running this demo has the following estimated system requirements (based on indexing ~180M certs from a staging log): +* **Architecture:** **64-bit OS** (Linux/Unix). The MPT library reserves a large virtual address space (16 TB) and cannot run on 32-bit systems. +* **RAM:** **32 GB+** (observed ~31 GB physical RAM usage). The current prototype stores the key-value index in a raw Go map in memory. +* **CPU:** **8+ cores** (actively utilizes ~6 cores during ingestion). +* **Disk:** **100 GB+ SSD** (uses ~45 GB for WAL and MPT files, fast I/O is required). + +> [!NOTE] +> The high memory usage is a limitation of the current prototype's in-memory key-value store. The planned [v1 architecture](../../docs/v1/IMPLEMENTATION.md) will move this store to a disk-backed Pebble database, which is expected to drastically reduce RAM requirements to approximately 6-8 GB. + +Running the above will run a web server hosting the following URLs: + - `/vindex/lookup` - the provisional [vindex lookup API](./api/api.go) + - `/outputlog/` - the [tlog-tiles][] base URL for the output log + +To inspect the log, you can use the woodpecker tool (using the corresponding public key to the private key used above): + +```shell +# To inspect the Output Log +go run github.com/mhutchinson/woodpecker@main --custom_log_type=tiles --custom_log_url=http://localhost:8088/outputlog/ --custom_log_vkey=example.com/outputlog+07392c46+AWyS8y8ZsRmQnTr6Fr2knaa8+t6CPYFh5Ho3wJEr14B8 +``` + +Use left/right cursor to browse, and `q` to quit. + +A domain indexed by the verifiable map can be looked up using the following command: + +```shell +go run ./vindex/cmd/client \ + --vindex_base_url http://localhost:8088/vindex/ \ + --out_log_pub_key=example.com/outputlog+07392c46+AWyS8y8ZsRmQnTr6Fr2knaa8+t6CPYFh5Ho3wJEr14B8 \ + --in_log_pub_key_der=MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAECHOhXfvYgTcu+Fnl7M7niFj3FgqWlQpXUSWUDw2KAaJXvhGxdJTtmyciN5rWTiDtpeNENVmsUTHFS4XQgeRE0g== \ + --in_log_origin=coachandhorses2026h1.staging.certificate.transparency.goog \ + --lookup=google.com + +I0610 15:02:17.112527 87150 client.go:83] in_log_base_url not provided, so cannot dereference pointers +148000245 +151898263 +152014951 +152015244 +152015262 +152015307 +... +154306826 +154307178 +154307232 +154307321 +154368790 +154368845 +``` + +To view the certs at the index, [woodpecker](https://github.com/mhutchinson/woodpecker) can be used. + diff --git a/vindex/cmd/ct/main.go b/vindex/cmd/ct/main.go new file mode 100644 index 0000000..e7f45a6 --- /dev/null +++ b/vindex/cmd/ct/main.go @@ -0,0 +1,416 @@ +// Copyright 2025 Google LLC. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// logandmap is a binary that serves as a demo of how to run a log and a map in the +// same process. +// The log is a Tessera POSIX log, and the map is an in-memory verifiable index. +// A web server is hosted that allows lookups in the map to be performed. +// The log is updated periodically with entries of type LogEntry, and the map keys +// each of the module names from that struct to each of the indices in the log where +// an entry for that module is stored. +package main + +import ( + "context" + "crypto/sha256" + "crypto/x509" + "encoding/base64" + "errors" + "flag" + "fmt" + "iter" + "net/http" + "os" + "os/signal" + "path" + "strings" + "syscall" + "time" + + "golang.org/x/net/publicsuffix" + + "filippo.io/sunlight" + "filippo.io/torchwood" + "github.com/gorilla/mux" + "github.com/transparency-dev/formats/log" + fnote "github.com/transparency-dev/formats/note" + "github.com/transparency-dev/incubator/vindex" + "go.opentelemetry.io/otel/exporters/prometheus" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "golang.org/x/crypto/cryptobyte" + "golang.org/x/mod/sumdb/note" + "golang.org/x/mod/sumdb/tlog" + "k8s.io/klog/v2" +) + +var ( + inputLogUrl = flag.String("monitoring_url", "", "Base URL of the static CT log to index") + origin = flag.String("origin", "", "Origin of the log to check") + pubKey = flag.String("public_key", "", "The log's public key in base64 encoded DER format") + userAgentInfo = flag.String("user_agent_info", "", "Optional string to append to the user agent (e.g. email address for Sunlight logs)") + persistentCacheDir = flag.String("persistent_cache_dir", "", "Optional location of a directory to cache Input Log tiles") + persistIndex = flag.Bool("persist_index", true, "Set to false to use a memory-based implementation of the verifiable index.") + + outputLogPrivKeyFile = flag.String("output_log_private_key", "", "Location of private key file. If unset, uses the contents of the OUTPUT_LOG_PRIVATE_KEY environment variable.") + storageDir = flag.String("storage_dir", "", "Root directory in which to store the data for the demo. This will create subdirectories for the Input Log, Output Log, and allocate space to store the verifiable map persistence.") + listen = flag.String("listen", ":8088", "Address to set up HTTP server listening on") +) + +const ( + userAgent = "TrustFabric VerifiableIndex" +) + +func main() { + klog.InitFlags(nil) + flag.Parse() + + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer cancel() + + if err := run(ctx); err != nil { + klog.Exitf("Run failed: %v", err) + } +} + +func run(ctx context.Context) error { + // Set up storage for the input log, index, and output log. + if *storageDir == "" { + return errors.New("storage_dir must be set") + } + outputLogDir := path.Join(*storageDir, "outputlog") + mapRoot := path.Join(*storageDir, "vindex") + + if err := os.MkdirAll(outputLogDir, 0o755); err != nil { + return fmt.Errorf("failed to create output log directory: %v", err) + } + if err := os.MkdirAll(mapRoot, 0o755); err != nil { + return fmt.Errorf("failed to create vindex directory: %v", err) + } + + exporter, err := prometheus.New() + if err != nil { + return fmt.Errorf("failed to create prometheus exporter: %v", err) + } + provider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(exporter)) + defer func() { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := provider.Shutdown(shutdownCtx); err != nil { + klog.Errorf("failed to shutdown meter provider: %v", err) + } + }() + + outputLog, outputCloser := outputLogOrDie(ctx, outputLogDir) + defer outputCloser(ctx) + + inputLog := newStaticCTInputLogFromFlags() + + vi, err := vindex.NewVerifiableIndex(ctx, inputLog, mapFn, outputLog, mapRoot, vindex.Options{ + PersistIndex: *persistIndex, + MeterProvider: provider, + }) + if err != nil { + return fmt.Errorf("failed to create vindex: %v", err) + } + klog.Info("Created verifiable index") + + // Keeps the map synced with the latest published input log state. + go maintainMap(ctx, vi) + + // Run a web server to serve the input log, index, and output log. + go runWebServer(vi, outputLogDir) + <-ctx.Done() + return nil +} + +func cutEntry(tile []byte) (entry []byte, rh tlog.Hash, rest []byte, err error) { + // This implementation is terribly inefficient, parsing the whole entry just + // to re-serialize and throw it away. If this function shows up in profiles, + // let me know and I'll improve it. + e, rest, err := sunlight.ReadTileLeaf(tile) + if err != nil { + return nil, tlog.Hash{}, nil, err + } + rh = tlog.RecordHash(e.MerkleTreeLeaf()) + entry = tile[:len(tile)-len(rest)] + return entry, rh, rest, nil +} + +func newStaticCTInputLogFromFlags() *staticCTInputLog { + ua := userAgent + if *userAgentInfo != "" { + ua = fmt.Sprintf("%s (%s)", userAgent, *userAgentInfo) + } + fetcher, err := torchwood.NewTileFetcher(*inputLogUrl, + torchwood.WithTilePath(sunlight.TilePath), + torchwood.WithUserAgent(ua)) + if err != nil { + klog.Exitf("failed to create client: %v", err) + } + var tileReader torchwood.TileReader = fetcher + if *persistentCacheDir != "" { + tileReader, err = torchwood.NewPermanentCache(fetcher, *persistentCacheDir) + if err != nil { + klog.Exitf("failed to create permanent cache: %v", err) + } + } + client, err := torchwood.NewClient(tileReader, torchwood.WithCutEntry(cutEntry)) + if err != nil { + klog.Exitf("failed to create client: %v", err) + } + return &staticCTInputLog{ + c: client, + f: fetcher, + v: verifierFromFlags(), + } +} + +type staticCTInputLog struct { + c *torchwood.Client + f *torchwood.TileFetcher + v note.Verifier + + lastCheckpoint log.Checkpoint +} + +func (l *staticCTInputLog) Checkpoint(ctx context.Context) (checkpoint []byte, err error) { + return l.f.ReadEndpoint(ctx, "checkpoint") +} + +// Parse unmarshals and verifies a checkpoint obtained from GetCheckpoint. +func (l *staticCTInputLog) Parse(checkpoint []byte) (*log.Checkpoint, error) { + cp, _, _, err := log.ParseCheckpoint(checkpoint, l.v.Name(), l.v) + if err != nil { + return nil, err + } + l.lastCheckpoint = *cp + return cp, err +} + +// Leaves returns all the leaves in the range [start, end), outputting them via +// the returned iterator. +func (l *staticCTInputLog) Leaves(ctx context.Context, start, end uint64) iter.Seq2[[]byte, error] { + tree := tlog.Tree{ + N: int64(end), + Hash: tlog.Hash(l.lastCheckpoint.Hash), + } + return func(yield func([]byte, error) bool) { + for _, entry := range l.c.Entries(ctx, tree, int64(start)) { + e, _, err := sunlight.ReadTileLeaf(entry) + if err != nil { + if !yield(nil, err) { + return + } + } + if !yield(e.MerkleTreeLeaf(), nil) { + return + } + } + if err := l.c.Err(); err != nil { + yield(nil, l.c.Err()) + } + } +} + +// outputLogOrDie returns an output log using a POSIX log in the given directory. +func outputLogOrDie(ctx context.Context, outputLogDir string) (log vindex.OutputLog, closer func(context.Context)) { + s, v := getOutputLogSignerVerifierOrDie() + + l, c, err := vindex.NewOutputLog(ctx, outputLogDir, s, v, vindex.OutputLogOpts{}) + if err != nil { + klog.Exit(err) + } + return l, c +} + +func verifierFromFlags() note.Verifier { + if *origin == "" { + klog.Exitf("Must provide the --origin flag") + } + if *pubKey == "" { + klog.Exitf("Must provide the --pub_key flag") + } + derBytes, err := base64.StdEncoding.DecodeString(*pubKey) + if err != nil { + klog.Exitf("Error decoding public key: %s", err) + } + pub, err := x509.ParsePKIXPublicKey(derBytes) + if err != nil { + klog.Exitf("Error parsing public key: %v", err) + } + + verifierKey, err := fnote.RFC6962VerifierString(*origin, pub) + if err != nil { + klog.Exitf("Error creating RFC6962 verifier string: %v", err) + } + logSigV, err := fnote.NewVerifier(verifierKey) + if err != nil { + klog.Exitf("Error creating verifier: %v", err) + } + + klog.Infof("Using verifier string: %v", verifierKey) + + return logSigV +} + +// maintainMap reads entries from the log and sync them to the vindex. +func maintainMap(ctx context.Context, vi *vindex.VerifiableIndex) { + ticker := time.NewTicker(10 * time.Second) + defer ticker.Stop() + + for { + if err := vi.Update(ctx); err != nil { + klog.Warning(err) + } + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } +} + +func runWebServer(vi *vindex.VerifiableIndex, outLogDir string) { + web := NewServer(vi.Lookup) + + olfs := http.FileServer(http.Dir(outLogDir)) + r := mux.NewRouter() + r.PathPrefix("/outputlog/").Handler(http.StripPrefix("/outputlog/", olfs)) + web.registerHandlers(r) + hServer := &http.Server{ + Addr: *listen, + Handler: r, + } + go func() { + _ = hServer.ListenAndServe() + }() + klog.Infof("Started HTTP server listening on %s", *listen) +} + +// Read output log private key from file or environment variable and generate the +// note Signer and Verifier pair for it. +func getOutputLogSignerVerifierOrDie() (note.Signer, note.Verifier) { + var privKey string + var err error + if len(*outputLogPrivKeyFile) > 0 { + privKey, err = getKeyFile(*outputLogPrivKeyFile) + if err != nil { + klog.Exitf("Unable to get private key: %v", err) + } + } else { + privKey = os.Getenv("OUTPUT_LOG_PRIVATE_KEY") + if len(privKey) == 0 { + klog.Exit("Supply private key file path using --output_log_private_key or set OUTPUT_LOG_PRIVATE_KEY environment variable") + } + } + s, v, err := fnote.NewEd25519SignerVerifier(privKey) + if err != nil { + klog.Exitf("Failed to get signer/verifier: %v", err) + } + return s, v +} + +func getKeyFile(path string) (string, error) { + k, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("failed to read key file: %w", err) + } + return string(k), nil +} + +func mapFn(data []byte) [][sha256.Size]byte { + s := cryptobyte.String(data) + + var version, leafType uint8 + var timestamp uint64 + var certType uint16 + if !s.ReadUint8(&version) || !s.ReadUint8(&leafType) || !s.ReadUint64(×tamp) || !s.ReadUint16(&certType) { + klog.Warningf("Failed to unmarshal headers") + // This should return a sentinel value (e.g. all zero hash) so unprocessable entries can be found + return nil + } + var isPreCert bool + var cert cryptobyte.String + switch certType { + case 0: + // x509 + isPreCert = false + s.ReadUint24LengthPrefixed(&cert) + case 1: + if true { + // Need to support parsing TBS certs + return nil + } + // precert + isPreCert = true + var ikh []byte + s.ReadBytes(&ikh, sha256.Size) + s.ReadUint24LengthPrefixed(&cert) + default: + panic("unknown cert type") + } + + parsedCert, err := x509.ParseCertificate(cert) + if err != nil { + klog.Warningf("failed to parse x509 cert (preCert=%t): %v", isPreCert, err) + // This could return a sentinel value (e.g. all zero hash) so unprocessable entries can be found + return nil + } + if klog.V(2).Enabled() { + klog.V(2).Info(parsedCert.DNSNames) + } + uniqueNames := make(map[string]bool) + for _, cn := range parsedCert.DNSNames { + cn = strings.ToLower(cn) + if strings.HasPrefix(cn, "*.") { + cn = cn[2:] + } else if strings.HasPrefix(cn, "*") { + cn = cn[1:] + } + if cn == "" { + continue + } + uniqueNames[cn] = true + + // TODO(mhutchinson): This is the most controversial part - the eTLD evolves. + // We can lock this down with WASM, but what to do when the ecosystem needs the new changes? + etld1, err := publicsuffix.EffectiveTLDPlusOne(cn) + if err != nil { + continue + } + if cn == etld1 { + continue + } + curr := cn + for { + idx := strings.Index(curr, ".") + if idx == -1 { + break + } + curr = curr[idx+1:] + if len(curr) < len(etld1) { + break + } + uniqueNames[curr] = true + if curr == etld1 { + break + } + } + } + hashes := make([][sha256.Size]byte, 0, len(uniqueNames)) + for name := range uniqueNames { + hashes = append(hashes, sha256.Sum256([]byte(name))) + } + return hashes +} diff --git a/vindex/cmd/ct/main_test.go b/vindex/cmd/ct/main_test.go new file mode 100644 index 0000000..df6c638 --- /dev/null +++ b/vindex/cmd/ct/main_test.go @@ -0,0 +1,148 @@ +// Copyright 2026 Google LLC. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "crypto/rand" + "crypto/rsa" + "crypto/sha256" + "crypto/x509" + "crypto/x509/pkix" + "math/big" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/gorilla/mux" + "github.com/transparency-dev/incubator/vindex/api" + "golang.org/x/crypto/cryptobyte" +) + +func createTestCertBytes(dnsNames []string) ([]byte, error) { + priv, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return nil, err + } + template := x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{ + Organization: []string{"Test Org"}, + }, + NotBefore: time.Now().Add(-1 * time.Hour), + NotAfter: time.Now().Add(1 * time.Hour), + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + DNSNames: dnsNames, + } + certDER, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv) + if err != nil { + return nil, err + } + + // Now marshal using cryptobyte into the form expected by mapFn + // version = 0, leafType = 0, timestamp = 12345678, certType = 0 (x509) + var b cryptobyte.Builder + b.AddUint8(0) // version + b.AddUint8(0) // leafType + b.AddUint64(12345678) // timestamp + b.AddUint16(0) // certType (x509) + b.AddUint24LengthPrefixed(func(b *cryptobyte.Builder) { + b.AddBytes(certDER) + }) + return b.Bytes() +} + +func TestMapFn(t *testing.T) { + testCases := []struct { + desc string + dnsNames []string + wantKeys []string + }{ + { + desc: "standard domains and wildcards", + dnsNames: []string{"*.google.com", "google.com", "maps.google.com"}, + wantKeys: []string{"google.com", "maps.google.com"}, + }, + { + desc: "deeper subdomain", + dnsNames: []string{"deep.maps.google.co.uk"}, + wantKeys: []string{"deep.maps.google.co.uk", "maps.google.co.uk", "google.co.uk"}, + }, + { + desc: "mixed case", + dnsNames: []string{"MAPS.GOOGLE.COM"}, + wantKeys: []string{"maps.google.com", "google.com"}, + }, + { + desc: "invalid or TLD", + dnsNames: []string{"localhost", "*.co.uk"}, + wantKeys: []string{"localhost", "co.uk"}, + }, + } + + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + data, err := createTestCertBytes(tc.dnsNames) + if err != nil { + t.Fatalf("failed to create test cert: %v", err) + } + hashes := mapFn(data) + + gotKeys := make(map[string]bool) + for _, h := range hashes { + found := false + for _, wk := range tc.wantKeys { + if sha256.Sum256([]byte(wk)) == h { + gotKeys[wk] = true + found = true + break + } + } + if !found { + t.Errorf("got unexpected hash for a key not in wantKeys") + } + } + + for _, wk := range tc.wantKeys { + if !gotKeys[wk] { + t.Errorf("missing expected key: %s", wk) + } + } + if len(gotKeys) != len(tc.wantKeys) { + t.Errorf("got %d unique keys, want %d", len(gotKeys), len(tc.wantKeys)) + } + }) + } +} + +func TestMetricsEndpoint(t *testing.T) { + s := NewServer(func(ctx context.Context, h [sha256.Size]byte) (api.LookupResponse, error) { + return api.LookupResponse{}, nil + }) + r := mux.NewRouter() + s.registerHandlers(r) + + req := httptest.NewRequest("GET", "/metrics", nil) + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Errorf("GET /metrics: got status %d, want %d", w.Code, http.StatusOK) + } +} + diff --git a/vindex/cmd/ct/web.go b/vindex/cmd/ct/web.go new file mode 100644 index 0000000..7fc62c2 --- /dev/null +++ b/vindex/cmd/ct/web.go @@ -0,0 +1,78 @@ +// Copyright 2025 Google LLC. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "crypto/sha256" + _ "embed" + "encoding/hex" + "encoding/json" + "fmt" + "net/http" + + "github.com/gorilla/mux" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/transparency-dev/incubator/vindex/api" + "k8s.io/klog/v2" +) + +func NewServer(lookup func(context.Context, [sha256.Size]byte) (api.LookupResponse, error)) Server { + return Server{ + lookup: lookup, + } +} + +type Server struct { + lookup func(context.Context, [sha256.Size]byte) (api.LookupResponse, error) +} + +// handleLookup handles GET requests for looking up map entries. +func (s Server) handleLookup(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + hashStr, ok := vars["hash"] + if !ok { + http.Error(w, "hash parameter not found", http.StatusBadRequest) + return + } + + h, err := hex.DecodeString(hashStr) + if err != nil { + http.Error(w, fmt.Sprintf("invalid hex hash: %v", err), http.StatusBadRequest) + return + } + if l := len(h); l != sha256.Size { + http.Error(w, fmt.Sprintf("hash wrong length (decoded %d bytes)", l), http.StatusBadRequest) + return + } + + klog.V(2).Infof("Received hash from request: '%s'", h) + + resp, err := s.lookup(r.Context(), [sha256.Size]byte(h)) + if err != nil { + http.Error(w, fmt.Sprintf("lookup failed: %v", err), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + if err := json.NewEncoder(w).Encode(resp); err != nil { + klog.Warningf("failed to encode response: %v", err) + } +} + +func (s Server) registerHandlers(r *mux.Router) { + r.HandleFunc("/vindex/lookup/{hash}", s.handleLookup).Methods("GET") + r.Handle("/metrics", promhttp.Handler()) +} diff --git a/vindex/outputlog_test.go b/vindex/outputlog_test.go index d55e67f..d002767 100644 --- a/vindex/outputlog_test.go +++ b/vindex/outputlog_test.go @@ -80,7 +80,7 @@ func TestOutputLog_Lookup(t *testing.T) { t.Fatal(err) } defer func() { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) defer cancel() closer(ctx) }()