OCP: support serving bundles directly

.github/workflows/pull-request.yaml

@@ -78,6 +78,7 @@ jobs:
           retry() { for i in $(seq 1 30); do if "$@"; then return 0; fi; echo "attempt $i failed, retrying in 5s..."; sleep 5; done; return 1; }
           retry sh -c 'curl --fail -s http://127.0.0.1:8282/v1/bundles -H "Authorization: bearer sesame" | jq -e ".result | length > 0"'
           retry sh -c 'curl --fail -s http://127.0.0.1:8181/v1/data/k8s/authz/decision/status/allowed -d "{\"input\":{\"apiVersion\":\"v1\"}}" | jq -e ".result == true"'
+          retry sh -c 'curl --fail -s http://127.0.0.1:8181/v1/data/example/allow -d "{\"input\":{\"method\":\"POST\",\"path\":[\"posts\"],\"subject\":{\"user\":\"alice\"}}}" | jq -e ".result == true"'
       - name: dump logs
         run: docker compose logs
         working-directory: examples/docker

cmd/backtest/backtest.go

@@ -28,6 +28,7 @@ import (
 	"github.com/open-policy-agent/opa-control-plane/internal/logging"
 	"github.com/open-policy-agent/opa-control-plane/internal/progress"
 	"github.com/open-policy-agent/opa-control-plane/internal/s3"
+	objectstore "github.com/open-policy-agent/opa-control-plane/pkg/objectstorage"
 	"github.com/open-policy-agent/opa/ast"    // nolint:staticcheck
 	"github.com/open-policy-agent/opa/bundle" // nolint:staticcheck
 	"github.com/open-policy-agent/opa/sdk"    // nolint:staticcheck
@@ -311,7 +312,12 @@ func backtestBundle(ctx context.Context, opts Options, styra *das.Client, b *con
 		return err
 	}
 
-	r, err := s.Download(ctx)
+	d, ok := s.(objectstore.Downloader)
+	if !ok {
+		return fmt.Errorf("object storage for %q does not support download", b.Name)
+	}
+
+	r, err := d.Download(ctx)
 	if err != nil {
 		return err
 	}

cmd/compare/compare.go

@@ -25,6 +25,7 @@ import (
 	"github.com/open-policy-agent/opa-control-plane/internal/config"
 	"github.com/open-policy-agent/opa-control-plane/internal/logging"
 	"github.com/open-policy-agent/opa-control-plane/internal/s3"
+	objectstore "github.com/open-policy-agent/opa-control-plane/pkg/objectstorage"
 )
 
 var log *logging.Logger
@@ -214,7 +215,11 @@ func compareSystem(ctx context.Context, client *das.Client, v1 *das.V1System, sy
 	if err != nil {
 		return nil, err
 	}
-	r, err := s.Download(ctx)
+	d, ok := s.(objectstore.Downloader)
+	if !ok {
+		return nil, fmt.Errorf("object storage for %q does not support download", system.Name)
+	}
+	r, err := d.Download(ctx)
 	if err != nil {
 		return nil, err
 	}

cmd/run/run.go

@@ -71,7 +71,13 @@ func init() {
 			}
 
 			go func() {
-				if err := server.New().WithDatabase(svc.Database()).WithReadiness(svc.Ready).WithConfig(config.Service).Init().ListenAndServe(params.addr); err != nil {
+				srv := server.New().
+					WithDatabase(svc.Database()).
+					WithReadiness(svc.Ready).
+					WithConfig(config.Service).
+					WithBundleStorages(svc.BundleStorages()).
+					Init()
+				if err := srv.ListenAndServe(params.addr); err != nil {
 					log.Fatalf("failed to start server: %v", err)
 				}
 			}()

config/schema.json

@@ -220,6 +220,14 @@
       },
       "type": "object"
     },
+    "ConfigHTTPServer": {
+      "properties": {
+        "path": {
+          "type": "string"
+        }
+      },
+      "type": "object"
+    },
     "ConfigLabels": {
       "additionalProperties": {
         "type": "string"
@@ -251,6 +259,9 @@
         },
         "gcp": {
           "$ref": "#/definitions/ConfigGCPCloudStorage"
+        },
+        "http_server": {
+          "$ref": "#/definitions/ConfigHTTPServer"
         }
       },
       "type": "object"
@@ -345,7 +356,8 @@
             "administrator",
             "viewer",
             "owner",
-            "stack_owner"
+            "stack_owner",
+            "downloader"
           ],
           "type": "string"
         }

e2e/cli/run_http_server.txtar

@@ -0,0 +1,63 @@
+! exec $OPACTL run --addr ./ocp.sock --config config.d/bundle.yml --data-dir tmp &opactl&
+
+exec curl --retry 5 --retry-all-errors --unix-socket ocp.sock http://localhost/health
+
+# Wait for bundle to be built and download it
+retry curl -f --unix-socket ocp.sock -H 'Authorization: Bearer test-token' http://localhost/v1/external/bundles/hello-world/bundle.tar.gz -o bundle.tar.gz
+
+# Verify the bundle is a valid tarball with expected contents
+exec tar tf bundle.tar.gz
+cmp stdout exp/tarball
+
+# Verify ETag header is returned
+exec curl -f -D headers.txt --unix-socket ocp.sock -H 'Authorization: Bearer test-token' http://localhost/v1/external/bundles/hello-world/bundle.tar.gz -o /dev/null
+exec grep Etag headers.txt
+
+# Verify unauthenticated request is rejected
+! exec curl -f --unix-socket ocp.sock http://localhost/v1/external/bundles/hello-world/bundle.tar.gz
+
+# Verify bad token is rejected
+! exec curl -f --unix-socket ocp.sock -H 'Authorization: Bearer wrong-token' http://localhost/v1/external/bundles/hello-world/bundle.tar.gz
+
+# Verify downloader-only token can download bundles
+exec curl -f --unix-socket ocp.sock -H 'Authorization: Bearer dl-token' http://localhost/v1/external/bundles/hello-world/bundle.tar.gz -o /dev/null
+
+# Verify downloader-only token cannot access the API
+! exec curl -f --unix-socket ocp.sock -H 'Authorization: Bearer dl-token' http://localhost/v1/bundles/hello-world
+
+# Verify unknown bundle path returns 404
+! exec curl -f --unix-socket ocp.sock -H 'Authorization: Bearer test-token' http://localhost/v1/external/bundles/nonexistent/bundle.tar.gz
+
+kill opactl
+wait opactl
+
+-- files/sources/hello-world/rules/rules.rego --
+package rules
+import rego.v1
+result if input.yay
+-- config.d/bundle.yml --
+tokens:
+  test:
+    api_key: test-token
+    scopes:
+    - role: viewer
+  downloader:
+    api_key: dl-token
+    scopes:
+    - role: downloader
+bundles:
+  hello-world:
+    object_storage:
+      http_server:
+        path: bundles/hello-world/bundle.tar.gz
+    requirements:
+    - source: hello-world
+sources:
+  hello-world:
+    directory: files/sources/hello-world
+    paths:
+    - rules/rules.rego
+-- exp/tarball --
+/data.json
+/hello-world/rules/rules.rego
+/.manifest

e2e/migrate_e2e_test.go

@@ -28,6 +28,7 @@ import (
 	ocp_fs "github.com/open-policy-agent/opa-control-plane/internal/fs"
 	"github.com/open-policy-agent/opa-control-plane/internal/logging"
 	"github.com/open-policy-agent/opa-control-plane/internal/s3"
+	objectstore "github.com/open-policy-agent/opa-control-plane/pkg/objectstorage"
 	"github.com/open-policy-agent/opa-control-plane/pkg/service"
 	"github.com/open-policy-agent/opa-control-plane/internal/test/tempfs"
 	"github.com/open-policy-agent/opa-control-plane/libraries"
@@ -653,7 +654,12 @@ func TestMigration(t *testing.T) {
 						t.Fatal(err)
 					}
 
-					r, err := s.Download(ctx)
+					d, ok := s.(objectstore.Downloader)
+					if !ok {
+						t.Fatal("object storage does not support download")
+					}
+
+					r, err := d.Download(ctx)
 					if err != nil {
 						t.Fatal(err)
 					}

examples/docker/README.md

@@ -27,7 +27,12 @@ This will generate the certs needed for the examples (via `tls/gencerts.sh`), an
 When it's running, you can go to http://127.0.0.1:9090 to examine the published Prometheus metrics.
 Enter `ocp_` in the expression field to see completion options for the various metrics in the expression field to see completion options for the various metrics in the expression field to see completion options for the various metrics.
 
-The OCP configuration already contains a bundle, pulling some rego from https://github.com/open-policy-agent/contrib, so that there are some metrics to explore.
+The OCP configuration contains two bundles:
+
+1. `hello-world` — pushed to S3, OPA pulls from s3proxy
+2. `hello-http` — served directly by OCP via in-memory `http_server` storage, OPA pulls from OCP using a `downloader` token
+
+Both pull rego from https://github.com/open-policy-agent/contrib, so that there are some metrics to explore.
 
 > [!WARNING]
 > Note that on startup, it will take a while until the system settles:

examples/docker/ocp.yml

@@ -9,18 +9,34 @@ bundles:
         credentials: s3-creds
     requirements:
     - source: git-policies
+  hello-http:
+    object_storage:
+      http_server:
+        path: bundles/hello-http/bundle.tar.gz
+    requirements:
+    - source: http-policies
 sources:
   git-policies:
     git:
       repo: https://github.com/open-policy-agent/contrib
       commit: 0f81d9a0018451d98dcd3f4bb885ee676f49f6fe
       included_files:
       - k8s_authorization/policy/policy.rego
+  http-policies:
+    git:
+      repo: https://github.com/open-policy-agent/contrib
+      commit: 0f81d9a0018451d98dcd3f4bb885ee676f49f6fe
+      included_files:
+      - data_filter_example/example.rego
 tokens:
   admin:
     api_key: sesame
     scopes:
     - role: administrator
+  opa:
+    api_key: opa-token
+    scopes:
+    - role: downloader
 database:
   sql:
     driver: postgres

examples/docker/opa.yml

@@ -1,7 +1,14 @@
 services:
   s3:
     url: http://s3proxy:80
+  ocp:
+    url: http://ocp:8282
+    headers:
+      Authorization: Bearer opa-token
 bundles:
   hello-world:
     service: s3
     resource: bundles/hello-world
+  hello-http:
+    service: ocp
+    resource: v1/external/bundles/hello-http/bundle.tar.gz

internal/authz/authz.rego

@@ -14,6 +14,7 @@ allow if {
 	in_tenant(data.principals.tenant_id)
 	input.permission in [
 		"bundles.view",
+		"bundles.download",
 		"sources.view",
 		"secrets.view",
 		"stacks.view",
@@ -40,6 +41,13 @@ allow if {
 	input.permission == "stacks.create"
 }
 
+allow if {
+	data.principals.id == input.principal
+	data.principals.role == "downloader"
+	in_tenant(data.principals.tenant_id)
+	input.permission == "bundles.download"
+}
+
 allow if {
 	data.resource_permissions.name == input.name
 	data.resource_permissions.resource == input.resource

internal/authz/authz_test.rego

@@ -14,6 +14,7 @@ test_admin_can_do_anything if {
 
 read_permissions := {
 	"bundles.view",
+	"bundles.download",
 	"sources.view",
 	"stacks.view",
 	"secrets.view",
@@ -129,3 +130,36 @@ test_explicit_permission_grant if {
 		with input.resource as "sources"
 		with input.tenant as "ten10"
 }
+
+test_downloader_can_download_bundles if {
+	data.authz.allow with input.principal as "testuser"
+		with data.principals.id as "testuser"
+		with data.principals.role as "downloader"
+		with data.principals.tenant_id as 10
+		with data.tenants.id as 10
+		with data.tenants.name as "ten10"
+		with input.tenant as "ten10"
+		with input.permission as "bundles.download"
+}
+
+test_downloader_cannot_view_bundles if {
+	not data.authz.allow with input.principal as "testuser"
+		with data.principals.id as "testuser"
+		with data.principals.role as "downloader"
+		with data.principals.tenant_id as 10
+		with data.tenants.id as 10
+		with data.tenants.name as "ten10"
+		with input.tenant as "ten10"
+		with input.permission as "bundles.view"
+}
+
+test_downloader_cannot_view_sources if {
+	not data.authz.allow with input.principal as "testuser"
+		with data.principals.id as "testuser"
+		with data.principals.role as "downloader"
+		with data.principals.tenant_id as 10
+		with data.tenants.id as 10
+		with data.tenants.name as "ten10"
+		with input.tenant as "ten10"
+		with input.permission as "sources.view"
+}

internal/config/config.go

@@ -35,6 +35,7 @@ type (
 	GCPCloudStorage   = extconfig.GCPCloudStorage
 	AzureBlobStorage  = extconfig.AzureBlobStorage
 	FileSystemStorage = extconfig.FileSystemStorage
+	HTTPServer        = extconfig.HTTPServer
 	StringSet         = extconfig.StringSet
 	Requirements      = extconfig.Requirements
 	Files             = extconfig.Files
@@ -449,7 +450,7 @@ func (t *Token) Equal(other *Token) bool {
 }
 
 type Scope struct {
-	Role string `json:"role" enum:"administrator,viewer,owner,stack_owner"`
+	Role string `json:"role" enum:"administrator,viewer,owner,stack_owner,downloader"`
 }
 
 func scopesEqual(a, b []Scope) bool {