diff --git a/ociauth/auth.go b/ociauth/auth.go index e83e9f6..4c7d140 100644 --- a/ociauth/auth.go +++ b/ociauth/auth.go @@ -151,9 +151,8 @@ func (a *stdTransport) RoundTrip(req *http.Request) (*http.Response, error) { ctx := req.Context() requiredScope := RequestInfoFromContext(ctx).RequiredScope - wantScope := ScopeFromContext(ctx) - if err := r.setAuthorization(ctx, req, requiredScope, wantScope); err != nil { + if err := r.setAuthorization(ctx, req, requiredScope); err != nil { return nil, err } resp, err := r.transport.RoundTrip(req) @@ -171,7 +170,7 @@ func (a *stdTransport) RoundTrip(req *http.Request) (*http.Response, error) { if challenge == nil { return resp, nil } - authAdded, tokenAcquired, err := r.setAuthorizationFromChallenge(ctx, req, challenge, requiredScope, wantScope) + authAdded, tokenAcquired, err := r.setAuthorizationFromChallenge(ctx, req, challenge) if err != nil { resp.Body.Close() return nil, err @@ -218,7 +217,7 @@ func (a *stdTransport) RoundTrip(req *http.Request) (*http.Response, error) { // setAuthorization sets up authorization on the given request using any // auth information currently available. -func (r *registry) setAuthorization(ctx context.Context, req *http.Request, requiredScope, wantScope Scope) error { +func (r *registry) setAuthorization(ctx context.Context, req *http.Request, requiredScope Scope) error { r.mu.Lock() defer r.mu.Unlock() // Remove tokens that have expired or will expire soon so that @@ -247,7 +246,7 @@ func (r *registry) setAuthorization(ctx context.Context, req *http.Request, requ // acquiring several tokens concurrently. We should relax the lock // to allow that. - accessToken, err := r.acquireAccessToken(ctx, requiredScope, wantScope) + accessToken, err := r.acquireAccessToken(ctx, requiredScope) if err != nil { // Avoid using %w to wrap the error because we don't want the // caller of RoundTrip (usually ociclient) to assume that the @@ -264,7 +263,7 @@ func (r *registry) setAuthorization(ctx context.Context, req *http.Request, requ return nil } -func (r *registry) setAuthorizationFromChallenge(ctx context.Context, req *http.Request, challenge *authHeader, requiredScope, wantScope Scope) (authAdded, tokenAcquired bool, _ error) { +func (r *registry) setAuthorizationFromChallenge(ctx context.Context, req *http.Request, challenge *authHeader) (authAdded, tokenAcquired bool, _ error) { r.mu.Lock() defer r.mu.Unlock() r.wwwAuthenticate = challenge @@ -272,7 +271,7 @@ func (r *registry) setAuthorizationFromChallenge(ctx context.Context, req *http. switch { case r.wwwAuthenticate.scheme == "bearer": scope := ParseScope(r.wwwAuthenticate.params["scope"]) - accessToken, err := r.acquireAccessToken(ctx, scope, wantScope.Union(requiredScope)) + accessToken, err := r.acquireAccessToken(ctx, scope) if err != nil { return false, false, err } @@ -320,41 +319,14 @@ func (r *registry) init() error { } // acquireAccessToken tries to acquire an access token for authorizing a request. -// The requiredScopeStr parameter indicates the scope that's definitely -// required. This is a string because apparently some servers are picky -// about getting exactly the same scope in the auth request that was -// returned in the challenge. The wantScope parameter indicates -// what scope might be required in the future. +// The scope comes from the registry's Www-Authenticate challenge. // // This method assumes that there has been a previous 401 response with // a Www-Authenticate: Bearer... header. -func (r *registry) acquireAccessToken(ctx context.Context, requiredScope, wantScope Scope) (string, error) { - scope := requiredScope.Union(wantScope) +func (r *registry) acquireAccessToken(ctx context.Context, scope Scope) (string, error) { tok, err := r.acquireToken(ctx, scope) if err != nil { - var herr oci.HTTPError - if !errors.As(err, &herr) || herr.StatusCode() != http.StatusUnauthorized { - return "", err - } - // The documentation says this: - // - // If the client only has a subset of the requested - // access it _must not be considered an error_ as it is - // not the responsibility of the token server to - // indicate authorization errors as part of this - // workflow. - // - // However it's apparently not uncommon for servers to reject - // such requests anyway, so if we've got an unauthorized error - // and wantScope goes beyond requiredScope, it may be because - // the server is rejecting the request. - scope = requiredScope - tok, err = r.acquireToken(ctx, scope) - if err != nil { - return "", err - } - // TODO mark the registry as picky about tokens so we don't - // attempt twice every time? + return "", err } if tok.RefreshToken != "" { r.refreshToken = tok.RefreshToken diff --git a/ociauth/auth_test.go b/ociauth/auth_test.go index b223e61..29c3d92 100644 --- a/ociauth/auth_test.go +++ b/ociauth/auth_test.go @@ -100,9 +100,9 @@ func TestBearerAuth(t *testing.T) { assertRequest(context.Background(), t, ts, "/test", client, Scope{}) } -func TestBearerAuthAdditionalScope(t *testing.T) { - // This tests the scenario where there's a larger scope in the context - // than the required scope. +func TestBearerAuthAdditionalScopeDoesNotOverrideChallenge(t *testing.T) { + // This tests that additional context scope is not unioned into the + // registry-provided challenge scope. requiredScope := ParseScope("repository:foo:push,pull") additionalScope := ParseScope("repository:bar:pull somethingElse") authSrv := newAuthServer(t, func(req *http.Request) (any, *httpError) { @@ -114,8 +114,7 @@ func TestBearerAuthAdditionalScope(t *testing.T) { } requestedScope := ParseScope(strings.Join(req.Form["scope"], " ")) runNonFatal(t, func(t testing.TB) { - wantScope := requiredScope.Union(additionalScope) - require.True(t, wantScope.Equal(requestedScope), "scope mismatch: got %v, want %v", requestedScope, wantScope) + require.True(t, requiredScope.Equal(requestedScope), "scope mismatch: got %v, want %v", requestedScope, requiredScope) require.Equal(t, []string{"someService"}, req.Form["service"]) }) return &wireToken{ @@ -132,8 +131,7 @@ func TestBearerAuthAdditionalScope(t *testing.T) { } } runNonFatal(t, func(t testing.TB) { - wantScope := requiredScope.Union(additionalScope) - require.True(t, wantScope.Equal(authScopeFromRequest(t, req)), "scope mismatch") + require.True(t, requiredScope.Equal(authScopeFromRequest(t, req)), "scope mismatch") }) return nil }) @@ -418,10 +416,14 @@ func TestLaterRequestCanUseEarlierTokenWithLargerScope(t *testing.T) { Action: ActionPull, }) if req.Header.Get("Authorization") == "" { + challengeScope := requiredScope + if resource == "foo1" { + challengeScope = ParseScope("repository:foo1:pull repository:foo2:pull") + } return &httpError{ statusCode: http.StatusUnauthorized, header: http.Header{ - "Www-Authenticate": []string{fmt.Sprintf("Bearer realm=%q,service=someService,scope=%q", authSrv, requiredScope)}, + "Www-Authenticate": []string{fmt.Sprintf("Bearer realm=%q,service=someService,scope=%q", authSrv, challengeScope)}, }, } } @@ -438,15 +440,72 @@ func TestLaterRequestCanUseEarlierTokenWithLargerScope(t *testing.T) { }), }), } - ctx := ContextWithScope(context.Background(), ParseScope("repository:foo1:pull repository:foo2:pull")) - assertRequest(ctx, t, ts, "/test/foo1", client, Scope{}) - assertRequest(ctx, t, ts, "/test/foo2", client, Scope{}) + assertRequest(context.Background(), t, ts, "/test/foo1", client, Scope{}) + assertRequest(context.Background(), t, ts, "/test/foo2", client, Scope{}) // One token fetch should have been sufficient for both requests. require.Equal(t, 1, authCount) } +func TestLaterRequestCanAcquireTokenProactively(t *testing.T) { + authCount := 0 + authSrv := newAuthServer(t, func(req *http.Request) (any, *httpError) { + authCount++ + requestedScope := ParseScope(strings.Join(req.Form["scope"], " ")) + return &wireToken{ + Token: token{requestedScope}.String(), + }, nil + }) + targetCount := 0 + ts := newTargetServer(t, func(req *http.Request) *httpError { + targetCount++ + resource := strings.TrimPrefix(req.URL.Path, "/test/") + requiredScope := NewScope(ResourceScope{ + ResourceType: TypeRepository, + Resource: resource, + Action: ActionPull, + }) + if req.Header.Get("Authorization") == "" { + return &httpError{ + statusCode: http.StatusUnauthorized, + header: http.Header{ + "Www-Authenticate": []string{fmt.Sprintf("Bearer realm=%q,service=someService,scope=%q", authSrv, requiredScope)}, + }, + } + } + runNonFatal(t, func(t testing.TB) { + requestScope := authScopeFromRequest(t, req) + require.True(t, requestScope.Contains(requiredScope), "request scope: %q; required scope: %q", requestScope, requiredScope) + }) + return nil + }) + client := &http.Client{ + Transport: NewStdTransport(StdTransportParams{ + Config: configFunc(func(host string) (ConfigEntry, error) { + if host == ts.Host { + return ConfigEntry{ + RefreshToken: "someRefreshToken", + }, nil + } + return ConfigEntry{}, nil + }), + }), + } + assertRequest1(ContextWithRequestInfo(context.Background(), RequestInfo{ + RequiredScope: ParseScope("repository:foo1:pull"), + }), t, ts, "/test/foo1", client) + require.Equal(t, 2, targetCount) + require.Equal(t, 1, authCount) + + assertRequest1(ContextWithRequestInfo(context.Background(), RequestInfo{ + RequiredScope: ParseScope("repository:foo2:pull"), + }), t, ts, "/test/foo2", client) + require.Equal(t, 3, targetCount) + require.Equal(t, 2, authCount) +} + func TestAuthServerRejectsRequestsWithTooMuchScope(t *testing.T) { - // This tests the scenario described in the comment in registry.acquireAccessToken. + // This verifies that caller-provided desired scope is not added to the + // registry-provided challenge scope. userHasScope := ParseScope("repository:foo:pull") authSrv := newAuthServer(t, func(req *http.Request) (any, *httpError) { diff --git a/ociauth/context.go b/ociauth/context.go index cd691d6..218aa5b 100644 --- a/ociauth/context.go +++ b/ociauth/context.go @@ -7,10 +7,8 @@ import ( type scopeKey struct{} // ContextWithScope returns ctx annotated with the given -// scope. When the ociauth transport receives a request with a scope in the context, -// it will treat it as "desired authorization scope"; new authorization tokens -// will be acquired with that scope as well as any scope required by -// the operation. +// scope. The ociauth transport does not add this scope to a registry +// challenge; challenges remain the source of truth for new token acquisition. func ContextWithScope(ctx context.Context, s Scope) context.Context { return context.WithValue(ctx, scopeKey{}, s) } @@ -29,18 +27,18 @@ type requestInfoKey struct{} // request context. The [ociclient] package will add this to all // requests that is makes. type RequestInfo struct { - // RequiredScope holds the authorization scope that's required - // by the request. The ociauth logic will reuse any available - // auth token that has this scope. When acquiring a new token, - // it will add any scope found in [ScopeFromContext] too. + // RequiredScope holds the authorization scope that can satisfy + // the request for cached-token reuse. When the transport already + // knows the registry's bearer token realm and has a refresh token, + // it may use this scope to acquire a token proactively. A + // Www-Authenticate challenge remains authoritative when present. RequiredScope Scope } // ContextWithRequestInfo returns ctx annotated with the given // request informaton. When ociclient receives a request with // this attached, it will respect info.RequiredScope to determine -// what auth tokens to reuse. When it acquires a new token, -// it will ask for the union of info.RequiredScope [ScopeFromContext]. +// what auth tokens to reuse or proactively acquire. func ContextWithRequestInfo(ctx context.Context, info RequestInfo) context.Context { return context.WithValue(ctx, requestInfoKey{}, info) } diff --git a/ociauth/scope.go b/ociauth/scope.go index 1dd65bf..f11cab2 100644 --- a/ociauth/scope.go +++ b/ociauth/scope.go @@ -13,6 +13,7 @@ type knownAction byte const ( unknownAction knownAction = iota // Note: ordered by lexical string representation. + deleteAction pullAction pushAction numActions @@ -24,6 +25,8 @@ const ( // TypeRegistry is the resource type for registry-wide operations. TypeRegistry = "registry" + // ActionDelete is the action for deleting content from a repository. + ActionDelete = "delete" // ActionPull is the action for pulling content from a repository. ActionPull = "pull" // ActionPush is the action for pushing content to a repository. @@ -32,6 +35,8 @@ const ( func (a knownAction) String() string { switch a { + case deleteAction: + return ActionDelete case pullAction: return ActionPull case pushAction: @@ -65,7 +70,7 @@ type ResourceScope struct { Resource string // Action names an action that can be performed on the resource. - // This is usually ActionPush or ActionPull. + // This is usually ActionPull, ActionPush or ActionDelete. Action string } @@ -487,6 +492,8 @@ func (s Scope) String() string { func parseKnownAction(s string) knownAction { switch s { + case ActionDelete: + return deleteAction case ActionPull: return pullAction case ActionPush: diff --git a/ociclient/auth_test.go b/ociclient/auth_test.go index 21e636d..a225684 100644 --- a/ociclient/auth_test.go +++ b/ociclient/auth_test.go @@ -54,14 +54,14 @@ func TestAuthScopes(t *testing.T) { assertScope("repository:foo/bar:pull", func(ctx context.Context, r oci.Interface) { r.ResolveTag(ctx, "foo/bar", "sometag") }) - assertScope("repository:foo/bar:push", func(ctx context.Context, r oci.Interface) { + assertScope("repository:foo/bar:pull,push", func(ctx context.Context, r oci.Interface) { r.PushBlob(ctx, "foo/bar", oci.Descriptor{ MediaType: "application/json", Digest: testDigest, Size: 3, }, strings.NewReader("foo")) }) - assertScope("repository:foo/bar:push", func(ctx context.Context, r oci.Interface) { + assertScope("repository:foo/bar:pull,push", func(ctx context.Context, r oci.Interface) { w, err := r.PushBlobChunked(ctx, "foo/bar", 0) require.NoError(t, err) w.Write([]byte("foo")) @@ -74,21 +74,21 @@ func TestAuthScopes(t *testing.T) { _, err = w.Commit(ocidigest.FromBytes([]byte("foobar"))) require.NoError(t, err) }) - assertScope("repository:x/y:pull repository:z/w:push", func(ctx context.Context, r oci.Interface) { + assertScope("repository:x/y:pull repository:z/w:pull,push", func(ctx context.Context, r oci.Interface) { r.MountBlob(ctx, "x/y", "z/w", testDigest) }) - assertScope("repository:foo/bar:push", func(ctx context.Context, r oci.Interface) { + assertScope("repository:foo/bar:pull,push", func(ctx context.Context, r oci.Interface) { r.PushManifest(ctx, "foo/bar", []byte("something"), "application/json", &oci.PushManifestParameters{ Tags: []string{"sometag"}, }) }) - assertScope("repository:foo/bar:push", func(ctx context.Context, r oci.Interface) { + assertScope("repository:foo/bar:delete", func(ctx context.Context, r oci.Interface) { r.DeleteBlob(ctx, "foo/bar", testDigest) }) - assertScope("repository:foo/bar:push", func(ctx context.Context, r oci.Interface) { + assertScope("repository:foo/bar:delete", func(ctx context.Context, r oci.Interface) { r.DeleteManifest(ctx, "foo/bar", testDigest) }) - assertScope("repository:foo/bar:push", func(ctx context.Context, r oci.Interface) { + assertScope("repository:foo/bar:delete", func(ctx context.Context, r oci.Interface) { r.DeleteTag(ctx, "foo/bar", "sometag") }) assertScope("registry:catalog:*", func(ctx context.Context, r oci.Interface) { diff --git a/ociclient/badname_test.go b/ociclient/badname_test.go index 77f228b..e2a31b1 100644 --- a/ociclient/badname_test.go +++ b/ociclient/badname_test.go @@ -19,9 +19,9 @@ func TestBadRepoName(t *testing.T) { }) require.NoError(t, err) _, err = r.GetBlob(ctx, "Invalid--Repo", ocidigest.FromBytes(nil)) - assert.Regexp(t, "invalid OCI request: name invalid: invalid repository name", err.Error()) + assert.Regexp(t, "no can do", err.Error()) _, err = r.ResolveTag(ctx, "okrepo", "bad-Tag!") - assert.Regexp(t, "invalid OCI request: 404 Not Found: page not found", err.Error()) + assert.Regexp(t, "no can do", err.Error()) } type noTransport struct{} diff --git a/ociclient/client.go b/ociclient/client.go index d0070a7..3bbeba3 100644 --- a/ociclient/client.go +++ b/ociclient/client.go @@ -31,7 +31,6 @@ import ( "github.com/docker/oci" - "github.com/docker/oci/internal/ocirequest" "github.com/docker/oci/ociauth" "github.com/docker/oci/ocidigest" "github.com/docker/oci/ociref" @@ -66,13 +65,12 @@ type Options struct { var debugID int32 -// New returns a registry implementation that uses the OCI -// HTTP API. A nil opts parameter is equivalent to a pointer -// to zero Options. +// New returns a client implementation that uses the OCI HTTP API. A nil opts +// parameter is equivalent to a pointer to zero Options. // // The host specifies the host name to talk to; it may // optionally be a host:port pair. -func New(host string, opts0 *Options) (oci.Interface, error) { +func New(host string, opts0 *Options) (*Client, error) { var opts Options if opts0 != nil { opts = *opts0 @@ -100,7 +98,7 @@ func New(host string, opts0 *Options) (oci.Interface, error) { if opts.Insecure { u.Scheme = "http" } - return &client{ + return &Client{ httpHost: host, httpScheme: u.Scheme, httpClient: &http.Client{ @@ -111,7 +109,9 @@ func New(host string, opts0 *Options) (oci.Interface, error) { }, nil } -type client struct { +// Client is an OCI registry client that uses HTTP to talk to the remote +// registry. +type Client struct { *oci.Funcs httpScheme string httpHost string @@ -121,6 +121,16 @@ type client struct { listPageSize int } +var _ oci.Interface = (*Client)(nil) + +// RequestOptions holds options for [Client.Do]. +type RequestOptions struct { + // Scope is the authorization scope required by the request. It is attached + // to the request context for use by transports created by + // [ociauth.NewStdTransport]. + Scope ociauth.Scope +} + type descriptorRequired byte const ( @@ -262,30 +272,16 @@ var knownManifestMediaTypes = []string{ "*/*", } -// doRequest performs the given OCI request, sending it with the given body (which may be nil). -func (c *client) doRequest(ctx context.Context, rreq *ocirequest.Request, okStatuses ...int) (*http.Response, error) { - req, err := newRequest(ctx, rreq, nil) - if err != nil { - return nil, err - } - if rreq.Kind == ocirequest.ReqManifestGet || rreq.Kind == ocirequest.ReqManifestHead { - // When getting manifests, some servers won't return - // the content unless there's an Accept header, so - // add all the manifest kinds that we know about. - req.Header["Accept"] = knownManifestMediaTypes - } - resp, err := c.do(req, okStatuses...) - if err != nil { - return nil, err - } - if resp.StatusCode/100 == 2 { - return resp, nil - } - defer resp.Body.Close() - return nil, makeError(resp) -} - -func (c *client) do(req *http.Request, okStatuses ...int) (*http.Response, error) { +// Do performs an HTTP request against the registry. +// +// If req.URL has no scheme or host, Do fills them in from the client's +// configured registry. It also sets the User-Agent header and, when the request +// has a body, sets Expect: 100-continue. +// +// Do does not interpret HTTP response statuses. Callers that want OCI error +// handling can use [CheckResponse] or [ErrorFromResponse]. On a nil error, the +// caller is responsible for closing resp.Body. +func (c *Client) Do(req *http.Request, opts *RequestOptions) (*http.Response, error) { if req.URL.Scheme == "" { req.URL.Scheme = c.httpScheme } @@ -293,6 +289,11 @@ func (c *client) do(req *http.Request, okStatuses ...int) (*http.Response, error req.URL.Host = c.httpHost } req.Header.Set("User-Agent", c.userAgent) + if opts != nil { + req = req.WithContext(ociauth.ContextWithRequestInfo(req.Context(), ociauth.RequestInfo{ + RequiredScope: opts.Scope, + })) + } if req.Body != nil { // Ensure that the body isn't consumed until the @@ -304,7 +305,7 @@ func (c *client) do(req *http.Request, okStatuses ...int) (*http.Response, error } var buf bytes.Buffer if debug { - fmt.Fprintf(&buf, "client.Do: %s %s {{\n", req.Method, req.URL) + fmt.Fprintf(&buf, "Client.Do: %s %s {{\n", req.Method, req.URL) fmt.Fprintf(&buf, "\tBODY: %#v\n", req.Body) for k, v := range req.Header { fmt.Fprintf(&buf, "\t%s: %q\n", k, v) @@ -330,20 +331,43 @@ func (c *client) do(req *http.Request, okStatuses ...int) (*http.Response, error resp.Body = io.NopCloser(bytes.NewReader(data)) c.logf("%s", buf.Bytes()) } - if len(okStatuses) == 0 && resp.StatusCode == http.StatusOK { - return resp, nil + return resp, nil +} + +// CheckResponse checks that resp has an acceptable status. +// +// If okStatuses is empty, any 2xx response is accepted. Otherwise, only the +// supplied statuses are accepted. Non-2xx responses are returned as OCI errors. +// CheckResponse reads but does not close resp.Body when constructing an error. +func CheckResponse(resp *http.Response, okStatuses ...int) error { + if len(okStatuses) == 0 { + if isOKStatus(resp.StatusCode) { + return nil + } + return ErrorFromResponse(resp) } if slices.Contains(okStatuses, resp.StatusCode) { - return resp, nil + return nil } - defer resp.Body.Close() if !isOKStatus(resp.StatusCode) { - return nil, makeError(resp) + return ErrorFromResponse(resp) + } + return unexpectedStatusError(resp.StatusCode) +} + +func (c *Client) do(req *http.Request, okStatuses ...int) (*http.Response, error) { + resp, err := c.Do(req, nil) + if err != nil { + return nil, err + } + if err := CheckResponse(resp, okStatuses...); err != nil { + resp.Body.Close() + return nil, err } - return nil, unexpectedStatusError(resp.StatusCode) + return resp, nil } -func (c *client) logf(f string, a ...any) { +func (c *Client) logf(f string, a ...any) { log.Printf("ociclient %s: %s", c.debugID, fmt.Sprintf(f, a...)) } @@ -373,58 +397,70 @@ func unexpectedStatusError(code int) error { return fmt.Errorf("unexpected HTTP response code %d", code) } -func scopeForRequest(r *ocirequest.Request) ociauth.Scope { - switch r.Kind { - case ocirequest.ReqPing: - return ociauth.Scope{} - case ocirequest.ReqBlobGet, - ocirequest.ReqBlobHead, - ocirequest.ReqManifestGet, - ocirequest.ReqManifestHead, - ocirequest.ReqTagsList, - ocirequest.ReqReferrersList: - return ociauth.NewScope(ociauth.ResourceScope{ - ResourceType: ociauth.TypeRepository, - Resource: r.Repo, - Action: ociauth.ActionPull, - }) - case ocirequest.ReqBlobDelete, - ocirequest.ReqBlobStartUpload, - ocirequest.ReqBlobUploadBlob, - ocirequest.ReqBlobUploadInfo, - ocirequest.ReqBlobUploadChunk, - ocirequest.ReqBlobCompleteUpload, - ocirequest.ReqManifestPut, - ocirequest.ReqManifestDelete: - return ociauth.NewScope(ociauth.ResourceScope{ - ResourceType: ociauth.TypeRepository, - Resource: r.Repo, - Action: ociauth.ActionPush, - }) - case ocirequest.ReqBlobMount: - return ociauth.NewScope(ociauth.ResourceScope{ - ResourceType: ociauth.TypeRepository, - Resource: r.Repo, - Action: ociauth.ActionPush, - }, ociauth.ResourceScope{ - ResourceType: ociauth.TypeRepository, - Resource: r.FromRepo, - Action: ociauth.ActionPull, - }) - case ocirequest.ReqCatalogList: - return ociauth.NewScope(ociauth.CatalogScope) - default: - panic(fmt.Errorf("unexpected request kind %v", r.Kind)) - } +func newRequest(ctx context.Context, method string, u string, body io.Reader, scope ociauth.Scope) (*http.Request, error) { + ctx = ociauth.ContextWithRequestInfo(ctx, ociauth.RequestInfo{ + RequiredScope: scope, + }) + return http.NewRequestWithContext(ctx, method, u, body) } -func newRequest(ctx context.Context, rreq *ocirequest.Request, body io.Reader) (*http.Request, error) { - method, u, err := rreq.Construct() +func newManifestRequest(ctx context.Context, method string, repo string, tagOrDigest string, scope ociauth.Scope) (*http.Request, error) { + req, err := newRequest(ctx, method, manifestURL(repo, tagOrDigest), nil, scope) if err != nil { return nil, err } - ctx = ociauth.ContextWithRequestInfo(ctx, ociauth.RequestInfo{ - RequiredScope: scopeForRequest(rreq), + // When getting manifests, some servers won't return the content unless + // there's an Accept header, so add all manifest kinds that we know about. + if method == http.MethodGet || method == http.MethodHead { + req.Header["Accept"] = knownManifestMediaTypes + } + return req, nil +} + +func pullScope(repo string) ociauth.Scope { + return ociauth.NewScope(ociauth.ResourceScope{ + ResourceType: ociauth.TypeRepository, + Resource: repo, + Action: ociauth.ActionPull, }) - return http.NewRequestWithContext(ctx, method, u, body) +} + +func pushScope(repo string) ociauth.Scope { + return ociauth.NewScope(ociauth.ResourceScope{ + ResourceType: ociauth.TypeRepository, + Resource: repo, + Action: ociauth.ActionPull, + }, ociauth.ResourceScope{ + ResourceType: ociauth.TypeRepository, + Resource: repo, + Action: ociauth.ActionPush, + }) +} + +func deleteScope(repo string) ociauth.Scope { + return ociauth.NewScope(ociauth.ResourceScope{ + ResourceType: ociauth.TypeRepository, + Resource: repo, + Action: ociauth.ActionDelete, + }) +} + +func mountScope(fromRepo, toRepo string) ociauth.Scope { + return ociauth.NewScope(ociauth.ResourceScope{ + ResourceType: ociauth.TypeRepository, + Resource: toRepo, + Action: ociauth.ActionPull, + }, ociauth.ResourceScope{ + ResourceType: ociauth.TypeRepository, + Resource: toRepo, + Action: ociauth.ActionPush, + }, ociauth.ResourceScope{ + ResourceType: ociauth.TypeRepository, + Resource: fromRepo, + Action: ociauth.ActionPull, + }) +} + +func catalogScope() ociauth.Scope { + return ociauth.NewScope(ociauth.CatalogScope) } diff --git a/ociclient/client_test.go b/ociclient/client_test.go new file mode 100644 index 0000000..8f9defe --- /dev/null +++ b/ociclient/client_test.go @@ -0,0 +1,54 @@ +package ociclient + +import ( + "context" + "io" + "net/http" + "strings" + "testing" + + "github.com/docker/oci" + "github.com/docker/oci/ociauth" + "github.com/stretchr/testify/require" +) + +func TestClientDo(t *testing.T) { + wantScope := ociauth.NewScope(ociauth.ResourceScope{ + ResourceType: ociauth.TypeRepository, + Resource: "foo/bar", + Action: ociauth.ActionPull, + }) + + c, err := New("registry.example", &Options{ + Transport: transportFunc(func(req *http.Request) (*http.Response, error) { + require.Equal(t, "https", req.URL.Scheme) + require.Equal(t, "registry.example", req.URL.Host) + require.Equal(t, "/v2/foo/bar/custom", req.URL.Path) + require.Equal(t, "docker/oci", req.Header.Get("User-Agent")) + require.True(t, wantScope.Equal(ociauth.RequestInfoFromContext(req.Context()).RequiredScope)) + return &http.Response{ + StatusCode: http.StatusTeapot, + Status: "418 I'm a teapot", + Header: http.Header{"Content-Type": {"application/json"}}, + Body: io.NopCloser(strings.NewReader(`{ + "errors": [{"code": "DENIED", "message": "custom denied"}] + }`)), + Request: req, + }, nil + }), + }) + require.NoError(t, err) + + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, "/v2/foo/bar/custom", nil) + require.NoError(t, err) + + resp, err := c.Do(req, &RequestOptions{Scope: wantScope}) + require.NoError(t, err) + require.Equal(t, http.StatusTeapot, resp.StatusCode) + + err = CheckResponse(resp) + require.Error(t, err) + var httpErr oci.HTTPError + require.ErrorAs(t, err, &httpErr) + require.Equal(t, http.StatusTeapot, httpErr.StatusCode()) +} diff --git a/ociclient/deleter.go b/ociclient/deleter.go index 2dc7a37..719e882 100644 --- a/ociclient/deleter.go +++ b/ociclient/deleter.go @@ -19,35 +19,37 @@ import ( "net/http" "github.com/docker/oci" - "github.com/docker/oci/internal/ocirequest" ) -func (c *client) DeleteBlob(ctx context.Context, repoName string, digest oci.Digest) error { - return c.delete(ctx, &ocirequest.Request{ - Kind: ocirequest.ReqBlobDelete, - Repo: repoName, - Digest: digest.String(), - }) +// DeleteBlob deletes the blob with the given digest from the repository. +func (c *Client) DeleteBlob(ctx context.Context, repoName string, digest oci.Digest) error { + req, err := newRequest(ctx, http.MethodDelete, blobURL(repoName, digest), nil, deleteScope(repoName)) + if err != nil { + return err + } + return c.delete(req) } -func (c *client) DeleteManifest(ctx context.Context, repoName string, digest oci.Digest) error { - return c.delete(ctx, &ocirequest.Request{ - Kind: ocirequest.ReqManifestDelete, - Repo: repoName, - Digest: digest.String(), - }) +// DeleteManifest deletes the manifest with the given digest from the repository. +func (c *Client) DeleteManifest(ctx context.Context, repoName string, digest oci.Digest) error { + req, err := newRequest(ctx, http.MethodDelete, manifestURL(repoName, digest.String()), nil, deleteScope(repoName)) + if err != nil { + return err + } + return c.delete(req) } -func (c *client) DeleteTag(ctx context.Context, repoName string, tagName string) error { - return c.delete(ctx, &ocirequest.Request{ - Kind: ocirequest.ReqManifestDelete, - Repo: repoName, - Tag: tagName, - }) +// DeleteTag deletes the manifest with the given tag from the repository. +func (c *Client) DeleteTag(ctx context.Context, repoName string, tagName string) error { + req, err := newRequest(ctx, http.MethodDelete, manifestURL(repoName, tagName), nil, deleteScope(repoName)) + if err != nil { + return err + } + return c.delete(req) } -func (c *client) delete(ctx context.Context, rreq *ocirequest.Request) error { - resp, err := c.doRequest(ctx, rreq, http.StatusAccepted) +func (c *Client) delete(req *http.Request) error { + resp, err := c.do(req, http.StatusAccepted) if err != nil { return err } diff --git a/ociclient/error.go b/ociclient/error.go index 73365a3..543dad1 100644 --- a/ociclient/error.go +++ b/ociclient/error.go @@ -30,6 +30,13 @@ import ( // bytes. Hence, 8 KiB should be sufficient. const errorBodySizeLimit = 8 * 1024 +// ErrorFromResponse forms an OCI error from an HTTP response. +// +// It reads but does not close resp.Body. +func ErrorFromResponse(resp *http.Response) error { + return makeError(resp) +} + // makeError forms an error from a non-OK response. // // It reads but does not close resp.Body. diff --git a/ociclient/extension.go b/ociclient/extension.go new file mode 100644 index 0000000..0b14f6e --- /dev/null +++ b/ociclient/extension.go @@ -0,0 +1,52 @@ +// Copyright 2023 CUE Labs AG +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ociclient + +import ( + "context" + "encoding/json" + "fmt" + "io" + "iter" + "net/http" +) + +// Repositories returns an iterator over repositories in the registry. +func (c *Client) Repositories(ctx context.Context, startAfter string) iter.Seq2[string, error] { + return pager(ctx, c, pageRequest{ + URL: catalogURL(startAfter), + Scope: catalogScope(), + Limit: -1, + Next: func(last any) string { + return catalogURL(fmt.Sprint(last)) + }, + }, true, func(resp *http.Response) ([]string, error) { + data, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + var catalog struct { + Repos []string `json:"repositories"` + } + if err := json.Unmarshal(data, &catalog); err != nil { + return nil, fmt.Errorf("cannot unmarshal catalog response: %v", err) + } + return catalog.Repos, nil + }) +} + +func catalogURL(last string) string { + return "/v2/_catalog" + listQuery(-1, last) +} diff --git a/ociclient/lister.go b/ociclient/lister.go index b18b6d0..a95b8a4 100644 --- a/ociclient/lister.go +++ b/ociclient/lister.go @@ -22,45 +22,29 @@ import ( "io" "iter" "net/http" + "net/url" "slices" "strings" "github.com/docker/oci" - - "github.com/docker/oci/internal/ocirequest" + "github.com/docker/oci/ociauth" ) -func (c *client) Repositories(ctx context.Context, startAfter string) iter.Seq2[string, error] { - return pager(ctx, c, &ocirequest.Request{ - Kind: ocirequest.ReqCatalogList, - ListLast: startAfter, - }, true, func(resp *http.Response) ([]string, error) { - data, err := io.ReadAll(resp.Body) - if err != nil { - return nil, err - } - var catalog struct { - Repos []string `json:"repositories"` - } - if err := json.Unmarshal(data, &catalog); err != nil { - return nil, fmt.Errorf("cannot unmarshal catalog response: %v", err) - } - return catalog.Repos, nil - }) -} - -func (c *client) Tags(ctx context.Context, repoName string, params *oci.TagsParameters) iter.Seq2[string, error] { +// Tags returns an iterator over tags in the given repository. +func (c *Client) Tags(ctx context.Context, repoName string, params *oci.TagsParameters) iter.Seq2[string, error] { var startAfter string limit := -1 if params != nil { startAfter = params.StartAfter limit = params.Limit } - return pager(ctx, c, &ocirequest.Request{ - Kind: ocirequest.ReqTagsList, - Repo: repoName, - ListN: limit, - ListLast: startAfter, + return pager(ctx, c, pageRequest{ + URL: tagsURL(repoName, limit, startAfter), + Scope: pullScope(repoName), + Limit: limit, + Next: func(last any) string { + return tagsURL(repoName, limit, fmt.Sprint(last)) + }, }, true, func(resp *http.Response) ([]string, error) { data, err := io.ReadAll(resp.Body) if err != nil { @@ -77,17 +61,16 @@ func (c *client) Tags(ctx context.Context, repoName string, params *oci.TagsPara }) } -func (c *client) Referrers(ctx context.Context, repoName string, digest oci.Digest, params *oci.ReferrersParameters) iter.Seq2[oci.Descriptor, error] { +// Referrers returns an iterator over manifests that refer to the given digest. +func (c *Client) Referrers(ctx context.Context, repoName string, digest oci.Digest, params *oci.ReferrersParameters) iter.Seq2[oci.Descriptor, error] { var artifactType string if params != nil { artifactType = params.ArtifactType } - return pager(ctx, c, &ocirequest.Request{ - Kind: ocirequest.ReqReferrersList, - Repo: repoName, - Digest: digest.String(), - ListN: -1, - ArtifactType: artifactType, + return pager(ctx, c, pageRequest{ + URL: referrersURL(repoName, digest, artifactType), + Scope: pullScope(repoName), + Limit: -1, }, false, func(resp *http.Response) ([]oci.Descriptor, error) { body := resp.Body if resp.StatusCode == http.StatusNotFound { @@ -137,15 +120,25 @@ func (c *client) Referrers(ctx context.Context, repoName string, digest oci.Dige }, http.StatusOK, http.StatusNotFound) } +type pageRequest struct { + URL string + Scope ociauth.Scope + Limit int + Next func(last any) string +} + // pager returns an iterator for a list entry point. It starts by sending the given // initial request and parses each response into its component items using // parseResponse. It tries to use the Link header in each response to continue // the iteration, falling back to using the "last" query parameter if // canUseLast is true. -func pager[T any](ctx context.Context, c *client, initialReq *ocirequest.Request, canUseLast bool, parseResponse func(*http.Response) ([]T, error), okStatuses ...int) iter.Seq2[T, error] { +func pager[T any](ctx context.Context, c *Client, initialReq pageRequest, canUseLast bool, parseResponse func(*http.Response) ([]T, error), okStatuses ...int) iter.Seq2[T, error] { + if len(okStatuses) == 0 { + okStatuses = []int{http.StatusOK} + } return func(yield func(T, error) bool) { // We assume that the same auth scope is applicable to all page requests. - req, err := newRequest(ctx, initialReq, nil) + req, err := newRequest(ctx, http.MethodGet, initialReq.URL, nil, initialReq.Scope) if err != nil { yield(*new(T), err) return @@ -167,7 +160,7 @@ func pager[T any](ctx context.Context, c *client, initialReq *ocirequest.Request return } } - if len(items) == 0 || (initialReq.ListN > 0 && len(items) < initialReq.ListN) { + if len(items) == 0 || (initialReq.Limit > 0 && len(items) < initialReq.Limit) { // From the distribution spec: // The response to such a request MAY return fewer than results, // but only when the total number of tags attached to the repository @@ -192,18 +185,16 @@ func pager[T any](ctx context.Context, c *client, initialReq *ocirequest.Request // The given response holds the response received from the previous // list request; initialReq holds the request that initiated the listing, // and last holds the final item returned in the previous response. -func nextLink[T any](ctx context.Context, resp *http.Response, initialReq *ocirequest.Request, canUseLast bool, last T) (*http.Request, error) { +func nextLink[T any](ctx context.Context, resp *http.Response, initialReq pageRequest, canUseLast bool, last T) (*http.Request, error) { link0 := resp.Header.Get("Link") if link0 == "" { - if !canUseLast { + if !canUseLast || initialReq.Next == nil { return nil, nil } // This is beyond the first page and there was no Link // in the previous response (the standard doesn't mandate // one), so add a "last" parameter to the initial request. - rreq := *initialReq - rreq.ListLast = fmt.Sprint(last) - req, err := newRequest(ctx, &rreq, nil) + req, err := newRequest(ctx, http.MethodGet, initialReq.Next(last), nil, initialReq.Scope) if err != nil { // Given that we could form the initial request, this should // never happen. @@ -226,7 +217,7 @@ func nextLink[T any](ctx context.Context, resp *http.Response, initialReq *ocire if err != nil { return nil, fmt.Errorf("invalid URL in Link=%q", link0) } - return http.NewRequestWithContext(ctx, "GET", linkURL.String(), nil) + return newRequest(ctx, http.MethodGet, linkURL.String(), nil, initialReq.Scope) } // referrersTag returns the referrers tag for the given digest, as described @@ -263,3 +254,29 @@ func truncateAndMap(s string, n int) string { } return s[:n] } + +func tagsURL(repo string, limit int, last string) string { + return "/v2/" + repo + "/tags/list" + listQuery(limit, last) +} + +func referrersURL(repo string, digest oci.Digest, artifactType string) string { + u := "/v2/" + repo + "/referrers/" + digest.String() + if artifactType != "" { + u += "?" + url.Values{"artifactType": {artifactType}}.Encode() + } + return u +} + +func listQuery(limit int, last string) string { + q := make(url.Values) + if limit >= 0 { + q.Set("n", fmt.Sprint(limit)) + } + if last != "" { + q.Set("last", last) + } + if len(q) == 0 { + return "" + } + return "?" + q.Encode() +} diff --git a/ociclient/reader.go b/ociclient/reader.go index d530fd3..5ce9fa3 100644 --- a/ociclient/reader.go +++ b/ociclient/reader.go @@ -22,28 +22,24 @@ import ( "net/http" "github.com/docker/oci" - "github.com/docker/oci/internal/ocirequest" "github.com/docker/oci/ocidigest" ) -func (c *client) GetBlob(ctx context.Context, repo string, digest oci.Digest) (oci.BlobReader, error) { - return c.read(ctx, &ocirequest.Request{ - Kind: ocirequest.ReqBlobGet, - Repo: repo, - Digest: digest.String(), - }) +// GetBlob returns the content of the blob with the given digest. +func (c *Client) GetBlob(ctx context.Context, repo string, digest oci.Digest) (oci.BlobReader, error) { + req, err := newRequest(ctx, http.MethodGet, blobURL(repo, digest), nil, pullScope(repo)) + if err != nil { + return nil, err + } + return c.read(req, digest, false) } -func (c *client) GetBlobRange(ctx context.Context, repo string, digest oci.Digest, o0, o1 int64) (_ oci.BlobReader, _err error) { +// GetBlobRange is like GetBlob but asks for only the given byte range. +func (c *Client) GetBlobRange(ctx context.Context, repo string, digest oci.Digest, o0, o1 int64) (_ oci.BlobReader, _err error) { if o0 == 0 && o1 < 0 { return c.GetBlob(ctx, repo, digest) } - rreq := &ocirequest.Request{ - Kind: ocirequest.ReqBlobGet, - Repo: repo, - Digest: digest.String(), - } - req, err := newRequest(ctx, rreq, nil) + req, err := newRequest(ctx, http.MethodGet, blobURL(repo, digest), nil, pullScope(repo)) if err != nil { return nil, err } @@ -60,45 +56,46 @@ func (c *client) GetBlobRange(ctx context.Context, repo string, digest oci.Diges // Fix that either by returning ErrUnsupported or by reading the whole // blob and returning only the required portion. defer closeOnError(&_err, resp.Body) - knownDigest, _ := ocidigest.Parse(rreq.Digest) - desc, err := descriptorFromResponse(resp, knownDigest, requireSize) + desc, err := descriptorFromResponse(resp, digest, requireSize) if err != nil { return nil, fmt.Errorf("invalid descriptor in response: %v", err) } return newBlobReaderUnverified(resp.Body, desc), nil } -func (c *client) ResolveBlob(ctx context.Context, repo string, digest oci.Digest) (oci.Descriptor, error) { - return c.resolve(ctx, &ocirequest.Request{ - Kind: ocirequest.ReqBlobHead, - Repo: repo, - Digest: digest.String(), - }) +// ResolveBlob returns the descriptor for the blob with the given digest. +func (c *Client) ResolveBlob(ctx context.Context, repo string, digest oci.Digest) (oci.Descriptor, error) { + req, err := newRequest(ctx, http.MethodHead, blobURL(repo, digest), nil, pullScope(repo)) + if err != nil { + return oci.Descriptor{}, err + } + return c.resolve(req, digest) } -func (c *client) ResolveManifest(ctx context.Context, repo string, digest oci.Digest) (oci.Descriptor, error) { - return c.resolve(ctx, &ocirequest.Request{ - Kind: ocirequest.ReqManifestHead, - Repo: repo, - Digest: digest.String(), - }) +// ResolveManifest returns the descriptor for the manifest with the given digest. +func (c *Client) ResolveManifest(ctx context.Context, repo string, digest oci.Digest) (oci.Descriptor, error) { + req, err := newManifestRequest(ctx, http.MethodHead, repo, digest.String(), pullScope(repo)) + if err != nil { + return oci.Descriptor{}, err + } + return c.resolve(req, digest) } -func (c *client) ResolveTag(ctx context.Context, repo string, tag string) (oci.Descriptor, error) { - return c.resolve(ctx, &ocirequest.Request{ - Kind: ocirequest.ReqManifestHead, - Repo: repo, - Tag: tag, - }) +// ResolveTag returns the descriptor for the manifest with the given tag. +func (c *Client) ResolveTag(ctx context.Context, repo string, tag string) (oci.Descriptor, error) { + req, err := newManifestRequest(ctx, http.MethodHead, repo, tag, pullScope(repo)) + if err != nil { + return oci.Descriptor{}, err + } + return c.resolve(req, "") } -func (c *client) resolve(ctx context.Context, rreq *ocirequest.Request) (oci.Descriptor, error) { - resp, err := c.doRequest(ctx, rreq) +func (c *Client) resolve(req *http.Request, knownDigest oci.Digest) (oci.Descriptor, error) { + resp, err := c.do(req) if err != nil { return oci.Descriptor{}, err } resp.Body.Close() - knownDigest, _ := ocidigest.Parse(rreq.Digest) desc, err := descriptorFromResponse(resp, knownDigest, requireSize|requireDigest) if err != nil { return oci.Descriptor{}, fmt.Errorf("invalid descriptor in response: %v", err) @@ -106,41 +103,32 @@ func (c *client) resolve(ctx context.Context, rreq *ocirequest.Request) (oci.Des return desc, nil } -func (c *client) GetManifest(ctx context.Context, repo string, digest oci.Digest) (oci.BlobReader, error) { - return c.read(ctx, &ocirequest.Request{ - Kind: ocirequest.ReqManifestGet, - Repo: repo, - Digest: digest.String(), - }) +// GetManifest returns the content of the manifest with the given digest. +func (c *Client) GetManifest(ctx context.Context, repo string, digest oci.Digest) (oci.BlobReader, error) { + req, err := newManifestRequest(ctx, http.MethodGet, repo, digest.String(), pullScope(repo)) + if err != nil { + return nil, err + } + return c.read(req, digest, true) } -func (c *client) GetTag(ctx context.Context, repo string, tagName string) (oci.BlobReader, error) { - return c.read(ctx, &ocirequest.Request{ - Kind: ocirequest.ReqManifestGet, - Repo: repo, - Tag: tagName, - }) +// GetTag returns the content of the manifest with the given tag. +func (c *Client) GetTag(ctx context.Context, repo string, tagName string) (oci.BlobReader, error) { + req, err := newManifestRequest(ctx, http.MethodGet, repo, tagName, pullScope(repo)) + if err != nil { + return nil, err + } + return c.read(req, "", true) } -// inMemThreshold holds the maximum number of bytes of manifest content -// that we'll hold in memory to obtain a digest before falling back do -// doing a HEAD request. -// -// This is hopefully large enough to be considerably larger than most -// manifests but small enough to fit comfortably into RAM on most -// platforms. -// -// Note: this is only used when talking to registries that fail to return -// a digest when doing a GET on a tag. -const inMemThreshold = 128 * 1024 +const maxManifestSize = 4 * 1024 * 1024 -func (c *client) read(ctx context.Context, rreq *ocirequest.Request) (_ oci.BlobReader, _err error) { - resp, err := c.doRequest(ctx, rreq) +func (c *Client) read(req *http.Request, knownDigest oci.Digest, isManifest bool) (_ oci.BlobReader, _err error) { + resp, err := c.do(req) if err != nil { return nil, err } defer closeOnError(&_err, resp.Body) - knownDigest, _ := ocidigest.Parse(rreq.Digest) desc, err := descriptorFromResponse(resp, knownDigest, requireSize) if err != nil { return nil, fmt.Errorf("invalid descriptor in response: %v", err) @@ -152,39 +140,32 @@ func (c *client) read(ctx context.Context, rreq *ocirequest.Request) (_ oci.Blob // We know the request must be a tag-getting // request because all other requests take a digest not a tag // but sanity check anyway. - if rreq.Kind != ocirequest.ReqManifestGet { + if !isManifest { return nil, fmt.Errorf("internal error: no digest available for non-tag request") } - // If the manifest is of a reasonable size, just read it into memory - // and calculate the digest that way, otherwise issue a HEAD - // request which should hopefully (and does in the ECR case) - // give us the digest we need. - if desc.Size <= inMemThreshold { - data, err := io.ReadAll(io.LimitReader(resp.Body, desc.Size+1)) - if err != nil { - return nil, fmt.Errorf("failed to read body to determine digest: %v", err) - } - if int64(len(data)) != desc.Size { - return nil, fmt.Errorf("body size mismatch") - } - desc.Digest = ocidigest.FromBytes(data) - resp.Body.Close() - resp.Body = io.NopCloser(bytes.NewReader(data)) - } else { - rreq1 := rreq - rreq1.Kind = ocirequest.ReqManifestHead - resp1, err := c.doRequest(ctx, rreq1) - if err != nil { - return nil, err - } - resp1.Body.Close() - knownDigest, _ := ocidigest.Parse(rreq1.Digest) - desc, err = descriptorFromResponse(resp1, knownDigest, requireSize|requireDigest) - if err != nil { - return nil, err - } + if desc.Size > maxManifestSize { + return nil, fmt.Errorf("manifest size %d exceeds maximum size %d", desc.Size, maxManifestSize) } + + data, err := io.ReadAll(io.LimitReader(resp.Body, maxManifestSize+1)) + if err != nil { + return nil, fmt.Errorf("failed to read body to determine digest: %v", err) + } + if int64(len(data)) != desc.Size { + return nil, fmt.Errorf("body size mismatch") + } + desc.Digest = ocidigest.FromBytes(data) + resp.Body.Close() + resp.Body = io.NopCloser(bytes.NewReader(data)) } return newBlobReader(resp.Body, desc), nil } + +func blobURL(repo string, digest oci.Digest) string { + return "/v2/" + repo + "/blobs/" + digest.String() +} + +func manifestURL(repo string, tagOrDigest string) string { + return "/v2/" + repo + "/manifests/" + tagOrDigest +} diff --git a/ociclient/writer.go b/ociclient/writer.go index 4693ec8..7214c4f 100644 --- a/ociclient/writer.go +++ b/ociclient/writer.go @@ -27,14 +27,12 @@ import ( "github.com/docker/oci" - "github.com/docker/oci/internal/ocirequest" "github.com/docker/oci/ociauth" "github.com/docker/oci/ocidigest" ) -// This file implements the oci.Writer methods. - -func (c *client) PushManifest(ctx context.Context, repo string, contents []byte, mediaType string, params *oci.PushManifestParameters) (oci.Descriptor, error) { +// PushManifest pushes a manifest with the given media type and contents. +func (c *Client) PushManifest(ctx context.Context, repo string, contents []byte, mediaType string, params *oci.PushManifestParameters) (oci.Descriptor, error) { if mediaType == "" { return oci.Descriptor{}, fmt.Errorf("PushManifest called with empty mediaType") } @@ -59,31 +57,14 @@ func (c *client) PushManifest(ctx context.Context, repo string, contents []byte, // If there are no tags, push once by digest. // If there are tags, push once per tag (all referencing the same contents). if len(tags) == 0 { - rreq := &ocirequest.Request{ - Kind: ocirequest.ReqManifestPut, - Repo: repo, - Digest: desc.Digest.String(), - } - _, err := c.putManifest(ctx, rreq, desc) + _, err := c.putManifest(ctx, repo, desc.Digest.String(), nil, desc) return desc, err } else { - rreq := &ocirequest.Request{ - Kind: ocirequest.ReqManifestPut, - Repo: repo, - Tags: tags, - Digest: desc.Digest.String(), - } - createdTags, err := c.putManifest(ctx, rreq, desc) + createdTags, err := c.putManifest(ctx, repo, desc.Digest.String(), tags, desc) if err != nil || len(createdTags) != len(tags) { // bulk send failed, fallback to sending one at a time for _, tag := range tags { - rreq := &ocirequest.Request{ - Kind: ocirequest.ReqManifestPut, - Repo: repo, - Tag: tag, - Digest: desc.Digest.String(), - } - _, err = c.putManifest(ctx, rreq, desc) + _, err = c.putManifest(ctx, repo, tag, nil, desc) if err != nil { return oci.Descriptor{}, fmt.Errorf("creating tag %s failed: %w", tag, err) } @@ -93,8 +74,16 @@ func (c *client) PushManifest(ctx context.Context, repo string, contents []byte, return desc, nil } -func (c *client) putManifest(ctx context.Context, rreq *ocirequest.Request, desc oci.Descriptor) ([]string, error) { - req, err := newRequest(ctx, rreq, bytes.NewReader(desc.Data)) +func (c *Client) putManifest(ctx context.Context, repo string, tagOrDigest string, tags []string, desc oci.Descriptor) ([]string, error) { + u := manifestURL(repo, tagOrDigest) + if len(tags) > 0 { + q := make(url.Values) + for _, tag := range tags { + q.Add("tag", tag) + } + u += "?" + q.Encode() + } + req, err := newRequest(ctx, http.MethodPut, u, bytes.NewReader(desc.Data), pushScope(repo)) if err != nil { return nil, err } @@ -106,23 +95,25 @@ func (c *client) putManifest(ctx context.Context, rreq *ocirequest.Request, desc } resp.Body.Close() - var tags []string + var createdTags []string if vs := resp.Header.Values("OCI-Tag"); len(vs) > 0 { for _, v := range vs { - tags = append(tags, strings.Split(v, ",")...) + createdTags = append(createdTags, strings.Split(v, ",")...) } } - return tags, nil + return createdTags, nil } -func (c *client) MountBlob(ctx context.Context, fromRepo, toRepo string, dig oci.Digest) (oci.Descriptor, error) { - rreq := &ocirequest.Request{ - Kind: ocirequest.ReqBlobMount, - Repo: toRepo, - FromRepo: fromRepo, - Digest: dig.String(), +// MountBlob makes a blob from fromRepo available in toRepo without uploading it again. +func (c *Client) MountBlob(ctx context.Context, fromRepo, toRepo string, dig oci.Digest) (oci.Descriptor, error) { + q := url.Values{} + q.Set("mount", dig.String()) + q.Set("from", fromRepo) + req, err := newRequest(ctx, http.MethodPost, blobUploadURL(toRepo, q), nil, mountScope(fromRepo, toRepo)) + if err != nil { + return oci.Descriptor{}, err } - resp, err := c.doRequest(ctx, rreq, http.StatusCreated, http.StatusAccepted) + resp, err := c.do(req, http.StatusCreated, http.StatusAccepted) if err != nil { return oci.Descriptor{}, err } @@ -137,16 +128,14 @@ func (c *client) MountBlob(ctx context.Context, fromRepo, toRepo string, dig oci return descriptorFromResponse(resp, dig, requireDigest) } -func (c *client) PushBlob(ctx context.Context, repo string, desc oci.Descriptor, r io.Reader) (_ oci.Descriptor, _err error) { - // TODO use the single-post blob-upload method (ReqBlobUploadBlob) +// PushBlob pushes a blob described by desc to the given repository. +func (c *Client) PushBlob(ctx context.Context, repo string, desc oci.Descriptor, r io.Reader) (_ oci.Descriptor, _err error) { + // TODO use the single-post blob-upload method. // See: // https://github.com/distribution/distribution/issues/4065 // https://github.com/golang/go/issues/63152 - rreq := &ocirequest.Request{ - Kind: ocirequest.ReqBlobStartUpload, - Repo: repo, - } - req, err := newRequest(ctx, rreq, nil) + scope := pushScope(repo) + req, err := newRequest(ctx, http.MethodPost, blobUploadURL(repo, nil), nil, scope) if err != nil { return oci.Descriptor{}, err } @@ -163,10 +152,8 @@ func (c *client) PushBlob(ctx context.Context, repo string, desc oci.Descriptor, // We've got the upload location. Now PUT the content. ctx = ociauth.ContextWithRequestInfo(ctx, ociauth.RequestInfo{ - RequiredScope: scopeForRequest(rreq), + RequiredScope: scope, }) - // Note: we can't use ocirequest.Request here because that's - // specific to the ociserver implementation in this case. req, err = http.NewRequestWithContext(ctx, "PUT", "", r) if err != nil { return oci.Descriptor{}, err @@ -175,7 +162,7 @@ func (c *client) PushBlob(ctx context.Context, repo string, desc oci.Descriptor, req.ContentLength = desc.Size req.Header.Set("Content-Type", "application/octet-stream") // TODO: per the spec, the content-range header here is unnecessary. - req.Header.Set("Content-Range", ocirequest.RangeString(0, desc.Size)) + req.Header.Set("Content-Range", contentRange(0, desc.Size)) resp, err = c.do(req, http.StatusCreated) if err != nil { return oci.Descriptor{}, err @@ -190,14 +177,16 @@ func (c *client) PushBlob(ctx context.Context, repo string, desc oci.Descriptor, // TODO: make this default configurable. const defaultChunkSize = 64 * 1024 -func (c *client) PushBlobChunked(ctx context.Context, repo string, chunkSize int) (oci.BlobWriter, error) { +// PushBlobChunked starts a chunked blob upload to the given repository. +func (c *Client) PushBlobChunked(ctx context.Context, repo string, chunkSize int) (oci.BlobWriter, error) { if chunkSize <= 0 { chunkSize = defaultChunkSize } - resp, err := c.doRequest(ctx, &ocirequest.Request{ - Kind: ocirequest.ReqBlobStartUpload, - Repo: repo, - }, http.StatusAccepted) + req, err := newRequest(ctx, http.MethodPost, blobUploadURL(repo, nil), nil, pushScope(repo)) + if err != nil { + return nil, err + } + resp, err := c.do(req, http.StatusAccepted) if err != nil { return nil, err } @@ -207,11 +196,7 @@ func (c *client) PushBlobChunked(ctx context.Context, repo string, chunkSize int return nil, err } ctx = ociauth.ContextWithRequestInfo(ctx, ociauth.RequestInfo{ - RequiredScope: ociauth.NewScope(ociauth.ResourceScope{ - ResourceType: "repository", - Resource: repo, - Action: "push", - }), + RequiredScope: pushScope(repo), }) return &blobWriter{ ctx: ctx, @@ -222,7 +207,8 @@ func (c *client) PushBlobChunked(ctx context.Context, repo string, chunkSize int }, nil } -func (c *client) PushBlobChunkedResume(ctx context.Context, repo string, id string, offset int64, chunkSize int) (oci.BlobWriter, error) { +// PushBlobChunkedResume resumes a previous chunked blob upload. +func (c *Client) PushBlobChunkedResume(ctx context.Context, repo string, id string, offset int64, chunkSize int) (oci.BlobWriter, error) { if id == "" { return nil, fmt.Errorf("id must be non-empty to resume a chunked upload") } @@ -234,17 +220,8 @@ func (c *client) PushBlobChunkedResume(ctx context.Context, repo string, id stri case offset == -1: // Try to find what offset we're meant to be writing at // by doing a GET to the location. - // TODO does resuming an upload require push or pull scope or both? ctx := ociauth.ContextWithRequestInfo(ctx, ociauth.RequestInfo{ - RequiredScope: ociauth.NewScope(ociauth.ResourceScope{ - ResourceType: "repository", - Resource: repo, - Action: "push", - }, ociauth.ResourceScope{ - ResourceType: "repository", - Resource: repo, - Action: "pull", - }), + RequiredScope: pushScope(repo), }) req, err := http.NewRequestWithContext(ctx, "GET", id, nil) if err != nil { @@ -259,7 +236,7 @@ func (c *client) PushBlobChunkedResume(ctx context.Context, repo string, id stri return nil, fmt.Errorf("cannot get location from response: %v", err) } rangeStr := resp.Header.Get("Range") - p0, p1, ok := ocirequest.ParseRange(rangeStr) + p0, p1, ok := parseContentRange(rangeStr) if !ok { return nil, fmt.Errorf("invalid range %q in response", rangeStr) } @@ -286,11 +263,7 @@ func (c *client) PushBlobChunkedResume(ctx context.Context, repo string, id stri } } ctx = ociauth.ContextWithRequestInfo(ctx, ociauth.RequestInfo{ - RequiredScope: ociauth.NewScope(ociauth.ResourceScope{ - ResourceType: "repository", - Resource: repo, - Action: "push", - }), + RequiredScope: pushScope(repo), }) return &blobWriter{ ctx: ctx, @@ -303,7 +276,7 @@ func (c *client) PushBlobChunkedResume(ctx context.Context, repo string, id stri } type blobWriter struct { - client *client + client *Client chunkSize int ctx context.Context @@ -371,7 +344,7 @@ func (w *blobWriter) flush(buf []byte, commitDigest oci.Digest) error { req.ContentLength = int64(len(w.chunk) + len(buf)) // TODO: per the spec, the content-range header here is unnecessary // if we are doing a final PUT without a body. - req.Header.Set("Content-Range", ocirequest.RangeString(w.flushed, w.flushed+req.ContentLength)) + req.Header.Set("Content-Range", contentRange(w.flushed, w.flushed+req.ContentLength)) resp, err := w.client.do(req, expect) if err != nil { return err @@ -482,3 +455,32 @@ func chunkSizeFromResponse(resp *http.Response, chunkSize int) int { } return chunkSize } + +func blobUploadURL(repo string, query url.Values) string { + u := "/v2/" + repo + "/blobs/uploads/" + if len(query) > 0 { + u += "?" + query.Encode() + } + return u +} + +func contentRange(start, end int64) string { + end-- + if end < 0 { + end = 0 + } + return fmt.Sprintf("%d-%d", start, end) +} + +func parseContentRange(s string) (start, end int64, ok bool) { + p0s, p1s, ok := strings.Cut(s, "-") + if !ok { + return 0, 0, false + } + p0, err0 := strconv.ParseInt(p0s, 10, 64) + p1, err1 := strconv.ParseInt(p1s, 10, 64) + if p1 > 0 { + p1++ + } + return p0, p1, err0 == nil && err1 == nil +} diff --git a/ociregistry/README.md b/ociregistry/README.md deleted file mode 100644 index ea55cf6..0000000 --- a/ociregistry/README.md +++ /dev/null @@ -1,18 +0,0 @@ -# `oci` -In the top level package (`oci`) this module defines a [Go interface](./interface.go) that encapsulates the operations provided by an OCI -registry. - -Full reference documentation can be found [here](https://pkg.go.dev/cuelabs.dev/go/oci/oci). - -It also provides a lightweight in-memory implementation of that interface (`ocimem`) -and an HTTP server that implements the [OCI registry protocol](https://github.com/opencontainers/distribution-spec/blob/main/spec.md) on top of it. - -The server currently passes the [conformance tests](https://pkg.go.dev/github.com/opencontainers/distribution-spec/conformance). - -The aim is to provide an ergonomic interface for defining and layering -OCI registry implementations. - -Although the API is fairly stable, it's still in v0 currently, so incompatible changes can't be ruled out. - -The code was originally derived from the [go-containerregistry](https://pkg.go.dev/github.com/google/go-containerregistry/pkg/registry) registry, but has considerably diverged since then. -