feat: redeem offline credentials for DIAL_NATIVE services and report honest status

docs/open_api_core.yaml

@@ -14753,12 +14753,12 @@ components:
     OfflineCredentialsStatus:
       type: object
       properties:
-        available:
-          type: boolean
         connect:
           $ref: "#/components/schemas/OfflineCredentialsStatusConnect"
         connected:
           type: boolean
+        available:
+          type: boolean
     OfflineCredentialsStatusConnect:
       type: object
       properties:

server/src/main/java/com/epam/aidial/core/server/controller/ApplicationController.java

@@ -27,6 +27,7 @@
 import com.epam.aidial.core.server.service.ApplicationSchemaService;
 import com.epam.aidial.core.server.service.ApplicationService;
 import com.epam.aidial.core.server.service.DeploymentService;
+import com.epam.aidial.core.server.service.ExternalServiceStatusEnricher;
 import com.epam.aidial.core.server.service.PermissionDeniedException;
 import com.epam.aidial.core.server.service.UserExternalServiceService;
 import com.epam.aidial.core.server.util.CredentialsLocatorFactory;
@@ -461,6 +462,7 @@ private void enrichExternalServiceStatuses(ApplicationData data) {
             return;
         }
         String appId = appPart(data.getId());
+        ExternalServiceStatusEnricher enricher = new ExternalServiceStatusEnricher(context, resourceAuthSettingsService);
         for (Map.Entry<String, ExternalService> entry : services.entrySet()) {
             ResourceAuthSettings authSettings = entry.getValue().getAuthSettings();
             if (authSettings == null) {
@@ -470,7 +472,7 @@ private void enrichExternalServiceStatuses(ApplicationData data) {
                 String scopeId = CredentialsLocatorFactory.APPLICATIONS_PREFIX + appId
                         + CredentialsLocatorFactory.EXTERNAL_SERVICES_SEPARATOR + entry.getKey();
                 CredentialsLocator locator = CredentialsLocatorFactory.fromExternalServiceScope(scopeId, context);
-                resourceAuthSettingsService.setExternalServiceAuthStatuses(locator, authSettings, context.getUserId());
+                enricher.enrich(locator, authSettings);
             } catch (RuntimeException e) {
                 log.warn("Failed to compute external-service status for '{}' on '{}'", entry.getKey(), data.getId(), e);
             }

server/src/main/java/com/epam/aidial/core/server/controller/ExternalServiceCredentialsController.java

@@ -8,6 +8,7 @@
 import com.epam.aidial.core.config.ResourceAccessType;
 import com.epam.aidial.core.config.ResourceAuthSettings;
 import com.epam.aidial.core.credentials.data.credentials.AuthorizationHeader;
+import com.epam.aidial.core.credentials.data.credentials.BucketInfo;
 import com.epam.aidial.core.credentials.data.credentials.CredentialsDescriptor;
 import com.epam.aidial.core.credentials.data.credentials.CredentialsLocator;
 import com.epam.aidial.core.credentials.data.credentials.ResourceCredentials;
@@ -29,12 +30,15 @@
 import com.epam.aidial.core.server.data.OboCredentialsRequest;
 import com.epam.aidial.core.server.log.ExternalServiceAuditLog;
 import com.epam.aidial.core.server.security.AccessService;
+import com.epam.aidial.core.server.security.AccessTokenValidator;
 import com.epam.aidial.core.server.security.AppIdentityMatcher;
 import com.epam.aidial.core.server.security.EncryptionService;
 import com.epam.aidial.core.server.service.ApplicationService;
 import com.epam.aidial.core.server.service.ConsentRequiredException;
+import com.epam.aidial.core.server.service.OfflineCredentialsRequiredException;
 import com.epam.aidial.core.server.service.PermissionDeniedException;
 import com.epam.aidial.core.server.service.UserExternalServiceService;
+import com.epam.aidial.core.server.util.CredentialsDescriptorFactory;
 import com.epam.aidial.core.server.util.CredentialsLocatorFactory;
 import com.epam.aidial.core.server.util.ProxyUtil;
 import com.epam.aidial.core.server.util.ResourceDescriptorFactory;
@@ -64,6 +68,7 @@ public class ExternalServiceCredentialsController {
     private final EncryptionService encryptionService;
     private final ApplicationService applicationService;
     private final UserExternalServiceService userExternalServiceService;
+    private final AccessTokenValidator accessTokenValidator;
 
     public ExternalServiceCredentialsController(Proxy proxy, ProxyContext context) {
         this.context = context;
@@ -74,6 +79,7 @@ public ExternalServiceCredentialsController(Proxy proxy, ProxyContext context) {
         this.resourceCredentialsService = proxy.getResourceCredentialsService();
         this.authorizationHeaderProvider = proxy.getAuthorizationHeaderProvider();
         this.userExternalServiceService = proxy.getUserExternalServiceService();
+        this.accessTokenValidator = proxy.getTokenValidator();
     }
 
     @ApiOperation(
@@ -290,6 +296,14 @@ public Future<?> getOboCredentials() {
                         ExternalService externalService = resolveExternalServiceDefinition(
                                 app.application, scope[0], scope[1], request.getOwnerUserId());
                         ResourceAuthSettings authSettings = externalService.getAuthSettings();
+
+                        if (AuthenticationType.DIAL_NATIVE.equals(authSettings.getAuthenticationType())) {
+                            ExternalServiceCredentialsResponse dialNative =
+                                    redeemOfflineCredentials(request, scope[0], scope[1]);
+                            ExternalServiceAuditLog.oboRetrieval(context, scope[0], scope[1], request.getOwnerUserId(), null);
+                            return dialNative;
+                        }
+
                         CredentialsLocator locator = CredentialsLocatorFactory.fromExternalServiceScopeForOwner(
                                 request.getUrl(), request.getOwnerUserId(), context);
                         ResourceCredentials credentials = resourceCredentialsService.getRefreshedUserCredentials(
@@ -319,6 +333,95 @@ public Future<?> getOboCredentials() {
         return Future.succeededFuture();
     }
 
+    /**
+     * Redemption for a DIAL-native service: the owner acts through their own offline credentials, and the
+     * application is authorized by an administrator's consent rather than by anything it holds.
+     */
+    private ExternalServiceCredentialsResponse redeemOfflineCredentials(OboCredentialsRequest request,
+                                                                        String appPart,
+                                                                        String serviceId) {
+        requireAdminConsent(request.getUrl(), appPart, serviceId);
+
+        CredentialsDescriptor descriptor =
+                CredentialsDescriptorFactory.offlineCredentialsForUser(context, request.getOwnerUserId());
+        ResourceCredentials stored = resourceCredentialsService.getResourceCredentials(descriptor);
+        if (stored == null) {
+            throw new OfflineCredentialsRequiredException(
+                    "The owner has not enabled offline access, so nothing can act on their behalf");
+        }
+
+        ResourceAuthSettings offlineClient = resolveOfflineClient(stored.getIssuer());
+
+        CredentialsLocator locator = new CredentialsLocator(descriptor.getResourceId(),
+                Map.of(CredentialsLevel.USER, new BucketInfo(descriptor.getBucketName(), descriptor.getBucketLocation())));
+        ResourceCredentials credentials;
+        try {
+            credentials = resourceCredentialsService.getRefreshedUserCredentials(
+                    locator, offlineClient, request.getOwnerUserId());
+        } catch (HttpException e) {
+            throw translateRefreshFailure(e);
+        }
+        if (credentials == null) {
+            throw new OfflineCredentialsRequiredException(
+                    "The owner's offline credentials are no longer valid; they must connect again");
+        }
+        // getRefreshedUserCredentials returns the record un-refreshed without consent, leaving the refusal here.
+        if (!credentials.isOfflineUsageConsent()) {
+            throw new OfflineCredentialsRequiredException(
+                    "The owner's credentials do not permit offline use; they must connect again");
+        }
+        return toCredentialsResponse(credentials, request.getUrl());
+    }
+
+    /**
+     * The offline client of the identity provider that issued the stored credentials. Both failures are server-side
+     * state — a provider dropped from the settings, or one never configured for offline use — not a bad request.
+     */
+    private ResourceAuthSettings resolveOfflineClient(String issuer) {
+        ResourceAuthSettings offlineClient;
+        try {
+            offlineClient = accessTokenValidator.resolveOfflineClientByIssuer(issuer);
+        } catch (IllegalArgumentException e) {
+            throw new HttpException(HttpStatus.SERVICE_UNAVAILABLE,
+                    "The identity provider that issued the owner's offline credentials is no longer configured");
+        }
+        if (offlineClient == null) {
+            throw new HttpException(HttpStatus.SERVICE_UNAVAILABLE,
+                    "Offline credentials are not configured for the identity provider that issued them");
+        }
+        return offlineClient;
+    }
+
+    /** An administrator must have approved this application's use of the service; declaring it grants nothing. */
+    private void requireAdminConsent(String url, String appPart, String serviceId) {
+        CredentialsLocator locator = CredentialsLocatorFactory.fromExternalServiceScope(url, context);
+        CredentialsDescriptor consent = locator.getCredentialsDescriptors().get(CredentialsLevel.APPLICATION);
+        ResourceCredentials record = consent == null ? null : resourceCredentialsService.getResourceCredentials(consent);
+        // Only a record the consent endpoint wrote counts. The same APPLICATION-level slot holds ordinary
+        // credentials while a service is OAUTH/API_KEY, and a leftover from before a switch to DIAL_NATIVE
+        // would otherwise pass as an administrator's approval the app owner granted themselves.
+        if (record == null || !AuthenticationType.DIAL_NATIVE.equals(record.getAuthenticationType())) {
+            throw new ConsentRequiredException(
+                    "Application '%s' is not approved to use '%s'".formatted(appPart, serviceId));
+        }
+    }
+
+    /**
+     * Keeps a permanent failure distinguishable from a retryable one. A 401 here means the <i>owner's</i> token was
+     * rejected, not the caller's, so it must not surface as one.
+     */
+    private RuntimeException translateRefreshFailure(HttpException e) {
+        if (e.getStatus() == HttpStatus.UNAUTHORIZED) {
+            return new OfflineCredentialsRequiredException(
+                    "The owner's offline credentials were rejected by the identity provider; they must connect again");
+        }
+        if (e.getStatus().getCode() >= 500) {
+            return new HttpException(HttpStatus.BAD_GATEWAY,
+                    "The identity provider could not be reached; this run may be retried");
+        }
+        return e;
+    }
+
     private ExternalServiceCredentialsResponse toCredentialsResponse(ResourceCredentials credentials, String url) {
         AuthorizationHeader header = authorizationHeaderProvider.createAuthorizationHeader(credentials);
         if (header == null) {

server/src/main/java/com/epam/aidial/core/server/controller/ExternalServiceManagementController.java

@@ -25,6 +25,7 @@
 import com.epam.aidial.core.server.security.EncryptionService;
 import com.epam.aidial.core.server.service.ApplicationService;
 import com.epam.aidial.core.server.service.ExternalServiceService;
+import com.epam.aidial.core.server.service.ExternalServiceStatusEnricher;
 import com.epam.aidial.core.server.service.PermissionDeniedException;
 import com.epam.aidial.core.server.service.UserExternalServiceService;
 import com.epam.aidial.core.server.util.CredentialsLocatorFactory;
@@ -60,6 +61,7 @@ public class ExternalServiceManagementController {
     private final EncryptionService encryptionService;
     private final ResourceCredentialsService resourceCredentialsService;
     private final ResourceAuthSettingsService resourceAuthSettingsService;
+    private final ExternalServiceStatusEnricher statusEnricher;
 
     public ExternalServiceManagementController(Proxy proxy, ProxyContext context) {
         this.context = context;
@@ -71,6 +73,7 @@ public ExternalServiceManagementController(Proxy proxy, ProxyContext context) {
         this.encryptionService = proxy.getEncryptionService();
         this.resourceCredentialsService = proxy.getResourceCredentialsService();
         this.resourceAuthSettingsService = proxy.getResourceAuthSettingsService();
+        this.statusEnricher = new ExternalServiceStatusEnricher(context, resourceAuthSettingsService);
     }
 
     @ApiOperation(
@@ -333,7 +336,7 @@ private ExternalServiceData toData(String appId, String serviceId, ExternalServi
         ResourceAuthSettings safe = authSettings == null ? null : authSettings.withoutSecrets();
         if (withStatus && safe != null) {
             CredentialsLocator locator = CredentialsLocatorFactory.fromExternalServiceScope(scopeId(appId, serviceId), context);
-            resourceAuthSettingsService.setExternalServiceAuthStatuses(locator, safe, context.getUserId());
+            statusEnricher.enrich(locator, safe);
         }
         return new ExternalServiceData()
                 .setId(serviceId)

server/src/main/java/com/epam/aidial/core/server/controller/ResourceController.java

@@ -24,6 +24,7 @@
 import com.epam.aidial.core.server.service.ApplicationSchemaService;
 import com.epam.aidial.core.server.service.ApplicationService;
 import com.epam.aidial.core.server.service.DeploymentService;
+import com.epam.aidial.core.server.service.ExternalServiceStatusEnricher;
 import com.epam.aidial.core.server.service.ExternalServicesWriteMode;
 import com.epam.aidial.core.server.service.PermissionDeniedException;
 import com.epam.aidial.core.server.service.ToolSetService;
@@ -590,6 +591,8 @@ private void enrichExternalServiceStatuses(ResourceDescriptor descriptor, Applic
         if (services == null || services.isEmpty()) {
             return;
         }
+        ExternalServiceStatusEnricher enricher = new ExternalServiceStatusEnricher(
+                context, proxy.getResourceAuthSettingsService());
         for (Map.Entry<String, ExternalService> entry : services.entrySet()) {
             ResourceAuthSettings authSettings = entry.getValue() == null ? null : entry.getValue().getAuthSettings();
             if (authSettings == null) {
@@ -598,7 +601,7 @@ private void enrichExternalServiceStatuses(ResourceDescriptor descriptor, Applic
             try {
                 String scopeId = descriptor.getUrl() + CredentialsLocatorFactory.EXTERNAL_SERVICES_SEPARATOR + entry.getKey();
                 CredentialsLocator locator = CredentialsLocatorFactory.fromExternalServiceScope(scopeId, context);
-                proxy.getResourceAuthSettingsService().setExternalServiceAuthStatuses(locator, authSettings, context.getUserId());
+                enricher.enrich(locator, authSettings);
             } catch (RuntimeException e) {
                 log.warn("Failed to compute external-service status for '{}' on '{}'", entry.getKey(), descriptor.getUrl(), e);
             }

server/src/main/java/com/epam/aidial/core/server/service/ExternalServiceStatusEnricher.java

@@ -0,0 +1,46 @@
+package com.epam.aidial.core.server.service;
+
+import com.epam.aidial.core.config.AuthenticationType;
+import com.epam.aidial.core.config.ResourceAuthSettings;
+import com.epam.aidial.core.config.ResourceAuthStatus;
+import com.epam.aidial.core.credentials.data.credentials.CredentialsLocator;
+import com.epam.aidial.core.credentials.service.ResourceAuthSettingsService;
+import com.epam.aidial.core.server.ProxyContext;
+import com.epam.aidial.core.server.util.CredentialsDescriptorFactory;
+
+/**
+ * Fills in the auth statuses of an application's external services for one response.
+ *
+ * <p>A DIAL-native service has no per-service credential, so its user level is answered from the caller's
+ * platform-wide offline credentials instead of its (always empty) USER-level records.
+ *
+ * <p>Memoizes that lookup, so create one per response rather than per service.
+ */
+public class ExternalServiceStatusEnricher {
+
+    private final ProxyContext context;
+    private final ResourceAuthSettingsService resourceAuthSettingsService;
+    private Boolean offlineCredentials;
+
+    public ExternalServiceStatusEnricher(ProxyContext context, ResourceAuthSettingsService resourceAuthSettingsService) {
+        this.context = context;
+        this.resourceAuthSettingsService = resourceAuthSettingsService;
+    }
+
+    public void enrich(CredentialsLocator credentialsLocator, ResourceAuthSettings authSettings) {
+        resourceAuthSettingsService.setExternalServiceAuthStatuses(credentialsLocator, authSettings, context.getUserId());
+        if (authSettings.getAuthenticationType() == AuthenticationType.DIAL_NATIVE) {
+            authSettings.setUserLevelAuthStatus(hasOfflineCredentials()
+                    ? ResourceAuthStatus.SIGNED_IN : ResourceAuthStatus.SIGNED_OUT);
+        }
+    }
+
+    private boolean hasOfflineCredentials() {
+        if (offlineCredentials == null) {
+            // A userless caller (an API key) is not a person, so it holds no offline credentials by definition.
+            offlineCredentials = context.getUserId() != null && resourceAuthSettingsService.hasUnexpiredCredentials(
+                    CredentialsDescriptorFactory.offlineCredentials(context));
+        }
+        return offlineCredentials;
+    }
+}

server/src/main/java/com/epam/aidial/core/server/service/OfflineCredentialsRequiredException.java

@@ -0,0 +1,12 @@
+package com.epam.aidial.core.server.service;
+
+import com.epam.aidial.core.storage.http.HttpException;
+import com.epam.aidial.core.storage.http.HttpStatus;
+
+/** The owner has no usable offline credentials — permanent until they connect again, unlike an IdP outage. */
+public class OfflineCredentialsRequiredException extends HttpException {
+
+    public OfflineCredentialsRequiredException(String message) {
+        super(HttpStatus.CONFLICT, message);
+    }
+}

server/src/main/java/com/epam/aidial/core/server/util/CredentialsDescriptorFactory.java

@@ -85,6 +85,12 @@ public static CredentialsDescriptor offlineCredentials(ProxyContext proxyContext
         return new CredentialsDescriptor(OFFLINE_CREDENTIALS_ID, bucket.name(), bucket.location());
     }
 
+    /** The same record for an arbitrary owner — the redemption path, where the caller is not the owner. */
+    public static CredentialsDescriptor offlineCredentialsForUser(ProxyContext proxyContext, String ownerUserId) {
+        BucketInfo bucket = getUserBucketInfoForUser(proxyContext, ownerUserId);
+        return new CredentialsDescriptor(OFFLINE_CREDENTIALS_ID, bucket.name(), bucket.location());
+    }
+
     public static BucketInfo getPublicBucketInfo() {
         return new BucketInfo(ResourceDescriptor.PUBLIC_BUCKET, ResourceDescriptor.PUBLIC_LOCATION);
     }