feat: add configuration for building images from private repositories

cypress-tests/cypress/e2e/v2/dataConnectors.cy.ts

@@ -10,16 +10,19 @@ import {
   deleteDataConnector,
 } from "../../support/utils/dataConnectors";
 import { login } from "../../support/utils/general";
+import { verifySearchIndexing } from "../../support/utils/search";
 
 const sessionId = ["dataConnectors", getRandomString()];
+const searchDataConnectorType = "type:DataConnector";
+const searchDataConnectorSlug = "slug:";
 
 describe("Data Connectors", () => {
   const randomString = getRandomString();
   const projectName = `Project for data connector tests ${randomString}`;
   const projectSlug = `project-for-data-connector-tests-${randomString}`;
   let userNamespace: string;
   let dataConnectorName: string;
-  let projectId: string;
+  let projectId: string | undefined;
   let groupName: string;
   let groupSlug: string;
 
@@ -43,7 +46,7 @@ describe("Data Connectors", () => {
   });
 
   after(() => {
-    deleteProject(projectId);
+    if (projectId) deleteProject(projectId);
   });
 
   beforeEach(() => {
@@ -121,8 +124,6 @@ describe("Data Connectors", () => {
       cy.getDataCy("data-connector-menu-dropdown").click();
       cy.getDataCy("data-connector-delete").click();
     });
-
-    // Confirm deletion by typing the slug
     cy.getDataCy("delete-confirmation-input").type(dataConnectorName);
     cy.getDataCy("delete-data-connector-modal-button").click();
 
@@ -248,7 +249,6 @@ describe("Data Connectors", () => {
       projectId,
     );
 
-    // ? Currently, data connectors newly linked might not appear immediately
     visitCurrentProject();
     cy.getDataCy("data-connector-box")
       .find(`[data-cy=data-connector-name]`)
@@ -279,70 +279,54 @@ describe("Data Connectors", () => {
     cy.getDataCy("data-connector-edit-close-button").click();
 
     // Verify the data connector is present with the edited name
-    visitCurrentProject();
     cy.getDataCy("data-connector-box")
       .find(`[data-cy=data-connector-name]`)
       .contains(newName);
   });
 
-  it("Link an existing data connector to a project", () => {
-    // Create a data connector not linked to a project
+  it("Link and unlink an existing data connector to a project", () => {
+    // Create a data connector not linked to a project and check it has been indexed
     const dataConnectorIdentifier = `${userNamespace}/${dataConnectorName}`;
     createDataConnector(dataConnectorIdentifier);
+    verifySearchIndexing(
+      `${searchDataConnectorType} ${searchDataConnectorSlug}${dataConnectorName}`,
+      1,
+    );
 
-    // Now link the data connector to the project
+    // Link the data connector to the project
     visitCurrentProject();
     cy.getDataCy("add-data-connector").click();
-    cy.getDataCy("project-data-controller-mode-link").click();
-
-    // Enter the data connector identifier
-    cy.get("#data-connector-identifier")
+    cy.getDataCy("data-connector-search-input")
       .should("be.empty")
       .type(dataConnectorIdentifier);
-    cy.getDataCy("link-data-connector-button").click();
+    cy.getDataCy("data-connector-search-body")
+      .contains("[data-cy=link-data-connector-list-item]", dataConnectorName)
+      .find(`[data-cy=data-connector-link-button]`)
+      .click();
 
     // Verify the data connector is linked to the project
-    visitCurrentProject();
+    cy.getDataCy("project-data-connector-connect-header")
+      .find('button[data-bs-dismiss="modal"]')
+      .click();
     cy.getDataCy("data-connector-box")
       .find(`[data-cy=data-connector-name]`)
       .contains(dataConnectorName);
-  });
 
-  it("Unlink a data connector from a project", () => {
-    // Create a data connector not linked to a project
-    const dataConnectorIdentifier = `${userNamespace}/${dataConnectorName}`;
-    createDataConnector(dataConnectorIdentifier);
-
-    // Now link the data connector to the project
-    visitCurrentProject();
-    cy.getDataCy("add-data-connector").click();
-    cy.getDataCy("project-data-controller-mode-link").click();
-
-    // Enter the data connector identifier
-    cy.get("#data-connector-identifier")
-      .should("be.empty")
-      .type(dataConnectorIdentifier);
-    cy.getDataCy("link-data-connector-button").click();
-
-    // ? Currently, data connectors newly linked might not appear immediately
-    visitCurrentProject();
+    // Unlink the data connector from the project
     cy.getDataCy("data-connector-box")
       .find(`[data-cy=data-connector-name]`)
       .contains(dataConnectorName)
       .click();
-
     cy.getDataCy("data-connector-view").within(() => {
       cy.getDataCy("data-connector-title")
         .should("be.visible")
         .contains(dataConnectorName);
       cy.getDataCy("data-connector-menu-dropdown").click();
-      cy.getDataCy("data-connector-unlink").should("be.visible").click();
+      cy.getDataCy("data-connector-unlink").click();
     });
-
     cy.getDataCy("delete-data-connector-modal-button").click();
 
     // Verify the data connector is no longer linked to the project
-    visitCurrentProject();
     cy.getDataCy("data-connector-box")
       .contains(`[data-cy=data-connector-name]`, dataConnectorName)
       .should("not.exist");
@@ -359,31 +343,38 @@ describe("Data Connectors", () => {
       slug: otherProjectName,
       visibility: "private",
     }).then((response) => {
-      const otherProjectId = response.body.id;
+      const otherProjectId = response.body.id ?? "";
 
       // Defer-delete the other project (which will also delete the data connector)
       cy.defer(() => {
-        deleteProject(otherProjectId);
+        if (otherProjectId) deleteProject(otherProjectId);
       });
 
-      // Create a data connector in the other project
+      // Create a data connector in the other project and check it has been indexed
       const dataConnectorIdentifier = `${userNamespace}/${otherProjectName}/${dataConnectorName}`;
       createDataConnector(dataConnectorIdentifier, otherProjectId);
+      verifySearchIndexing(
+        `${searchDataConnectorType} ${searchDataConnectorSlug}${dataConnectorName}`,
+        1,
+      );
 
       // Navigate to the main project
       visitCurrentProject();
 
       // Link the data connector from the other project to the main project
       cy.getDataCy("add-data-connector").click();
-      cy.getDataCy("project-data-controller-mode-link").click();
-
-      cy.get("#data-connector-identifier")
+      cy.getDataCy("data-connector-search-input")
         .should("be.empty")
         .type(dataConnectorIdentifier);
-      cy.getDataCy("link-data-connector-button").click();
+      cy.getDataCy("data-connector-search-body")
+        .contains("[data-cy=link-data-connector-list-item]", dataConnectorName)
+        .find(`[data-cy=data-connector-link-button]`)
+        .click();
 
       // Verify the data connector is linked to the main project
-      visitCurrentProject();
+      cy.getDataCy("project-data-connector-connect-header")
+        .find('button[data-bs-dismiss="modal"]')
+        .click();
       cy.getDataCy("data-connector-box")
         .find(`[data-cy=data-connector-name]`)
         .contains(dataConnectorName)
@@ -395,12 +386,11 @@ describe("Data Connectors", () => {
           .should("be.visible")
           .contains(dataConnectorName);
         cy.getDataCy("data-connector-menu-dropdown").click();
-        cy.getDataCy("data-connector-unlink").should("be.visible").click();
+        cy.getDataCy("data-connector-unlink").click();
       });
       cy.getDataCy("delete-data-connector-modal-button").click();
 
       // Verify the data connector is no longer linked to the main project
-      visitCurrentProject();
       cy.getDataCy("data-connector-box")
         .contains(`[data-cy=data-connector-name]`, dataConnectorName)
         .should("not.exist");
@@ -468,21 +458,28 @@ describe("Data Connectors", () => {
     // Create a data connector owned by the group
     const dataConnectorIdentifier = `${groupSlug}/${dataConnectorName}`;
     createDataConnector(dataConnectorIdentifier);
+    verifySearchIndexing(
+      `${searchDataConnectorType} ${searchDataConnectorSlug}${dataConnectorName}`,
+      1,
+    );
 
     // Navigate to the user's project
     visitCurrentProject();
 
     // Link the group data connector to the user's project
     cy.getDataCy("add-data-connector").click();
-    cy.getDataCy("project-data-controller-mode-link").click();
-
-    // Enter the data connector identifier
-    cy.get("#data-connector-identifier")
+    cy.getDataCy("data-connector-search-input")
       .should("be.empty")
       .type(dataConnectorIdentifier);
-    cy.getDataCy("link-data-connector-button").click();
+    cy.getDataCy("data-connector-search-body")
+      .contains("[data-cy=link-data-connector-list-item]", dataConnectorName)
+      .find(`[data-cy=data-connector-link-button]`)
+      .click();
 
     // Verify the data connector is linked to the project
+    cy.getDataCy("project-data-connector-connect-header")
+      .find('button[data-bs-dismiss="modal"]')
+      .click();
     cy.getDataCy("data-connector-box")
       .find(`[data-cy=data-connector-name]`)
       .contains(dataConnectorName)
@@ -493,8 +490,8 @@ describe("Data Connectors", () => {
       cy.getDataCy("data-connector-title")
         .should("be.visible")
         .contains(dataConnectorName);
-      cy.getDataCy("data-connector-menu-dropdown").should("be.visible").click();
-      cy.getDataCy("data-connector-unlink").should("be.visible").click();
+      cy.getDataCy("data-connector-menu-dropdown").click();
+      cy.getDataCy("data-connector-unlink").click();
     });
     cy.getDataCy("delete-data-connector-modal-button").click();
 

cypress-tests/cypress/support/utils/dataConnectors.ts

@@ -46,11 +46,9 @@ export function createDataConnector(
         visibility: body.visibility || "private",
         description: body.description || "Test data connector description",
         storage: {
-          storage_type: "s3",
           configuration: {
             type: "s3",
             provider: "AWS",
-            region: "us-east-1",
           },
           source_path: "giab",
           target_path: body.slug || "/",

helm-chart/renku/requirements.yaml

@@ -13,7 +13,7 @@ dependencies:
     condition: redis.install
   - name: amalthea-sessions
     repository: "https://swissdatasciencecenter.github.io/helm-charts/"
-    version: "0.27.2"
+    version: "0.28.0"
   - name: dlf-chart
     repository: "https://swissdatasciencecenter.github.io/datashim/"
     version: "0.3.9-renku-2"

helm-chart/renku/templates/data-service/deployment.yaml

@@ -75,6 +75,8 @@ spec:
               value: /secrets/encryptionKey/encryptionKey
             - name: SECRETS_SERVICE_PUBLIC_KEY_PATH
               value: /secrets/publicKey/publicKey
+            - name: INTERNAL_AUTHN_SECRET_KEY_PATH
+              value: /secrets/internalSecretKey/secretKey
             - name: K8S_NAMESPACE
               value: {{ .Release.Namespace | quote }}
             - name: MAX_PINNED_PROJECTS
@@ -130,6 +132,8 @@ spec:
               value: {{ .Values.dataService.imageBuilders.enabled | quote }}
             - name: BUILD_OUTPUT_IMAGE_PREFIX
               value: {{ .Values.dataService.imageBuilders.outputImagePrefix | default "" | quote }}
+            - name: BUILD_OUTPUT_PRIVATE_IMAGE_PREFIX
+              value: {{ .Values.dataService.imageBuilders.outputPrivateImagePrefix | default "" | quote }}
             - name: BUILD_BUILDER_IMAGE
               value: {{ .Values.dataService.imageBuilders.builderImage | default "" | quote }}
             - name: BUILD_RUN_IMAGE
@@ -138,6 +142,10 @@ spec:
               value: {{ .Values.dataService.imageBuilders.strategyName | default "" | quote }}
             - name: BUILD_PUSH_SECRET_NAME
               value: {{ .Values.dataService.imageBuilders.pushSecretName | default "" | quote }}
+            - name: BUILD_PUSH_PRIVATE_SECRET_NAME
+              value: {{ .Values.dataService.imageBuilders.pushPrivateSecretName | default "" | quote }}
+            - name: BUILD_PULL_PRIVATE_SECRET_NAME
+              value: {{ .Values.dataService.imageBuilders.pullPrivateSecretName | default "" | quote }}
             - name: BUILD_RUN_RETENTION_AFTER_FAILED_SECONDS
               value: {{ .Values.dataService.imageBuilders.buildRunRetentionAfterFailedSeconds | default "" | quote }}
             - name: BUILD_RUN_RETENTION_AFTER_SUCCEEDED_SECONDS
@@ -159,7 +167,7 @@ spec:
             - name: V1_SESSIONS_ENABLED
               value: {{ .Values.ui.client.supportLegacySessions | default false | quote }}
             - name: ENABLE_INTERNAL_GITLAB
-              value: {{ .Values.enableInternalGitlab | default false | quote }}
+              value: "false"
             - name: POSTHOG_ENABLED
               value: {{ .Values.posthog.enabled | quote }}
             - name: LOG_FORMAT_STYLE
@@ -215,6 +223,9 @@ spec:
             - mountPath: "/secrets/publicKey"
               name: secret-service-public-key
               readOnly: true
+            - mountPath: "/secrets/internalSecretKey"
+              name: internal-authn-key
+              readOnly: true
             {{- if .Values.dataService.remoteClustersKubeconfigSecretName }}
             - name: remote-cluster-kubeconfigs
               mountPath: "/secrets/kube_configs"
@@ -271,6 +282,12 @@ spec:
             items:
             - key: publicKey
               path: publicKey
+        - name: internal-authn-key
+          secret:
+            secretName: {{ template "renku.fullname" . }}-internal-authn
+            items:
+            - key: secretKey
+              path: secretKey
         {{- if .Values.dataService.remoteClustersKubeconfigSecretName }}
         - name: remote-cluster-kubeconfigs
           secret:

helm-chart/renku/templates/data-service/deployment_k8s_watcher.yaml

@@ -69,6 +69,15 @@ spec:
               value: {{ .Values.dataService.k8sWatcher.sentry.environment | quote }}
             - name: SENTRY_SAMPLE_RATE
               value: {{ .Values.dataService.k8sWatcher.sentry.sampleRate | quote }}
+            - name: AUTHZ_DB_HOST
+              value: {{ include "renku.fullname" . }}-authz
+            - name: AUTHZ_DB_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: {{ template "renku.fullname" . }}-authz
+                  key: SPICEDB_GRPC_PRESHARED_KEY
+            - name: AUTHZ_DB_GRPC_PORT
+              value: "50051"
             {{- if .Values.dataService.remoteClustersKubeconfigSecretName }}
             - name: K8S_CONFIGS_ROOT
               value: "/secrets/kube_configs"

helm-chart/renku/templates/gateway/configmap.yaml

@@ -36,7 +36,7 @@ data:
           audience: renku
           authorizedParty: renku-cli
     revproxy:
-      enableInternalGitlab: {{ .Values.enableInternalGitlab | default false }}
+      enableInternalGitlab: "false"
       renkuBaseUrl: {{ include "renku.baseUrl" . | quote }}
       externalGitlabUrl: {{ .Values.global.gitlab.url | default "" | quote }}
       k8sNamespace: {{ .Release.Namespace }}
@@ -47,7 +47,7 @@ data:
         uiserver: {{ printf "http://%s" (include "ui-server.fullname" .) | quote }} 
         search: {{ printf "http://%s-search-api" .Release.Name | quote }}
     login:
-      enableInternalGitlab: {{ .Values.enableInternalGitlab | default false }}
+      enableInternalGitlab: "false"
       renkuBaseUrl: {{ include "renku.baseUrl" . | quote }}
       loginRoutesBasePath: "/api/auth"
       defaultAppRedirectURL: {{ include "renku.baseUrl" . | quote }}
@@ -60,14 +60,6 @@ data:
           scopes: ["profile", "email", "openid", "microprofile-jwt"]
           callbackURI:  {{ printf "%s/api/auth/callback" (include "renku.baseUrl" .) }}
           usePKCE: false
-        {{- if .Values.enableInternalGitlab }}
-        gitlab:
-          issuer: {{ .Values.global.gitlab.url | quote }}
-          clientID: {{ .Values.gateway.gitlabClientId | default .Values.global.gateway.gitlabClientId | quote }}
-          scopes: ["openid", "api", "read_user", "read_repository"] 
-          callbackURI: {{ printf "%s/api/auth/callback" (include "renku.baseUrl" .) }}
-          usePKCE: false
-        {{- end }}
       oldGitLabLogout: {{ .Values.gateway.oldGitLabLogout | default false }}
       logoutGitLabUponRenkuLogout: {{ .Values.gateway.logoutGitLabUponRenkuLogout | default true }}
     redis: