diff --git a/.github/workflows/auto-cherry-pick-labeled-prs.yaml b/.github/workflows/auto-cherry-pick-labeled-prs.yaml index d3c8529fea42..30168ca04bf1 100644 --- a/.github/workflows/auto-cherry-pick-labeled-prs.yaml +++ b/.github/workflows/auto-cherry-pick-labeled-prs.yaml @@ -5,13 +5,14 @@ run-name: OpenMetadata release cherry-pick PR #${{ github.event.pull_request.num # yamllint disable-line rule:truthy on: - pull_request: + pull_request_target: types: [closed] branches: - main permissions: contents: write pull-requests: write + issues: write env: CURRENT_RELEASE_ENDPOINT: ${{ vars.CURRENT_RELEASE_ENDPOINT }} # Endpoint that returns the current release version in json format jobs: diff --git a/.github/workflows/playwright-mysql-e2e-skip.yml b/.github/workflows/playwright-mysql-e2e-skip.yml index 29b27a8287ac..54298c6561d1 100644 --- a/.github/workflows/playwright-mysql-e2e-skip.yml +++ b/.github/workflows/playwright-mysql-e2e-skip.yml @@ -31,9 +31,9 @@ jobs: playwright-ci-mysql: runs-on: ubuntu-latest strategy: - fail-fast: false - matrix: - shardIndex: [1, 2] - shardTotal: [2] + fail-fast: false + matrix: + shardIndex: [1, 2] + shardTotal: [2] steps: - run: 'echo "Step is not required"' diff --git a/.github/workflows/playwright-mysql-e2e.yml b/.github/workflows/playwright-mysql-e2e.yml index dd97f72597a6..0857cc61c65e 100644 --- a/.github/workflows/playwright-mysql-e2e.yml +++ b/.github/workflows/playwright-mysql-e2e.yml @@ -36,7 +36,7 @@ concurrency: jobs: playwright-ci-mysql: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 environment: test strategy: fail-fast: false diff --git a/.github/workflows/playwright-postgresql-e2e-skip.yml b/.github/workflows/playwright-postgresql-e2e-skip.yml index 119e36fbb650..f9be1a79152c 100644 --- a/.github/workflows/playwright-postgresql-e2e-skip.yml +++ b/.github/workflows/playwright-postgresql-e2e-skip.yml @@ -31,9 +31,9 @@ jobs: playwright-ci-postgresql: runs-on: ubuntu-latest strategy: - fail-fast: false - matrix: - shardIndex: [1, 2] - shardTotal: [2] + fail-fast: false + matrix: + shardIndex: [1, 2] + shardTotal: [2] steps: - run: 'echo "Step is not required"' diff --git a/.github/workflows/playwright-postgresql-e2e.yml b/.github/workflows/playwright-postgresql-e2e.yml index b3ea6c978623..11d8d4408f9b 100644 --- a/.github/workflows/playwright-postgresql-e2e.yml +++ b/.github/workflows/playwright-postgresql-e2e.yml @@ -36,7 +36,7 @@ concurrency: jobs: playwright-ci-postgresql: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 environment: test strategy: fail-fast: false diff --git a/.github/workflows/py-cli-e2e-tests.yml b/.github/workflows/py-cli-e2e-tests.yml index 20572da9a070..51ed3263c153 100644 --- a/.github/workflows/py-cli-e2e-tests.yml +++ b/.github/workflows/py-cli-e2e-tests.yml @@ -65,7 +65,7 @@ jobs: - name: configure aws credentials if: contains('quicksight', matrix.e2e-test) || contains('datalake_s3', matrix.e2e-test) || contains('athena', matrix.e2e-test) - uses: aws-actions/configure-aws-credentials@v1 + uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ secrets.E2E_AWS_IAM_ROLE_ARN }} role-session-name: github-ci-aws-e2e-tests diff --git a/.github/workflows/py-tests.yml b/.github/workflows/py-tests.yml index 586489b012d8..7ca4a5f620d4 100644 --- a/.github/workflows/py-tests.yml +++ b/.github/workflows/py-tests.yml @@ -55,7 +55,7 @@ jobs: docker-images: false - name: Wait for the labeler - uses: lewagon/wait-on-check-action@v1.3.3 + uses: lewagon/wait-on-check-action@v1.3.4 if: ${{ github.event_name == 'pull_request_target' }} with: ref: ${{ github.event.pull_request.head.sha }} diff --git a/bootstrap/sql/migrations/native/1.5.15/mysql/postDataMigrationSQLScript.sql b/bootstrap/sql/migrations/native/1.5.15/mysql/postDataMigrationSQLScript.sql new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/bootstrap/sql/migrations/native/1.5.15/mysql/schemaChanges.sql b/bootstrap/sql/migrations/native/1.5.15/mysql/schemaChanges.sql new file mode 100644 index 000000000000..19762625a9d8 --- /dev/null +++ b/bootstrap/sql/migrations/native/1.5.15/mysql/schemaChanges.sql @@ -0,0 +1,5 @@ +-- Make domain policy and role non-system +UPDATE policy_entity SET json = JSON_SET(json, '$.provider', 'user') where name = 'DomainOnlyAccessPolicy'; +UPDATE policy_entity SET json = JSON_SET(json, '$.allowDelete', true) where name = 'DomainOnlyAccessPolicy'; +UPDATE role_entity SET json = JSON_SET(json, '$.provider', 'user') where name = 'DomainOnlyAccessRole'; +UPDATE role_entity SET json = JSON_SET(json, '$.allowDelete', true) where name = 'DomainOnlyAccessRole'; \ No newline at end of file diff --git a/bootstrap/sql/migrations/native/1.5.15/postgres/postDataMigrationSQLScript.sql b/bootstrap/sql/migrations/native/1.5.15/postgres/postDataMigrationSQLScript.sql new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/bootstrap/sql/migrations/native/1.5.15/postgres/schemaChanges.sql b/bootstrap/sql/migrations/native/1.5.15/postgres/schemaChanges.sql new file mode 100644 index 000000000000..6f92fbea754c --- /dev/null +++ b/bootstrap/sql/migrations/native/1.5.15/postgres/schemaChanges.sql @@ -0,0 +1,5 @@ +-- Make domain policy and role non-system +UPDATE policy_entity SET json = JSONB_SET(json::jsonb, '{provider}', '"user"', true) where name = 'DomainOnlyAccessPolicy'; +UPDATE policy_entity SET json = JSONB_SET(json::jsonb, '{allowDelete}', 'true', true) WHERE name = 'DomainOnlyAccessPolicy'; +UPDATE role_entity SET json = JSONB_SET(json::jsonb, '{provider}', '"user"', true) where name = 'DomainOnlyAccessRole'; +UPDATE role_entity SET json = JSONB_SET(json::jsonb, '{allowDelete}', 'true', true) WHERE name = 'DomainOnlyAccessRole'; diff --git a/bootstrap/sql/migrations/native/1.6.2/mysql/schemaChanges.sql b/bootstrap/sql/migrations/native/1.6.2/mysql/schemaChanges.sql new file mode 100644 index 000000000000..49ea32e96d49 --- /dev/null +++ b/bootstrap/sql/migrations/native/1.6.2/mysql/schemaChanges.sql @@ -0,0 +1,44 @@ +-- add timestamp index for test case result reindex performance +ALTER TABLE data_quality_data_time_series ADD INDEX `idx_timestamp_desc` (timestamp DESC); + +CREATE TABLE background_jobs ( + id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY, + jobType VARCHAR(256) NOT NULL, + methodName VARCHAR(256) NOT NULL, + jobArgs JSON NOT NULL, + status VARCHAR(50) NOT NULL DEFAULT 'PENDING', + createdBy VARCHAR(256) NOT NULL, + createdAt BIGINT UNSIGNED NOT NULL DEFAULT (UNIX_TIMESTAMP(NOW(3)) * 1000), + updatedAt BIGINT UNSIGNED NOT NULL DEFAULT (UNIX_TIMESTAMP(NOW(3)) * 1000) +); + +CREATE INDEX idx_status_createdAt ON background_jobs (status, createdAt); +CREATE INDEX idx_createdBy ON background_jobs (createdBy); +CREATE INDEX idx_status ON background_jobs (status); +CREATE INDEX idx_jobType ON background_jobs (jobType); +CREATE INDEX idx_updatedAt ON background_jobs (updatedAt); + +-- rename executable -> basic for test suites +UPDATE test_suite +SET json = JSON_INSERT( + JSON_REMOVE(json, '$.executable'), + '$.basic', + JSON_EXTRACT(json, '$.executable') +) +WHERE JSON_EXTRACT(json, '$.executable') IS NOT NULL; + +-- rename executableEntityReference -> basicEntityReference for test suites +UPDATE test_suite +SET json = JSON_INSERT( + JSON_REMOVE(json, '$.executableEntityReference'), + '$.basicEntityReference', + JSON_EXTRACT(json, '$.executableEntityReference') +) +WHERE JSON_EXTRACT(json, '$.executableEntityReference') IS NOT NULL; + +-- clean up the testSuites +UPDATE test_case SET json = json_remove(json, '$.testSuites'); + +-- clean up the testSuites in the version history too +UPDATE entity_extension SET json = json_remove(json, '$.testSuites') WHERE jsonSchema = 'testCase'; + diff --git a/bootstrap/sql/migrations/native/1.6.2/postgres/schemaChanges.sql b/bootstrap/sql/migrations/native/1.6.2/postgres/schemaChanges.sql new file mode 100644 index 000000000000..2b517db2ee7e --- /dev/null +++ b/bootstrap/sql/migrations/native/1.6.2/postgres/schemaChanges.sql @@ -0,0 +1,46 @@ +-- add timestamp index for test case result reindex performance +CREATE INDEX idx_timestamp_desc ON data_quality_data_time_series (timestamp DESC); + +CREATE TABLE background_jobs ( + id BIGSERIAL PRIMARY KEY, + jobType VARCHAR(256) NOT NULL, + methodName VARCHAR(256) NOT NULL, + jobArgs JSONB NOT NULL, + status VARCHAR(50) NOT NULL DEFAULT 'PENDING', + createdBy VARCHAR(256) NOT NULL, + createdAt BIGINT NOT NULL DEFAULT (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT, + updatedAt BIGINT NOT NULL DEFAULT (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT +); + +CREATE INDEX idx_status_createdAt ON background_jobs (status, createdAt); +CREATE INDEX idx_createdBy ON background_jobs (createdBy); +CREATE INDEX idx_status ON background_jobs (status); +CREATE INDEX idx_jobType ON background_jobs (jobType); +CREATE INDEX idx_updatedAt ON background_jobs (updatedAt); + +-- rename executable -> basic for test suites +UPDATE test_suite +SET json = jsonb_set( + json::jsonb #- '{executable}', + '{basic}', + (json #> '{executable}')::jsonb, + true +) +WHERE json #>> '{executable}' IS NOT NULL; + +-- rename executableEntityReference -> basicEntityReference for test suites +UPDATE test_suite +SET json = jsonb_set( + json::jsonb #- '{executableEntityReference}', + '{basicEntityReference}', + (json #> '{executableEntityReference}')::jsonb, + true +) +WHERE json #>> '{executableEntityReference}' IS NOT NULL; + +-- clean up the testSuites +UPDATE test_case SET json = json::jsonb #- '{testSuites}'; + +-- clean up the testSuites in the version history too +UPDATE entity_extension SET json = json::jsonb #- '{testSuites}' WHERE jsonSchema = 'testCase'; + diff --git a/bootstrap/sql/migrations/native/1.6.3/mysql/schemaChanges.sql b/bootstrap/sql/migrations/native/1.6.3/mysql/schemaChanges.sql new file mode 100644 index 000000000000..0c3231f17dad --- /dev/null +++ b/bootstrap/sql/migrations/native/1.6.3/mysql/schemaChanges.sql @@ -0,0 +1,6 @@ +-- Add constraint to apps_data_store +ALTER TABLE apps_data_store ADD CONSTRAINT entity_relationship_pky PRIMARY KEY (identifier, type); +UPDATE user_entity SET json = JSON_SET(json, '$.isBot', false) WHERE JSON_EXTRACT(json, '$.isBot') IS NULL; +ALTER TABLE user_entity ADD COLUMN isBot BOOLEAN GENERATED ALWAYS AS (json -> '$.isBot') NOT NULL; +CREATE INDEX idx_isBot ON user_entity (isBot); +DELETE from apps_data_store; \ No newline at end of file diff --git a/bootstrap/sql/migrations/native/1.6.3/postgres/schemaChanges.sql b/bootstrap/sql/migrations/native/1.6.3/postgres/schemaChanges.sql new file mode 100644 index 000000000000..cb734e7b0c6e --- /dev/null +++ b/bootstrap/sql/migrations/native/1.6.3/postgres/schemaChanges.sql @@ -0,0 +1,6 @@ +-- Add constraint to apps_data_store +ALTER TABLE apps_data_store ADD CONSTRAINT entity_relationship_pky PRIMARY KEY (identifier, type); +UPDATE user_entity SET json = jsonb_set(json::jsonb, '{isBot}', 'false'::jsonb, true) WHERE NOT (json ?? 'isBot'); +ALTER TABLE user_entity ADD COLUMN isBot BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED NOT NULL; +CREATE INDEX idx_isBot ON user_entity (isBot); +DELETE from apps_data_store; \ No newline at end of file diff --git a/common/pom.xml b/common/pom.xml index 65d7d15b61dd..d357ed0d6a1d 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -18,7 +18,7 @@ platform org.open-metadata - 1.6.0-SNAPSHOT + 1.6.4 4.0.0 diff --git a/common/src/main/java/org/openmetadata/annotations/DeprecatedAnnotator.java b/common/src/main/java/org/openmetadata/annotations/DeprecatedAnnotator.java new file mode 100644 index 000000000000..98e568274a37 --- /dev/null +++ b/common/src/main/java/org/openmetadata/annotations/DeprecatedAnnotator.java @@ -0,0 +1,76 @@ +package org.openmetadata.annotations; + +import com.fasterxml.jackson.databind.JsonNode; +import com.sun.codemodel.JAnnotationUse; +import com.sun.codemodel.JClass; +import com.sun.codemodel.JDefinedClass; +import com.sun.codemodel.JFieldVar; +import com.sun.codemodel.JMethod; +import java.lang.reflect.Field; +import java.util.TreeMap; +import org.jsonschema2pojo.AbstractAnnotator; + +/** Add {@link Deprecated} annotation to generated Java classes */ +public class DeprecatedAnnotator extends AbstractAnnotator { + + /** Add {@link Deprecated} annotation to property fields */ + @Override + public void propertyField( + JFieldVar field, JDefinedClass clazz, String propertyName, JsonNode propertyNode) { + super.propertyField(field, clazz, propertyName, propertyNode); + if (propertyNode.get("deprecated") != null && propertyNode.get("deprecated").asBoolean()) { + field.annotate(Deprecated.class); + } + } + + /** Add {@link Deprecated} annotation to getter methods */ + @Override + public void propertyGetter(JMethod getter, JDefinedClass clazz, String propertyName) { + super.propertyGetter(getter, clazz, propertyName); + addDeprecatedAnnotationIfApplies(getter, propertyName); + } + + /** Add {@link Deprecated} annotation to setter methods */ + @Override + public void propertySetter(JMethod setter, JDefinedClass clazz, String propertyName) { + super.propertySetter(setter, clazz, propertyName); + addDeprecatedAnnotationIfApplies(setter, propertyName); + } + + /** + * Use reflection methods to access the {@link JDefinedClass} of the {@link JMethod} object. If + * the {@link JMethod} is pointing to a field annotated with {@link Deprecated} then annotates + * the {@link JMethod} object with {@link Deprecated} + */ + private void addDeprecatedAnnotationIfApplies(JMethod jMethod, String propertyName) { + try { + Field outerClassField = JMethod.class.getDeclaredField("outer"); + outerClassField.setAccessible(true); + JDefinedClass outerClass = (JDefinedClass) outerClassField.get(jMethod); + + TreeMap insensitiveFieldsMap = + new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + insensitiveFieldsMap.putAll(outerClass.fields()); + + if (insensitiveFieldsMap.containsKey(propertyName) + && insensitiveFieldsMap.get(propertyName).annotations().stream() + .anyMatch( + annotation -> + Deprecated.class.getName().equals(getAnnotationClassName(annotation)))) { + jMethod.annotate(Deprecated.class); + } + } catch (NoSuchFieldException | IllegalAccessException e) { + throw new RuntimeException(e); + } + } + + private String getAnnotationClassName(JAnnotationUse annotation) { + try { + Field clazzField = JAnnotationUse.class.getDeclaredField("clazz"); + clazzField.setAccessible(true); + return ((JClass) clazzField.get(annotation)).fullName(); + } catch (NoSuchFieldException | IllegalAccessException e) { + throw new RuntimeException(e); + } + } +} diff --git a/common/src/main/java/org/openmetadata/annotations/OpenMetadataAnnotator.java b/common/src/main/java/org/openmetadata/annotations/OpenMetadataAnnotator.java index 15f0bcd948be..79209d8515d5 100644 --- a/common/src/main/java/org/openmetadata/annotations/OpenMetadataAnnotator.java +++ b/common/src/main/java/org/openmetadata/annotations/OpenMetadataAnnotator.java @@ -19,6 +19,10 @@ public class OpenMetadataAnnotator extends CompositeAnnotator { public OpenMetadataAnnotator() { // we can add multiple annotators - super(new ExposedAnnotator(), new MaskedAnnotator(), new PasswordAnnotator()); + super( + new ExposedAnnotator(), + new MaskedAnnotator(), + new PasswordAnnotator(), + new DeprecatedAnnotator()); } } diff --git a/conf/openmetadata.yaml b/conf/openmetadata.yaml index dedceb705dd9..cc6909053c8a 100644 --- a/conf/openmetadata.yaml +++ b/conf/openmetadata.yaml @@ -180,6 +180,7 @@ authenticationConfiguration: # This will only be valid when provider type specified is customOidc providerName: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""} publicKeyUrls: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]} + tokenValidationAlgorithm: ${AUTHENTICATION_TOKEN_VALIDATION_ALGORITHM:-"RS256"} authority: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com} clientId: ${AUTHENTICATION_CLIENT_ID:-""} callbackUrl: ${AUTHENTICATION_CALLBACK_URL:-""} @@ -295,6 +296,7 @@ eventMonitoringConfiguration: batchSize: ${EVENT_MONITOR_BATCH_SIZE:-10} pathPattern: ${EVENT_MONITOR_PATH_PATTERN:-["/api/v1/tables/*", "/api/v1/health-check"]} latency: ${EVENT_MONITOR_LATENCY:-[0.99, 0.90]} # For value p99=0.99, p90=0.90, p50=0.50 etc. + servicesHealthCheckInterval: ${EVENT_MONITOR_SERVICES_HEALTH_CHECK_INTERVAL:-300} # it will use the default auth provider for AWS services if parameters are not set # parameters: # region: ${OM_MONITOR_REGION:-""} diff --git a/docker/development/docker-compose.yml b/docker/development/docker-compose.yml index b5509137a521..ceaef3cececc 100644 --- a/docker/development/docker-compose.yml +++ b/docker/development/docker-compose.yml @@ -492,7 +492,7 @@ services: DB_HOST: ${AIRFLOW_DB_HOST:-mysql} DB_PORT: ${AIRFLOW_DB_PORT:-3306} AIRFLOW_DB: ${AIRFLOW_DB:-airflow_db} - DB_SCHEME: ${AIRFLOW_DB_SCHEME:-mysql+pymysql} + DB_SCHEME: ${AIRFLOW_DB_SCHEME:-mysql+mysqldb} DB_USER: ${AIRFLOW_DB_USER:-airflow_user} DB_PASSWORD: ${AIRFLOW_DB_PASSWORD:-airflow_pass} diff --git a/docker/docker-compose-ingestion/docker-compose-ingestion.yml b/docker/docker-compose-ingestion/docker-compose-ingestion.yml index b6adb2319e31..d20b7d6e11f5 100644 --- a/docker/docker-compose-ingestion/docker-compose-ingestion.yml +++ b/docker/docker-compose-ingestion/docker-compose-ingestion.yml @@ -18,7 +18,7 @@ volumes: services: ingestion: container_name: openmetadata_ingestion - image: docker.getcollate.io/openmetadata/ingestion:1.5.0-SNAPSHOT + image: docker.getcollate.io/openmetadata/ingestion:1.6.4 environment: AIRFLOW__API__AUTH_BACKENDS: "airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session" AIRFLOW__CORE__EXECUTOR: LocalExecutor @@ -26,7 +26,7 @@ services: DB_HOST: ${AIRFLOW_DB_HOST:-mysql} DB_PORT: ${AIRFLOW_DB_PORT:-3306} AIRFLOW_DB: ${AIRFLOW_DB:-airflow_db} - DB_SCHEME: ${AIRFLOW_DB_SCHEME:-mysql+pymysql} + DB_SCHEME: ${AIRFLOW_DB_SCHEME:-mysql+mysqldb} DB_USER: ${AIRFLOW_DB_USER:-airflow_user} DB_PASSWORD: ${AIRFLOW_DB_PASSWORD:-airflow_pass} # extra connection-string properties for the database diff --git a/docker/docker-compose-openmetadata/docker-compose-openmetadata.yml b/docker/docker-compose-openmetadata/docker-compose-openmetadata.yml index bfc903d62f73..c5ced415e891 100644 --- a/docker/docker-compose-openmetadata/docker-compose-openmetadata.yml +++ b/docker/docker-compose-openmetadata/docker-compose-openmetadata.yml @@ -14,7 +14,7 @@ services: execute-migrate-all: container_name: execute_migrate_all command: "./bootstrap/openmetadata-ops.sh migrate" - image: docker.getcollate.io/openmetadata/server:1.5.0-SNAPSHOT + image: docker.getcollate.io/openmetadata/server:1.6.4 environment: OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata} SERVER_PORT: ${SERVER_PORT:-8585} @@ -227,7 +227,7 @@ services: openmetadata-server: container_name: openmetadata_server restart: always - image: docker.getcollate.io/openmetadata/server:1.5.0-SNAPSHOT + image: docker.getcollate.io/openmetadata/server:1.6.4 environment: OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata} SERVER_PORT: ${SERVER_PORT:-8585} diff --git a/docker/docker-compose-quickstart/Dockerfile b/docker/docker-compose-quickstart/Dockerfile index 81c9750f2d9f..e756eb4fdd44 100644 --- a/docker/docker-compose-quickstart/Dockerfile +++ b/docker/docker-compose-quickstart/Dockerfile @@ -11,7 +11,7 @@ # Build stage FROM alpine:3 AS build -ARG RI_VERSION="1.6.0-SNAPSHOT" +ARG RI_VERSION="1.6.4" ENV RELEASE_URL="https://github.com/open-metadata/OpenMetadata/releases/download/${RI_VERSION}-release/openmetadata-${RI_VERSION}.tar.gz" RUN mkdir -p /opt/openmetadata && \ @@ -21,7 +21,7 @@ RUN mkdir -p /opt/openmetadata && \ # Final stage FROM alpine:3 -ARG RI_VERSION="1.6.0-SNAPSHOT" +ARG RI_VERSION="1.6.4" ARG BUILD_DATE ARG COMMIT_ID LABEL maintainer="OpenMetadata" diff --git a/docker/docker-compose-quickstart/docker-compose-postgres.yml b/docker/docker-compose-quickstart/docker-compose-postgres.yml index 4b279df6cb28..875e90dc4336 100644 --- a/docker/docker-compose-quickstart/docker-compose-postgres.yml +++ b/docker/docker-compose-quickstart/docker-compose-postgres.yml @@ -18,7 +18,7 @@ volumes: services: postgresql: container_name: openmetadata_postgresql - image: docker.getcollate.io/openmetadata/postgresql:1.5.0-SNAPSHOT + image: docker.getcollate.io/openmetadata/postgresql:1.6.4 restart: always command: "--work_mem=10MB" environment: @@ -61,7 +61,7 @@ services: execute-migrate-all: container_name: execute_migrate_all - image: docker.getcollate.io/openmetadata/server:1.5.0-SNAPSHOT + image: docker.getcollate.io/openmetadata/server:1.6.4 command: "./bootstrap/openmetadata-ops.sh migrate" environment: OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata} @@ -275,7 +275,7 @@ services: openmetadata-server: container_name: openmetadata_server restart: always - image: docker.getcollate.io/openmetadata/server:1.5.0-SNAPSHOT + image: docker.getcollate.io/openmetadata/server:1.6.4 environment: OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata} SERVER_PORT: ${SERVER_PORT:-8585} @@ -483,7 +483,7 @@ services: ingestion: container_name: openmetadata_ingestion - image: docker.getcollate.io/openmetadata/ingestion:1.5.0-SNAPSHOT + image: docker.getcollate.io/openmetadata/ingestion:1.6.4 depends_on: elasticsearch: condition: service_started diff --git a/docker/docker-compose-quickstart/docker-compose.yml b/docker/docker-compose-quickstart/docker-compose.yml index a66b57ddfb63..187e05dad997 100644 --- a/docker/docker-compose-quickstart/docker-compose.yml +++ b/docker/docker-compose-quickstart/docker-compose.yml @@ -18,7 +18,7 @@ volumes: services: mysql: container_name: openmetadata_mysql - image: docker.getcollate.io/openmetadata/db:1.5.0-SNAPSHOT + image: docker.getcollate.io/openmetadata/db:1.6.4 command: "--sort_buffer_size=10M" restart: always environment: @@ -59,7 +59,7 @@ services: execute-migrate-all: container_name: execute_migrate_all - image: docker.getcollate.io/openmetadata/server:1.5.0-SNAPSHOT + image: docker.getcollate.io/openmetadata/server:1.6.4 command: "./bootstrap/openmetadata-ops.sh migrate" environment: OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata} @@ -273,7 +273,7 @@ services: openmetadata-server: container_name: openmetadata_server restart: always - image: docker.getcollate.io/openmetadata/server:1.5.0-SNAPSHOT + image: docker.getcollate.io/openmetadata/server:1.6.4 environment: OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata} SERVER_PORT: ${SERVER_PORT:-8585} @@ -482,7 +482,7 @@ services: ingestion: container_name: openmetadata_ingestion - image: docker.getcollate.io/openmetadata/ingestion:1.5.0-SNAPSHOT + image: docker.getcollate.io/openmetadata/ingestion:1.6.4 depends_on: elasticsearch: condition: service_started @@ -497,7 +497,7 @@ services: DB_HOST: ${AIRFLOW_DB_HOST:-mysql} DB_PORT: ${AIRFLOW_DB_PORT:-3306} AIRFLOW_DB: ${AIRFLOW_DB:-airflow_db} - DB_SCHEME: ${AIRFLOW_DB_SCHEME:-mysql+pymysql} + DB_SCHEME: ${AIRFLOW_DB_SCHEME:-mysql+mysqldb} DB_USER: ${AIRFLOW_DB_USER:-airflow_user} DB_PASSWORD: ${AIRFLOW_DB_PASSWORD:-airflow_pass} # extra connection-string properties for the database diff --git a/ingestion/Dockerfile b/ingestion/Dockerfile index cc0df5ad259c..ef4b874f8b5b 100644 --- a/ingestion/Dockerfile +++ b/ingestion/Dockerfile @@ -1,6 +1,6 @@ FROM mysql:8.3 as mysql -FROM apache/airflow:2.9.1-python3.10 +FROM apache/airflow:2.9.3-python3.10 USER root RUN curl -sS https://packages.microsoft.com/keys/microsoft.asc | apt-key add - RUN curl -sS https://packages.microsoft.com/config/debian/11/prod.list > /etc/apt/sources.list.d/mssql-release.list @@ -76,9 +76,9 @@ ARG INGESTION_DEPENDENCY="all" ENV PIP_NO_CACHE_DIR=1 # Make pip silent ENV PIP_QUIET=1 -ARG RI_VERSION="1.6.0.0.dev0" +ARG RI_VERSION="1.6.4.0" RUN pip install --upgrade pip -RUN pip install "openmetadata-managed-apis~=${RI_VERSION}" --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.9.1/constraints-3.10.txt" +RUN pip install "openmetadata-managed-apis~=${RI_VERSION}" --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.9.3/constraints-3.10.txt" RUN pip install "openmetadata-ingestion[${INGESTION_DEPENDENCY}]~=${RI_VERSION}" # Temporary workaround for https://github.com/open-metadata/OpenMetadata/issues/9593 diff --git a/ingestion/Dockerfile.ci b/ingestion/Dockerfile.ci index f8433f776ca3..6e7c6e2a6406 100644 --- a/ingestion/Dockerfile.ci +++ b/ingestion/Dockerfile.ci @@ -1,6 +1,6 @@ FROM mysql:8.3 as mysql -FROM apache/airflow:2.9.1-python3.10 +FROM apache/airflow:2.9.3-python3.10 USER root RUN curl -sS https://packages.microsoft.com/keys/microsoft.asc | apt-key add - RUN curl -sS https://packages.microsoft.com/config/debian/11/prod.list > /etc/apt/sources.list.d/mssql-release.list @@ -73,7 +73,7 @@ COPY --chown=airflow:0 openmetadata-airflow-apis /home/airflow/openmetadata-airf COPY --chown=airflow:0 ingestion/examples/airflow/dags /opt/airflow/dags USER airflow -ARG AIRFLOW_CONSTRAINTS_LOCATION="https://raw.githubusercontent.com/apache/airflow/constraints-2.9.1/constraints-3.10.txt" +ARG AIRFLOW_CONSTRAINTS_LOCATION="https://raw.githubusercontent.com/apache/airflow/constraints-2.9.3/constraints-3.10.txt" # Disable pip cache dir # https://pip.pypa.io/en/stable/topics/caching/#avoiding-caching diff --git a/ingestion/ingestion_dependency.sh b/ingestion/ingestion_dependency.sh index ee54d6f6ac95..2b8372f852f4 100755 --- a/ingestion/ingestion_dependency.sh +++ b/ingestion/ingestion_dependency.sh @@ -15,7 +15,7 @@ DB_PORT=${DB_PORT:-3306} AIRFLOW_DB=${AIRFLOW_DB:-airflow_db} DB_USER=${DB_USER:-airflow_user} -DB_SCHEME=${DB_SCHEME:-mysql+pymysql} +DB_SCHEME=${DB_SCHEME:-mysql+mysqldb} DB_PASSWORD=${DB_PASSWORD:-airflow_pass} DB_PROPERTIES=${DB_PROPERTIES:-""} diff --git a/ingestion/operators/docker/Dockerfile b/ingestion/operators/docker/Dockerfile index d5d57088d256..f5a8d94bf048 100644 --- a/ingestion/operators/docker/Dockerfile +++ b/ingestion/operators/docker/Dockerfile @@ -41,6 +41,7 @@ RUN dpkg --configure -a \ && rm -rf /var/lib/apt/lists/* # Add updated postgres/redshift dependencies based on libq +ENV DEBIAN_FRONTEND=noninteractive RUN curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - RUN echo "deb https://apt.postgresql.org/pub/repos/apt/ bullseye-pgdg main" > /etc/apt/sources.list.d/pgdg.list; \ apt-get -qq update; \ @@ -81,7 +82,7 @@ ENV PIP_QUIET=1 RUN pip install --upgrade pip ARG INGESTION_DEPENDENCY="all" -ARG RI_VERSION="1.6.0.0.dev0" +ARG RI_VERSION="1.6.4.0" RUN pip install --upgrade pip RUN pip install "openmetadata-ingestion[airflow]~=${RI_VERSION}" RUN pip install "openmetadata-ingestion[${INGESTION_DEPENDENCY}]~=${RI_VERSION}" diff --git a/ingestion/operators/docker/Dockerfile.ci b/ingestion/operators/docker/Dockerfile.ci index 27752f954383..a12409b92956 100644 --- a/ingestion/operators/docker/Dockerfile.ci +++ b/ingestion/operators/docker/Dockerfile.ci @@ -41,6 +41,7 @@ RUN apt-get -qq update \ && rm -rf /var/lib/apt/lists/* # Add updated postgres/redshift dependencies based on libq +ENV DEBIAN_FRONTEND=noninteractive RUN curl -sS https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - RUN echo "deb https://apt.postgresql.org/pub/repos/apt/ bullseye-pgdg main" > /etc/apt/sources.list.d/pgdg.list; \ apt-get -qq update; \ diff --git a/ingestion/pyproject.toml b/ingestion/pyproject.toml index ec2b46ea578a..44d98481dc26 100644 --- a/ingestion/pyproject.toml +++ b/ingestion/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" # since it helps us organize and isolate version management [project] name = "openmetadata-ingestion" -version = "1.6.0.0.dev0" +version = "1.6.4.0" dynamic = ["readme", "dependencies", "optional-dependencies"] authors = [ { name = "OpenMetadata Committers" } diff --git a/ingestion/setup.py b/ingestion/setup.py index b117351d6a0f..b35f1f7e340b 100644 --- a/ingestion/setup.py +++ b/ingestion/setup.py @@ -19,7 +19,7 @@ # Add here versions required for multiple plugins VERSIONS = { - "airflow": "apache-airflow==2.9.1", + "airflow": "apache-airflow==2.9.3", "adlfs": "adlfs>=2023.1.0", "avro": "avro>=1.11.3,<1.12", "boto3": "boto3>=1.20,<2.0", # No need to add botocore separately. It's a dep from boto3 @@ -56,7 +56,13 @@ "elasticsearch8": "elasticsearch8~=8.9.0", "giturlparse": "giturlparse", "validators": "validators~=0.22.0", - "teradata": "teradatasqlalchemy>=20.0.0.0", + "teradata": "teradatasqlalchemy==20.0.0.2", + "cockroach": "sqlalchemy-cockroachdb~=2.0", + "cassandra": "cassandra-driver>=3.28.0", + "pydoris": "pydoris==1.0.2", + "pyiceberg": "pyiceberg==0.5.1", + "google-cloud-bigtable": "google-cloud-bigtable>=2.0.0", + "pyathena": "pyathena~=3.0", } COMMONS = { @@ -76,7 +82,7 @@ }, "kafka": { VERSIONS["avro"], - "confluent_kafka==2.1.1", + "confluent_kafka>=2.1.1,<=2.6.1", "fastavro>=1.2.0", # Due to https://github.com/grpc/grpc/issues/30843#issuecomment-1303816925 # use >= v1.47.2 https://github.com/grpc/grpc/blob/v1.47.2/tools/distrib/python/grpcio_tools/grpc_version.py#L17 @@ -96,7 +102,7 @@ DATA_DIFF = { driver: f"collate-data-diff[{driver}]" # data-diff uses different drivers out-of-the-box than OpenMetadata - # the exrtas are described here: + # the extras are described here: # https://github.com/open-metadata/collate-data-diff/blob/main/pyproject.toml#L68 # install all data diffs with "pip install collate-data-diff[all-dbs]" for driver in [ @@ -137,12 +143,17 @@ "requests>=2.23", "requests-aws4auth~=1.1", # Only depends on requests as external package. Leaving as base. "sqlalchemy>=1.4.0,<2", - "collate-sqllineage~=1.5.0", + "collate-sqllineage~=1.6.0", "tabulate==0.9.0", "typing-inspect", "packaging", # For version parsing + "setuptools~=70.0", "shapely", "collate-data-diff", + # TODO: Remove one once we have updated datadiff version + "snowflake-connector-python>=3.13.1,<4.0.0", + "mysql-connector-python>=8.0.29;python_version<'3.9'", + "mysql-connector-python>=9.1;python_version>='3.9'", } plugins: Dict[str, Set[str]] = { @@ -183,7 +194,7 @@ "google-cloud", VERSIONS["boto3"], VERSIONS["google-cloud-storage"], - "dbt-artifacts-parser", + "collate-dbt-artifacts-parser", VERSIONS["azure-storage-blob"], VERSIONS["azure-identity"], }, @@ -306,7 +317,7 @@ VERSIONS["geoalchemy2"], }, "sagemaker": {VERSIONS["boto3"]}, - "salesforce": {"simple_salesforce~=1.11"}, + "salesforce": {"simple_salesforce~=1.11", "authlib>=1.3.1"}, "sample-data": {VERSIONS["avro"], VERSIONS["grpc-tools"]}, "sap-hana": {"hdbcli", "sqlalchemy-hana"}, "sas": {}, @@ -356,7 +367,7 @@ "pytest-order", "dirty-equals", # install dbt dependency - "dbt-artifacts-parser", + "collate-dbt-artifacts-parser", "freezegun", VERSIONS["sqlalchemy-databricks"], VERSIONS["databricks-sdk"], diff --git a/ingestion/src/metadata/clients/aws_client.py b/ingestion/src/metadata/clients/aws_client.py index 8e87c600f96c..751f64da61dc 100644 --- a/ingestion/src/metadata/clients/aws_client.py +++ b/ingestion/src/metadata/clients/aws_client.py @@ -11,12 +11,16 @@ """ Module containing AWS Client """ +import datetime from enum import Enum -from typing import Any, Optional +from functools import partial +from typing import Any, Callable, Dict, Optional, Type, TypeVar import boto3 from boto3 import Session -from pydantic import BaseModel +from botocore.credentials import RefreshableCredentials +from botocore.session import get_session +from pydantic import BaseModel, Field from metadata.generated.schema.security.credentials.awsCredentials import AWSCredentials from metadata.ingestion.models.custom_pydantic import CustomSecretStr @@ -44,10 +48,28 @@ class AWSAssumeRoleException(Exception): """ +class AWSAssumeRoleCredentialResponse(BaseModel): + AccessKeyId: str = Field() + SecretAccessKey: str = Field() + SessionToken: Optional[str] = Field( + default=None, + ) + Expiration: Optional[datetime.datetime] = None + + class AWSAssumeRoleCredentialWrapper(BaseModel): - accessKeyId: str - secretAccessKey: CustomSecretStr - sessionToken: Optional[str] = None + accessKeyId: str = Field(alias="access_key") + secretAccessKey: CustomSecretStr = Field(alias="secret_key") + sessionToken: Optional[str] = Field(default=None, alias="token") + expiryTime: Optional[str] = Field(alias="expiry_time") + + class Config: + populate_by_name = True + + +AWSAssumeRoleCredentialFormat = TypeVar( + "AWSAssumeRoleCredentialFormat", AWSAssumeRoleCredentialWrapper, Dict +) class AWSClient: @@ -65,7 +87,10 @@ def __init__(self, config: "AWSCredentials"): @staticmethod def get_assume_role_config( config: AWSCredentials, - ) -> Optional[AWSAssumeRoleCredentialWrapper]: + return_type: Type[ + AWSAssumeRoleCredentialFormat + ] = AWSAssumeRoleCredentialWrapper, + ) -> Optional[AWSAssumeRoleCredentialFormat]: """ Get temporary credentials from assumed role """ @@ -90,12 +115,19 @@ def get_assume_role_config( ) if resp: - credentials = resp.get("Credentials", {}) - return AWSAssumeRoleCredentialWrapper( - accessKeyId=credentials.get("AccessKeyId"), - secretAccessKey=credentials.get("SecretAccessKey"), - sessionToken=credentials.get("SessionToken"), + credentials: AWSAssumeRoleCredentialResponse = ( + AWSAssumeRoleCredentialResponse(**resp.get("Credentials", {})) + ) + creds_wrapper = AWSAssumeRoleCredentialWrapper( + accessKeyId=credentials.AccessKeyId, + secretAccessKey=credentials.SecretAccessKey, + sessionToken=credentials.SessionToken, + expiryTime=credentials.Expiration.isoformat(), ) + if return_type == Dict: + return creds_wrapper.model_dump(by_alias=True) + return creds_wrapper + return None @staticmethod @@ -105,17 +137,32 @@ def _get_session( aws_session_token: Optional[str], aws_region: str, profile=None, + refresh_using: Optional[Callable] = None, ) -> Session: """ The only required param for boto3 is the region. The rest of credentials will have fallback strategies based on https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html#configuring-credentials """ + if refresh_using: + refreshable_creds = RefreshableCredentials.create_from_metadata( + metadata=refresh_using(), + refresh_using=refresh_using, + method="sts-assume-role", + ) + session = get_session() + session._credentials = refreshable_creds # pylint: disable=protected-access + return Session( + botocore_session=session, region_name=aws_region, profile_name=profile + ) + return Session( aws_access_key_id=aws_access_key_id, - aws_secret_access_key=aws_secret_access_key.get_secret_value() - if aws_secret_access_key - else None, + aws_secret_access_key=( + aws_secret_access_key.get_secret_value() + if aws_secret_access_key + else None + ), aws_session_token=aws_session_token, region_name=aws_region, profile_name=profile, @@ -123,15 +170,16 @@ def _get_session( def create_session(self) -> Session: if self.config.assumeRoleArn: - assume_creds = AWSClient.get_assume_role_config(self.config) - if assume_creds: - return AWSClient._get_session( - assume_creds.accessKeyId, - assume_creds.secretAccessKey, - assume_creds.sessionToken, - self.config.awsRegion, - self.config.profileName, - ) + return AWSClient._get_session( + None, + None, + None, + self.config.awsRegion, + self.config.profileName, + refresh_using=partial( + AWSClient.get_assume_role_config, self.config, Dict + ), + ) return AWSClient._get_session( self.config.awsAccessKeyId, diff --git a/ingestion/src/metadata/data_quality/api/models.py b/ingestion/src/metadata/data_quality/api/models.py index 82de6625cffb..39af90c06de6 100644 --- a/ingestion/src/metadata/data_quality/api/models.py +++ b/ingestion/src/metadata/data_quality/api/models.py @@ -23,6 +23,7 @@ from metadata.config.common import ConfigModel from metadata.generated.schema.api.tests.createTestSuite import CreateTestSuiteRequest from metadata.generated.schema.entity.data.table import Table +from metadata.generated.schema.entity.services.databaseService import DatabaseConnection from metadata.generated.schema.tests.basic import TestCaseResult from metadata.generated.schema.tests.testCase import TestCase, TestCaseParameterValue from metadata.ingestion.models.custom_pydantic import BaseModel @@ -63,6 +64,9 @@ class TableAndTests(BaseModel): executable_test_suite: Optional[CreateTestSuiteRequest] = Field( None, description="If no executable test suite is found, we'll create one" ) + service_connection: DatabaseConnection = Field( + ..., description="Service connection for the given table" + ) class TestCaseResults(BaseModel): diff --git a/ingestion/src/metadata/data_quality/processor/test_case_runner.py b/ingestion/src/metadata/data_quality/processor/test_case_runner.py index 696caaae0ce9..0e363eafb18d 100644 --- a/ingestion/src/metadata/data_quality/processor/test_case_runner.py +++ b/ingestion/src/metadata/data_quality/processor/test_case_runner.py @@ -27,6 +27,7 @@ from metadata.data_quality.runner.core import DataTestsRunner from metadata.generated.schema.api.tests.createTestCase import CreateTestCaseRequest from metadata.generated.schema.entity.data.table import Table +from metadata.generated.schema.entity.services.databaseService import DatabaseConnection from metadata.generated.schema.entity.services.ingestionPipelines.status import ( StackTraceError, ) @@ -93,7 +94,9 @@ def _run(self, record: TableAndTests) -> Either: record.table, openmetadata_test_cases ) - test_suite_runner = self.get_test_suite_runner(record.table) + test_suite_runner = self.get_test_suite_runner( + record.table, record.service_connection + ) logger.debug( f"Found {len(openmetadata_test_cases)} test cases for table {record.table.fullyQualifiedName.root}" @@ -351,9 +354,9 @@ def filter_incompatible_test_cases( result.append(tc) return result - def get_test_suite_runner(self, table: Table): + def get_test_suite_runner( + self, table: Table, service_connection: DatabaseConnection + ): return BaseTestSuiteRunner( - self.config, - self.metadata, - table, + self.config, self.metadata, table, service_connection ).get_data_quality_runner() diff --git a/ingestion/src/metadata/data_quality/runner/base_test_suite_source.py b/ingestion/src/metadata/data_quality/runner/base_test_suite_source.py index bf4897843a9f..7fbfcb29caba 100644 --- a/ingestion/src/metadata/data_quality/runner/base_test_suite_source.py +++ b/ingestion/src/metadata/data_quality/runner/base_test_suite_source.py @@ -46,11 +46,12 @@ def __init__( config: OpenMetadataWorkflowConfig, ometa_client: OpenMetadata, entity: Table, + service_connection: DatabaseConnection, ): self.validator_builder_class = ValidatorBuilder self._interface = None self.entity = entity - self.service_conn_config = self._copy_service_config(config, self.entity.database) # type: ignore + self.service_conn_config = self._copy_service_config(service_connection, self.entity.database) # type: ignore self._interface_type: str = self.service_conn_config.type.value.lower() self.source_config = TestSuitePipeline.model_validate( @@ -67,7 +68,7 @@ def interface(self, interface): self._interface = interface def _copy_service_config( - self, config: OpenMetadataWorkflowConfig, database: EntityReference + self, service_connection: DatabaseConnection, database: EntityReference ) -> DatabaseConnection: """Make a copy of the service config and update the database name @@ -77,9 +78,7 @@ def _copy_service_config( Returns: DatabaseService.__config__ """ - config_copy = deepcopy( - config.source.serviceConnection.root.config # type: ignore - ) + config_copy = deepcopy(service_connection.config) # type: ignore if hasattr( config_copy, # type: ignore "supportsDatabase", @@ -121,9 +120,9 @@ def create_data_quality_interface(self) -> TestSuiteInterface: schema_entity=schema_entity, database_entity=database_entity, default_sample_config=SampleConfig( - profile_sample=self.source_config.profileSample, - profile_sample_type=self.source_config.profileSampleType, - sampling_method_type=self.source_config.samplingMethodType, + profileSample=self.source_config.profileSample, + profileSampleType=self.source_config.profileSampleType, + samplingMethodType=self.source_config.samplingMethodType, ), ) diff --git a/ingestion/src/metadata/data_quality/source/test_suite.py b/ingestion/src/metadata/data_quality/source/test_suite.py index a2fb2cfae2ee..9558d36a1785 100644 --- a/ingestion/src/metadata/data_quality/source/test_suite.py +++ b/ingestion/src/metadata/data_quality/source/test_suite.py @@ -14,11 +14,17 @@ The main goal is to get the configured table from the API. """ -from typing import Iterable, List, Optional, cast +import itertools +import traceback +from typing import Dict, Iterable, List, Optional, cast from metadata.data_quality.api.models import TableAndTests from metadata.generated.schema.api.tests.createTestSuite import CreateTestSuiteRequest from metadata.generated.schema.entity.data.table import Table +from metadata.generated.schema.entity.services.databaseService import ( + DatabaseConnection, + DatabaseService, +) from metadata.generated.schema.entity.services.ingestionPipelines.status import ( StackTraceError, ) @@ -36,7 +42,7 @@ from metadata.ingestion.api.step import Step from metadata.ingestion.api.steps import Source from metadata.ingestion.ometa.ometa_api import OpenMetadata -from metadata.utils import fqn +from metadata.utils import entity_link, fqn from metadata.utils.constants import CUSTOM_CONNECTOR_PREFIX from metadata.utils.logger import test_suite_logger from metadata.utils.service_spec.service_spec import import_source_class @@ -61,18 +67,36 @@ def __init__( self.source_config: TestSuitePipeline = self.config.source.sourceConfig.config + # Build at runtime - if not informed in the yaml - the service connection map + self.service_connection_map: Dict[ + str, DatabaseConnection + ] = self._load_yaml_service_connections() + self.test_connection() @property def name(self) -> str: return "OpenMetadata" + def _load_yaml_service_connections(self) -> Dict[str, DatabaseConnection]: + """Load the service connections from the YAML file""" + service_connections = self.source_config.serviceConnections + if not service_connections: + return {} + return { + conn.serviceName: cast(DatabaseConnection, conn.serviceConnection.root) + for conn in service_connections + } + def _get_table_entity(self) -> Optional[Table]: """given an entity fqn return the table entity Args: entity_fqn: entity fqn for the test case """ + # Logical test suites don't have associated tables + if self.source_config.entityFullyQualifiedName is None: + return None table: Table = self.metadata.get_by_name( entity=Table, fqn=self.source_config.entityFullyQualifiedName.root, @@ -81,23 +105,51 @@ def _get_table_entity(self) -> Optional[Table]: return table - def _get_test_cases_from_test_suite( - self, test_suite: Optional[TestSuite] - ) -> List[TestCase]: + def _get_table_service_connection(self, table: Table) -> DatabaseConnection: + """Get the service connection for the table""" + service_name = table.service.name + + if service_name not in self.service_connection_map: + try: + service: DatabaseService = self.metadata.get_by_name( + DatabaseService, service_name + ) + if not service: + raise ConnectionError( + f"Could not retrieve service with name `{service_name}`. " + "Typically caused by the `entityFullyQualifiedName` does not exists in OpenMetadata " + "or the JWT Token is invalid." + ) + if not service.connection: + raise ConnectionError( + f"Service with name `{service_name}` does not have a connection. " + "If the connection is not stored in OpenMetadata, please provide it in the YAML file." + ) + self.service_connection_map[service_name] = service.connection + + except Exception as exc: + logger.debug(traceback.format_exc()) + logger.error( + f"Error getting service connection for service name [{service_name}]" + f" using the secrets manager provider [{self.metadata.config.secretsManagerProvider}]: {exc}" + ) + raise exc + + return self.service_connection_map[service_name] + + def _get_test_cases_from_test_suite(self, test_suite: TestSuite) -> List[TestCase]: """Return test cases if the test suite exists and has them""" - if test_suite: - test_cases = self.metadata.list_all_entities( - entity=TestCase, - fields=["testSuite", "entityLink", "testDefinition"], - params={"testSuiteId": test_suite.id.root}, - ) - test_cases = cast(List[TestCase], test_cases) # satisfy type checker - if self.source_config.testCases is not None: - test_cases = [ - t for t in test_cases if t.name in self.source_config.testCases - ] - return test_cases - return [] + test_cases = self.metadata.list_all_entities( + entity=TestCase, + fields=["testSuite", "entityLink", "testDefinition"], + params={"testSuiteId": test_suite.id.root}, + ) + test_cases = cast(List[TestCase], test_cases) # satisfy type checker + if self.source_config.testCases is not None: + test_cases = [ + t for t in test_cases if t.name in self.source_config.testCases + ] + return test_cases def prepare(self): """Nothing to prepare""" @@ -106,6 +158,7 @@ def test_connection(self) -> None: self.metadata.health_check() def _iter(self) -> Iterable[Either[TableAndTests]]: + # Basic tests suites will have a table informed table: Table = self._get_table_entity() if table: source_type = table.serviceType.value.lower() @@ -119,20 +172,29 @@ def _iter(self) -> Iterable[Either[TableAndTests]]: ) yield from self._process_table_suite(table) + # Logical test suites won't have a table, we'll need to group the execution by tests else: - yield Either( - left=StackTraceError( - name="Missing Table", - error=f"Could not retrieve table entity for {self.source_config.entityFullyQualifiedName.root}." - " Make sure the table exists in OpenMetadata and/or the JWT Token provided is valid.", - ) - ) + yield from self._process_logical_suite() def _process_table_suite(self, table: Table) -> Iterable[Either[TableAndTests]]: """ Check that the table has the proper test suite built in """ + try: + service_connection: DatabaseConnection = self._get_table_service_connection( + table + ) + except Exception as exc: + yield Either( + left=StackTraceError( + name="Error getting service connection", + error=f"Error getting the service connection for table {table.name.root}: {exc}", + stackTrace=traceback.format_exc(), + ) + ) + return # If there is no executable test suite yet for the table, we'll need to create one + # Then, the suite won't have yet any tests if not table.testSuite: executable_test_suite = CreateTestSuiteRequest( name=fqn.build( @@ -143,38 +205,80 @@ def _process_table_suite(self, table: Table) -> Iterable[Either[TableAndTests]]: displayName=f"{self.source_config.entityFullyQualifiedName.root} Test Suite", description="Test Suite created from YAML processor config file", owners=None, - executableEntityReference=self.source_config.entityFullyQualifiedName.root, + basicEntityReference=self.source_config.entityFullyQualifiedName.root, ) yield Either( right=TableAndTests( executable_test_suite=executable_test_suite, - service_type=self.config.source.serviceConnection.root.config.type.value, + service_type=service_connection.config.type.value, + service_connection=service_connection, ) ) + test_suite_cases = [] - test_suite: Optional[TestSuite] = None - if table.testSuite: - test_suite = self.metadata.get_by_id( + # Otherwise, we pick the tests already registered in the suite + else: + test_suite: Optional[TestSuite] = self.metadata.get_by_id( entity=TestSuite, entity_id=table.testSuite.id.root ) + test_suite_cases = self._get_test_cases_from_test_suite(test_suite) + + yield Either( + right=TableAndTests( + table=table, + test_cases=test_suite_cases, + service_type=service_connection.config.type.value, + service_connection=service_connection, + ) + ) - if test_suite and not test_suite.executable: + def _process_logical_suite(self): + """Process logical test suite, collect all test cases and yield them in batches by table""" + test_suite = self.metadata.get_by_name( + entity=TestSuite, fqn=self.config.source.serviceName + ) + if test_suite is None: yield Either( left=StackTraceError( - name="Non-executable Test Suite", - error=f"The table {self.source_config.entityFullyQualifiedName.root} " - "has a test suite that is not executable.", + name="Test Suite not found", + error=f"Test Suite with name {self.config.source.serviceName} not found", ) ) + test_cases: List[TestCase] = self._get_test_cases_from_test_suite(test_suite) + grouped_by_table = itertools.groupby( + test_cases, key=lambda t: entity_link.get_table_fqn(t.entityLink.root) + ) + for table_fqn, group in grouped_by_table: + table_entity: Table = self.metadata.get_by_name(Table, table_fqn) + if table_entity is None: + yield Either( + left=StackTraceError( + name="Table not found", + error=f"Table with fqn {table_fqn} not found for test suite {test_suite.name.root}", + ) + ) + continue - else: - test_suite_cases = self._get_test_cases_from_test_suite(test_suite) + try: + service_connection: DatabaseConnection = ( + self._get_table_service_connection(table_entity) + ) + except Exception as exc: + yield Either( + left=StackTraceError( + name="Error getting service connection", + error=f"Error getting the service connection for table {table_entity.name.root}: {exc}", + stackTrace=traceback.format_exc(), + ) + ) + continue yield Either( right=TableAndTests( - table=table, - test_cases=test_suite_cases, - service_type=self.config.source.serviceConnection.root.config.type.value, + table=table_entity, + test_cases=list(group), + service_type=service_connection.config.type.value, + service_connection=service_connection, ) ) diff --git a/ingestion/src/metadata/great_expectations/action.py b/ingestion/src/metadata/great_expectations/action.py index c51d146785c7..62fc07309a36 100644 --- a/ingestion/src/metadata/great_expectations/action.py +++ b/ingestion/src/metadata/great_expectations/action.py @@ -257,7 +257,7 @@ def _check_or_create_test_suite(self, table_entity: Table) -> TestSuite: create_test_suite = CreateTestSuiteRequest( name=f"{table_entity.fullyQualifiedName.root}.TestSuite", - executableEntityReference=table_entity.fullyQualifiedName.root, + basicEntityReference=table_entity.fullyQualifiedName.root, ) # type: ignore test_suite = self.ometa_conn.create_or_update_executable_test_suite( create_test_suite diff --git a/ingestion/src/metadata/ingestion/lineage/masker.py b/ingestion/src/metadata/ingestion/lineage/masker.py index 49a5999e72c6..e55783934052 100644 --- a/ingestion/src/metadata/ingestion/lineage/masker.py +++ b/ingestion/src/metadata/ingestion/lineage/masker.py @@ -14,8 +14,7 @@ import traceback -import sqlparse -from sqlfluff.core import Linter +from collate_sqllineage.runner import SQLPARSE_DIALECT, LineageRunner from sqlparse.sql import Comparison from sqlparse.tokens import Literal, Number, String @@ -24,6 +23,7 @@ MASK_TOKEN = "?" +# pylint: disable=protected-access def get_logger(): # pylint: disable=import-outside-toplevel from metadata.utils.logger import utils_logger @@ -31,18 +31,14 @@ def get_logger(): return utils_logger() -def mask_literals_with_sqlparse(query: str): +def mask_literals_with_sqlparse(query: str, parser: LineageRunner): """ Mask literals in a query using sqlparse. """ logger = get_logger() try: - parsed = sqlparse.parse(query) # Parse the query - - if not parsed: - return query - parsed = parsed[0] + parsed = parser._parsed_result def mask_token(token): # Mask all literals: strings, numbers, or other literal values @@ -79,17 +75,16 @@ def mask_token(token): return query -def mask_literals_with_sqlfluff(query: str, dialect: str = Dialect.ANSI.value) -> str: +def mask_literals_with_sqlfluff(query: str, parser: LineageRunner) -> str: """ Mask literals in a query using SQLFluff. """ logger = get_logger() try: - # Initialize SQLFluff linter - linter = Linter(dialect=dialect) + if not parser._evaluated: + parser._eval() - # Parse the query - parsed = linter.parse_string(query) + parsed = parser._parsed_result def replace_literals(segment): """Recursively replace literals with placeholders.""" @@ -114,18 +109,22 @@ def replace_literals(segment): return query -def mask_query(query: str, dialect: str = Dialect.ANSI.value) -> str: +def mask_query( + query: str, dialect: str = Dialect.ANSI.value, parser: LineageRunner = None +) -> str: logger = get_logger() try: - sqlfluff_masked_query = mask_literals_with_sqlfluff(query, dialect) - sqlparse_masked_query = mask_literals_with_sqlparse(query) - # compare both masked queries and return the one with more masked tokens - if sqlfluff_masked_query.count(MASK_TOKEN) >= sqlparse_masked_query.count( - MASK_TOKEN - ): - return sqlfluff_masked_query - return sqlparse_masked_query + if not parser: + try: + parser = LineageRunner(query, dialect=dialect) + len(parser.source_tables) + except Exception: + parser = LineageRunner(query) + len(parser.source_tables) + if parser._dialect == SQLPARSE_DIALECT: + return mask_literals_with_sqlparse(query, parser) + return mask_literals_with_sqlfluff(query, parser) except Exception as exc: logger.debug(f"Failed to mask query with sqlfluff: {exc}") logger.debug(traceback.format_exc()) - return query + return None diff --git a/ingestion/src/metadata/ingestion/lineage/models.py b/ingestion/src/metadata/ingestion/lineage/models.py index 6e0e4385854c..a3f50208c5f7 100644 --- a/ingestion/src/metadata/ingestion/lineage/models.py +++ b/ingestion/src/metadata/ingestion/lineage/models.py @@ -43,6 +43,9 @@ from metadata.generated.schema.entity.services.connections.database.impalaConnection import ( ImpalaType, ) +from metadata.generated.schema.entity.services.connections.database.mariaDBConnection import ( + MariaDBType, +) from metadata.generated.schema.entity.services.connections.database.mssqlConnection import ( MssqlType, ) @@ -58,6 +61,9 @@ from metadata.generated.schema.entity.services.connections.database.redshiftConnection import ( RedshiftType, ) +from metadata.generated.schema.entity.services.connections.database.singleStoreConnection import ( + SingleStoreType, +) from metadata.generated.schema.entity.services.connections.database.snowflakeConnection import ( SnowflakeType, ) @@ -120,6 +126,8 @@ class Dialect(Enum): str(MssqlType.Mssql.value): Dialect.TSQL, str(AzureSQLType.AzureSQL.value): Dialect.TSQL, str(TeradataType.Teradata.value): Dialect.TERADATA, + str(MariaDBType.MariaDB.value): Dialect.MYSQL, + str(SingleStoreType.SingleStore.value): Dialect.MYSQL, } diff --git a/ingestion/src/metadata/ingestion/lineage/parser.py b/ingestion/src/metadata/ingestion/lineage/parser.py index f7fad5fe812a..b9925fb1e35e 100644 --- a/ingestion/src/metadata/ingestion/lineage/parser.py +++ b/ingestion/src/metadata/ingestion/lineage/parser.py @@ -71,12 +71,13 @@ def __init__( self.query_parsing_success = True self.query_parsing_failure_reason = None self.dialect = dialect - self._masked_query = mask_query(self.query, dialect.value) + self.masked_query = None self._clean_query = self.clean_raw_query(query) - self._masked_clean_query = mask_query(self._clean_query, dialect.value) self.parser = self._evaluate_best_parser( self._clean_query, dialect=dialect, timeout_seconds=timeout_seconds ) + if self.masked_query is None: + self.masked_query = mask_query(self._clean_query, parser=self.parser) @cached_property def involved_tables(self) -> Optional[List[Table]]: @@ -95,7 +96,7 @@ def involved_tables(self) -> Optional[List[Table]]: except SQLLineageException as exc: logger.debug(traceback.format_exc()) logger.warning( - f"Cannot extract source table information from query [{self._masked_query}]: {exc}" + f"Cannot extract source table information from query [{self.masked_query or self.query}]: {exc}" ) return None @@ -334,12 +335,10 @@ def stateful_add_joins_from_statement( ) if not table_left or not table_right: - logger.warning( - f"Can't extract table names when parsing JOIN information from {comparison}" - ) logger.debug( - f"Query: {mask_query(sql_statement, self.dialect.value)}" + f"Can't extract table names when parsing JOIN information from {comparison}" ) + logger.debug(f"Query: {self.masked_query or self.query}") continue left_table_column = TableColumn(table=table_left, column=column_left) @@ -422,10 +421,9 @@ def get_sqlfluff_lineage_runner(qry: str, dlct: str) -> LineageRunner: lr_dialect.get_column_lineage() return lr_dialect - sqlfluff_count = 0 try: lr_sqlfluff = get_sqlfluff_lineage_runner(query, dialect.value) - sqlfluff_count = len(lr_sqlfluff.get_column_lineage()) + len( + _ = len(lr_sqlfluff.get_column_lineage()) + len( set(lr_sqlfluff.source_tables).union( set(lr_sqlfluff.target_tables).union( set(lr_sqlfluff.intermediate_tables) @@ -438,23 +436,20 @@ def get_sqlfluff_lineage_runner(qry: str, dlct: str) -> LineageRunner: f"Lineage with SqlFluff failed for the [{dialect.value}]. " f"Parser has been running for more than {timeout_seconds} seconds." ) - logger.debug( - f"{self.query_parsing_failure_reason}] query: [{self._masked_clean_query}]" - ) lr_sqlfluff = None except Exception: self.query_parsing_success = False self.query_parsing_failure_reason = ( f"Lineage with SqlFluff failed for the [{dialect.value}]" ) - logger.debug( - f"{self.query_parsing_failure_reason} query: [{self._masked_clean_query}]" - ) lr_sqlfluff = None + if lr_sqlfluff: + return lr_sqlfluff + lr_sqlparser = LineageRunner(query) try: - sqlparser_count = len(lr_sqlparser.get_column_lineage()) + len( + _ = len(lr_sqlparser.get_column_lineage()) + len( set(lr_sqlparser.source_tables).union( set(lr_sqlparser.target_tables).union( set(lr_sqlparser.intermediate_tables) @@ -463,21 +458,13 @@ def get_sqlfluff_lineage_runner(qry: str, dlct: str) -> LineageRunner: ) except Exception: # if both runner have failed we return the usual one + logger.debug(f"Failed to parse query with sqlparse & sqlfluff: {query}") return lr_sqlfluff if lr_sqlfluff else lr_sqlparser - if lr_sqlfluff: - # if sqlparser retrieve more lineage info that sqlfluff - if sqlparser_count > sqlfluff_count: - self.query_parsing_success = False - self.query_parsing_failure_reason = ( - "Lineage computed with SqlFluff did not perform as expected " - f"for the [{dialect.value}]" - ) - logger.debug( - f"{self.query_parsing_failure_reason} query: [{self._masked_clean_query}]" - ) - return lr_sqlparser - return lr_sqlfluff + self.masked_query = mask_query(self._clean_query, parser=lr_sqlparser) + logger.debug( + f"Using sqlparse for lineage parsing for query: {self.masked_query or self.query}" + ) return lr_sqlparser @staticmethod diff --git a/ingestion/src/metadata/ingestion/lineage/sql_lineage.py b/ingestion/src/metadata/ingestion/lineage/sql_lineage.py index c8954fad6f0f..577e6a87498a 100644 --- a/ingestion/src/metadata/ingestion/lineage/sql_lineage.py +++ b/ingestion/src/metadata/ingestion/lineage/sql_lineage.py @@ -16,8 +16,10 @@ from collections import defaultdict from typing import Any, Iterable, List, Optional, Tuple, Union +import networkx as nx from collate_sqllineage.core.models import Column, DataFunction from collate_sqllineage.core.models import Table as LineageTable +from networkx import DiGraph from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest from metadata.generated.schema.entity.data.storedProcedure import ( @@ -37,7 +39,6 @@ from metadata.generated.schema.type.entityLineage import Source as LineageSource from metadata.generated.schema.type.entityReference import EntityReference from metadata.ingestion.api.models import Either -from metadata.ingestion.lineage.masker import mask_query from metadata.ingestion.lineage.models import ( Dialect, QueryParsingError, @@ -510,6 +511,7 @@ def _create_lineage_by_table_name( column_lineage_map: dict, lineage_source: LineageSource = LineageSource.QueryLineage, procedure: Optional[EntityReference] = None, + graph: DiGraph = None, ) -> Iterable[Either[AddLineageRequest]]: """ This method is to create a lineage between two tables @@ -540,6 +542,11 @@ def _create_lineage_by_table_name( logger.debug( f"WARNING: Table entity [{table_name}] not found in OpenMetadata" ) + if graph is not None and (not from_table_entities or not to_table_entities): + graph.add_node(from_table, entity=from_table_entities) + graph.add_node(to_table, entity=to_table_entities) + graph.add_edge(from_table, to_table, query=masked_query) + return for from_entity, to_entity in itertools.product( from_table_entities or [], to_table_entities or [] @@ -607,6 +614,7 @@ def get_lineage_by_query( dialect: Dialect, timeout_seconds: int = LINEAGE_PARSING_TIMEOUT, lineage_source: LineageSource = LineageSource.QueryLineage, + graph: DiGraph = None, ) -> Iterable[Either[AddLineageRequest]]: """ This method parses the query to get source, target and intermediate table names to create lineage, @@ -614,11 +622,11 @@ def get_lineage_by_query( """ column_lineage = {} query_parsing_failures = QueryParsingFailures() - masked_query = mask_query(query, dialect.value) try: - logger.debug(f"Running lineage with query: {masked_query}") lineage_parser = LineageParser(query, dialect, timeout_seconds=timeout_seconds) + masked_query = lineage_parser.masked_query + logger.debug(f"Running lineage with query: {masked_query or query}") raw_column_lineage = lineage_parser.column_lineage column_lineage.update(populate_column_lineage_map(raw_column_lineage)) @@ -646,6 +654,7 @@ def get_lineage_by_query( column_lineage_map=column_lineage, lineage_source=lineage_source, procedure=procedure, + graph=graph, ) for target_table in lineage_parser.target_tables: yield from _create_lineage_by_table_name( @@ -683,11 +692,12 @@ def get_lineage_by_query( column_lineage_map=column_lineage, lineage_source=lineage_source, procedure=procedure, + graph=graph, ) if not lineage_parser.query_parsing_success: query_parsing_failures.add( QueryParsingError( - query=masked_query, + query=masked_query or query, error=lineage_parser.query_parsing_failure_reason, ) ) @@ -711,15 +721,18 @@ def get_lineage_via_table_entity( dialect: Dialect, timeout_seconds: int = LINEAGE_PARSING_TIMEOUT, lineage_source: LineageSource = LineageSource.QueryLineage, + graph: DiGraph = None, ) -> Iterable[Either[AddLineageRequest]]: """Get lineage from table entity""" column_lineage = {} query_parsing_failures = QueryParsingFailures() - masked_query = mask_query(query, dialect.value) try: - logger.debug(f"Getting lineage via table entity using query: {masked_query}") lineage_parser = LineageParser(query, dialect, timeout_seconds=timeout_seconds) + masked_query = lineage_parser.masked_query + logger.debug( + f"Getting lineage via table entity using query: {masked_query or query}" + ) to_table_name = table_entity.name.root for from_table_name in lineage_parser.source_tables: @@ -744,6 +757,7 @@ def get_lineage_via_table_entity( column_lineage_map=column_lineage, lineage_source=lineage_source, procedure=procedure, + graph=graph, ) or [] if not lineage_parser.query_parsing_success: query_parsing_failures.add( @@ -760,3 +774,94 @@ def get_lineage_via_table_entity( stackTrace=traceback.format_exc(), ) ) + + +def _process_sequence( + sequence: List[Any], graph: DiGraph +) -> Iterable[Either[AddLineageRequest]]: + """ + Process a sequence of nodes to generate lineage information. + """ + from_node = None + queries = set() + clean_queries = False + previous_node = None + for node in sequence: + try: + if clean_queries: + queries.clear() + clean_queries = False + current_node = graph.nodes[node] + current_entity = current_node.get("entity") + + if ( + previous_node is not None + and graph.edges[(previous_node, node)].get("query") is not None + ): + queries.add(graph.edges[(previous_node, node)].get("query")) + + if current_entity and from_node is not None: + for from_entity, to_entity in itertools.product( + from_node.get("entity") or [], current_entity or [] + ): + if to_entity and from_entity: + yield _build_table_lineage( + to_entity=to_entity, + from_entity=from_entity, + to_table_raw_name=str(node), + from_table_raw_name=str(from_node), + masked_query="\n--------\n".join(queries), + column_lineage_map={}, + lineage_source=LineageSource.QueryLineage, + procedure=None, + ) + clean_queries = True + + if current_entity: + from_node = graph.nodes[node] + previous_node = node + except Exception as exc: + logger.debug(traceback.format_exc()) + logger.error(f"Error creating lineage for node [{node}]: {exc}") + + +def get_lineage_by_graph( + graph: DiGraph, +) -> Iterable[Either[AddLineageRequest]]: + """ + Generate lineage information from a directed graph. + This method processes a directed graph to extract lineage information by identifying + weakly connected components and traversing each component to generate sequences of nodes. + It then yields lineage information for each sequence. + Args: + graph (DiGraph): A directed graph representing the lineage. + Raises: + Exception: If an error occurs during the lineage creation process, it logs the error. + """ + if graph is None: + return + + # Get all weakly connected components + components = list(nx.weakly_connected_components(graph)) + + # Extract each component as an independent subgraph + independent_subtrees = [ + graph.subgraph(component).copy() for component in components + ] + + # Print results in the desired format + for subtree in independent_subtrees: + # Find a root node (node with no incoming edges) + root = [node for node in subtree if subtree.in_degree(node) == 0][0] + + # Traverse from the root to get the sequence of nodes + current = root + sequence = [current] + while True: + successors = list(subtree.successors(current)) + if not successors: + break + current = successors[0] + sequence.append(current) + + yield from _process_sequence(sequence, subtree) diff --git a/ingestion/src/metadata/ingestion/ometa/mixins/es_mixin.py b/ingestion/src/metadata/ingestion/ometa/mixins/es_mixin.py index 27d4e6cef9e5..fbd84b4e92e1 100644 --- a/ingestion/src/metadata/ingestion/ometa/mixins/es_mixin.py +++ b/ingestion/src/metadata/ingestion/ometa/mixins/es_mixin.py @@ -344,7 +344,7 @@ def _paginate_es_internal( # Get next page last_hit = response.hits.hits[-1] if response.hits.hits else None if not last_hit or not last_hit.sort: - logger.info("No more pages to fetch") + logger.debug("No more pages to fetch") break after = ",".join(last_hit.sort) @@ -402,10 +402,51 @@ def yield_es_view_def( "query": { "bool": { "must": [ - {"term": {"service.name.keyword": service_name}}, - {"term": {"tableType": TableType.View.value}}, - {"term": {"deleted": False}}, - {"exists": {"field": "schemaDefinition"}}, + { + "bool": { + "should": [ + {"term": {"service.name.keyword": service_name}} + ] + } + }, + { + "bool": { + "must": [ + { + "bool": { + "should": [ + { + "term": { + "tableType": TableType.View.value + } + }, + { + "term": { + "tableType": TableType.MaterializedView.value + } + }, + { + "term": { + "tableType": TableType.SecureView.value + } + }, + { + "term": { + "tableType": TableType.Dynamic.value + } + }, + ] + } + } + ] + } + }, + {"bool": {"should": [{"term": {"deleted": False}}]}}, + { + "bool": { + "should": [{"exists": {"field": "schemaDefinition"}}] + } + }, ] } } @@ -420,10 +461,11 @@ def yield_es_view_def( _, database_name, schema_name, table_name = fqn.split( hit.source["fullyQualifiedName"] ) - yield TableView( - view_definition=hit.source["schemaDefinition"], - service_name=service_name, - db_name=database_name, - schema_name=schema_name, - table_name=table_name, - ) + if hit.source.get("schemaDefinition"): + yield TableView( + view_definition=hit.source["schemaDefinition"], + service_name=service_name, + db_name=database_name, + schema_name=schema_name, + table_name=table_name, + ) diff --git a/ingestion/src/metadata/ingestion/ometa/mixins/lineage_mixin.py b/ingestion/src/metadata/ingestion/ometa/mixins/lineage_mixin.py index 45769f5935c4..5adc2c18296f 100644 --- a/ingestion/src/metadata/ingestion/ometa/mixins/lineage_mixin.py +++ b/ingestion/src/metadata/ingestion/ometa/mixins/lineage_mixin.py @@ -63,7 +63,7 @@ def _merge_column_lineage( ) for column in updated or []: if not isinstance(column, dict): - data = column.dict() + data = column.model_dump() else: data = column if data.get("toColumn") and data.get("fromColumns"): diff --git a/ingestion/src/metadata/ingestion/ometa/mixins/query_mixin.py b/ingestion/src/metadata/ingestion/ometa/mixins/query_mixin.py index be2de4fbf4a0..c0b217a804e8 100644 --- a/ingestion/src/metadata/ingestion/ometa/mixins/query_mixin.py +++ b/ingestion/src/metadata/ingestion/ometa/mixins/query_mixin.py @@ -42,6 +42,8 @@ def _get_query_hash(self, query: str) -> str: return str(result.hexdigest()) def _get_or_create_query(self, query: CreateQueryRequest) -> Optional[Query]: + if query.query.root is None: + return None query_hash = self._get_query_hash(query=query.query.root) query_entity = self.get_by_name(entity=Query, fqn=query_hash) if query_entity is None: diff --git a/ingestion/src/metadata/ingestion/ometa/mixins/tests_mixin.py b/ingestion/src/metadata/ingestion/ometa/mixins/tests_mixin.py index c76ddc896318..d4eb2fd1b742 100644 --- a/ingestion/src/metadata/ingestion/ometa/mixins/tests_mixin.py +++ b/ingestion/src/metadata/ingestion/ometa/mixins/tests_mixin.py @@ -228,7 +228,7 @@ def get_or_create_executable_test_suite( create_test_suite = CreateTestSuiteRequest( name=f"{table_entity.fullyQualifiedName.root}.TestSuite", - executableEntityReference=table_entity.fullyQualifiedName.root, + basicEntityReference=table_entity.fullyQualifiedName.root, ) # type: ignore test_suite = self.create_or_update_executable_test_suite(create_test_suite) return test_suite diff --git a/ingestion/src/metadata/ingestion/ometa/mixins/user_mixin.py b/ingestion/src/metadata/ingestion/ometa/mixins/user_mixin.py index 2ee604973f1a..c389fbc2842a 100644 --- a/ingestion/src/metadata/ingestion/ometa/mixins/user_mixin.py +++ b/ingestion/src/metadata/ingestion/ometa/mixins/user_mixin.py @@ -13,6 +13,7 @@ To be used by OpenMetadata class """ +import json from functools import lru_cache from typing import Optional, Type @@ -46,16 +47,29 @@ def email_search_query_es(entity: Type[T]) -> str: ) @staticmethod - def name_search_query_es(entity: Type[T]) -> str: + def name_search_query_es(entity: Type[T], name: str, from_: int, size: int) -> str: """ Allow for more flexible lookup following what the UI is doing when searching users. We don't want to stick to `q=name:{name}` since in case a user is named `random.user` but looked as `Random User`, we want to find this match. + + Search should only look in name and displayName fields and should not return bots. """ + query_filter = { + "query": { + "query_string": { + "query": f"{name} AND isBot:false", + "fields": ["name", "displayName"], + "default_operator": "AND", + "fuzziness": "AUTO", + } + } + } + return ( - "/search/query?q={name} AND isBot:false&from={from_}&size={size}&index=" - + ES_INDEX_MAP[entity.__name__] + f"""/search/query?query_filter={json.dumps(query_filter)}""" + f"&from={from_}&size={size}&index=" + ES_INDEX_MAP[entity.__name__] ) def _search_by_email( @@ -103,8 +117,8 @@ def _search_by_name( fields: Optional field list to pass to ES request """ if name: - query_string = self.name_search_query_es(entity=entity).format( - name=name, from_=from_count, size=size + query_string = self.name_search_query_es( + entity=entity, name=name, from_=from_count, size=size ) return self.get_entity_from_es( entity=entity, query_string=query_string, fields=fields diff --git a/ingestion/src/metadata/ingestion/ometa/routes.py b/ingestion/src/metadata/ingestion/ometa/routes.py index 2750991a0537..b8538923110a 100644 --- a/ingestion/src/metadata/ingestion/ometa/routes.py +++ b/ingestion/src/metadata/ingestion/ometa/routes.py @@ -54,6 +54,7 @@ ) from metadata.generated.schema.api.data.createTable import CreateTableRequest from metadata.generated.schema.api.data.createTopic import CreateTopicRequest +from metadata.generated.schema.api.docStore.createDocument import CreateDocumentRequest from metadata.generated.schema.api.domains.createDataProduct import ( CreateDataProductRequest, ) @@ -91,6 +92,7 @@ from metadata.generated.schema.api.services.ingestionPipelines.createIngestionPipeline import ( CreateIngestionPipelineRequest, ) +from metadata.generated.schema.api.teams.createPersona import CreatePersonaRequest from metadata.generated.schema.api.teams.createRole import CreateRoleRequest from metadata.generated.schema.api.teams.createTeam import CreateTeamRequest from metadata.generated.schema.api.teams.createUser import CreateUserRequest @@ -136,6 +138,7 @@ from metadata.generated.schema.entity.data.storedProcedure import StoredProcedure from metadata.generated.schema.entity.data.table import Table from metadata.generated.schema.entity.data.topic import Topic +from metadata.generated.schema.entity.docStore.document import Document from metadata.generated.schema.entity.domains.dataProduct import DataProduct from metadata.generated.schema.entity.domains.domain import Domain from metadata.generated.schema.entity.feed.suggestion import Suggestion @@ -155,6 +158,7 @@ from metadata.generated.schema.entity.services.pipelineService import PipelineService from metadata.generated.schema.entity.services.searchService import SearchService from metadata.generated.schema.entity.services.storageService import StorageService +from metadata.generated.schema.entity.teams.persona import Persona from metadata.generated.schema.entity.teams.role import Role from metadata.generated.schema.entity.teams.team import Team from metadata.generated.schema.entity.teams.user import AuthenticationMechanism, User @@ -198,6 +202,8 @@ CreateAPIEndpointRequest.__name__: "/apiEndpoints", APICollection.__name__: "/apiCollections", CreateAPICollectionRequest.__name__: "/apiCollections", + Document.__name__: "/docStore", + CreateDocumentRequest.__name__: "/docStore", # Classifications Tag.__name__: "/tags", CreateTagRequest.__name__: "/tags", @@ -213,6 +219,8 @@ CreateTeamRequest.__name__: "/teams", User.__name__: "/users", CreateUserRequest.__name__: "/users", + Persona.__name__: "/personas", + CreatePersonaRequest.__name__: "/personas", AuthenticationMechanism.__name__: "/users/auth-mechanism", Bot.__name__: "/bots", CreateBot.__name__: "/bots", diff --git a/ingestion/src/metadata/ingestion/source/api/rest/metadata.py b/ingestion/src/metadata/ingestion/source/api/rest/metadata.py index 925c0f1e943f..bf59b1db6767 100644 --- a/ingestion/src/metadata/ingestion/source/api/rest/metadata.py +++ b/ingestion/src/metadata/ingestion/source/api/rest/metadata.py @@ -171,7 +171,7 @@ def _filter_collection_endpoints( break return filtered_paths except Exception as err: - logger.info( + logger.warning( f"Error while filtering endpoints for collection {collection.name.root}" ) return None @@ -184,7 +184,7 @@ def _prepare_endpoint_data(self, path, method_type, info) -> Optional[RESTEndpoi endpoint.name = f"{path} - {method_type}" return endpoint except Exception as err: - logger.info(f"Error while parsing endpoint data: {err}") + logger.warning(f"Error while parsing endpoint data: {err}") return None def _generate_collection_url(self, collection_name: str) -> Optional[AnyUrl]: @@ -195,7 +195,7 @@ def _generate_collection_url(self, collection_name: str) -> Optional[AnyUrl]: f"{self.config.serviceConnection.root.config.openAPISchemaURL}#tag/{collection_name}" ) except Exception as err: - logger.info(f"Error while generating collection url: {err}") + logger.warning(f"Error while generating collection url: {err}") return None def _generate_endpoint_url(self, endpoint_name: str) -> AnyUrl: @@ -203,15 +203,14 @@ def _generate_endpoint_url(self, endpoint_name: str) -> AnyUrl: base_url = self.config.serviceConnection.root.config.openAPISchemaURL if endpoint_name: return AnyUrl(f"{base_url}#operation/{endpoint_name}") - else: - return AnyUrl(base_url) + return AnyUrl(base_url) def _get_api_request_method(self, method_type: str) -> Optional[str]: """fetch endpoint request method""" try: return ApiRequestMethod[method_type.upper()] except KeyError as err: - logger.info(f"Keyerror while fetching request method: {err}") + logger.warning(f"Keyerror while fetching request method: {err}") return None def _get_request_schema(self, info: dict) -> Optional[APISchema]: @@ -227,9 +226,9 @@ def _get_request_schema(self, info: dict) -> Optional[APISchema]: if not schema_ref: logger.debug("No request schema found for the endpoint") return None - return self._process_schema(schema_ref) + return APISchema(schemaFields=self.process_schema_fields(schema_ref)) except Exception as err: - logger.info(f"Error while parsing request schema: {err}") + logger.warning(f"Error while parsing request schema: {err}") return None def _get_response_schema(self, info: dict) -> Optional[APISchema]: @@ -246,33 +245,43 @@ def _get_response_schema(self, info: dict) -> Optional[APISchema]: if not schema_ref: logger.debug("No response schema found for the endpoint") return None - return self._process_schema(schema_ref) + return APISchema(schemaFields=self.process_schema_fields(schema_ref)) except Exception as err: - logger.info(f"Error while parsing response schema: {err}") + logger.warning(f"Error while parsing response schema: {err}") return None - def _process_schema(self, schema_ref: str) -> Optional[List[APISchema]]: - """process schema""" + def process_schema_fields(self, schema_ref: str) -> Optional[List[FieldModel]]: try: - schema_ref = schema_ref.split("/")[-1] + schema_name = schema_ref.split("/")[-1] schema_fields = ( - self.json_response.get("components").get("schemas").get(schema_ref) + self.json_response.get("components").get("schemas").get(schema_name) ) fetched_fields = [] for key, val in schema_fields.get("properties", {}).items(): dtype = val.get("type") - if not dtype: - continue - fetched_fields.append( - FieldModel( - name=key, - dataType=DataTypeTopic[dtype.upper()] + if dtype: + parsed_dtype = ( + DataTypeTopic[dtype.upper()] if dtype.upper() in DataTypeTopic.__members__ - else DataTypeTopic.UNKNOWN, + else DataTypeTopic.UNKNOWN ) - ) - return APISchema(schemaFields=fetched_fields) + fetched_fields.append(FieldModel(name=key, dataType=parsed_dtype)) + else: + # If type of field is not defined then check for sub-schema + # Check if it's `object` type field + # check infinite recrusrion by comparing with parent(schema_ref) + object_children = None + if val.get("$ref") and val.get("$ref") != schema_ref: + object_children = self.process_schema_fields(val.get("$ref")) + fetched_fields.append( + FieldModel( + name=key, + dataType=DataTypeTopic.UNKNOWN, + children=object_children, + ) + ) + return fetched_fields except Exception as err: - logger.info(f"Error while processing request schema: {err}") + logger.warning(f"Error while processing schema fields: {err}") return None diff --git a/ingestion/src/metadata/ingestion/source/dashboard/looker/metadata.py b/ingestion/src/metadata/ingestion/source/dashboard/looker/metadata.py index 4cf1858227f9..fafc59429bb5 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/looker/metadata.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/looker/metadata.py @@ -21,13 +21,26 @@ """ import copy import os +import re import traceback from datetime import datetime from pathlib import Path -from typing import Dict, Iterable, List, Optional, Sequence, Set, Type, Union, cast +from typing import ( + Dict, + Iterable, + List, + Optional, + Sequence, + Set, + Type, + Union, + cast, + get_args, +) import giturlparse import lkml +import networkx as nx from liquid import Template from looker_sdk.sdk.api40.methods import Looker40SDK from looker_sdk.sdk.api40.models import Dashboard as LookerDashboard @@ -91,7 +104,7 @@ from metadata.generated.schema.type.usageRequest import UsageRequest from metadata.ingestion.api.models import Either from metadata.ingestion.api.steps import InvalidSourceException -from metadata.ingestion.lineage.models import ConnectionTypeDialectMapper +from metadata.ingestion.lineage.models import ConnectionTypeDialectMapper, Dialect from metadata.ingestion.lineage.parser import LineageParser from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.dashboard.dashboard_service import ( @@ -108,7 +121,6 @@ LookMlView, ViewName, ) -from metadata.ingestion.source.dashboard.looker.parser import LkmlParser from metadata.ingestion.source.dashboard.looker.utils import _clone_repo from metadata.readers.file.api_reader import ReadersCredentials from metadata.readers.file.base import Reader @@ -121,10 +133,13 @@ logger = ingestion_logger() - LIST_DASHBOARD_FIELDS = ["id", "title"] IMPORTED_PROJECTS_DIR = "imported_projects" +# we need to find the derived references in the SQL query using regex +# https://cloud.google.com/looker/docs/derived-tables#referencing_derived_tables_in_other_derived_tables +DERIVED_REFERENCES = r"\${([\w\s\d_.]+)\.SQL_TABLE_NAME}" + # Here we can update the fields to get further information, such as: # created_at, updated_at, last_updater_id, deleted_at, deleter_id, favorite_count, last_viewed_at GET_DASHBOARD_FIELDS = [ @@ -156,6 +171,13 @@ def build_datamodel_name(model_name: str, explore_name: str) -> str: return clean_dashboard_name(model_name + "_" + explore_name) +def find_derived_references(sql_query: str) -> List[str]: + if sql_query is None: + return [] + matches = re.findall(DERIVED_REFERENCES, sql_query) + return matches + + class LookerSource(DashboardServiceSource): """ Looker Source Class. @@ -163,6 +185,8 @@ class LookerSource(DashboardServiceSource): Its client uses Looker 40 from the SDK: client = looker_sdk.init40() """ + # pylint: disable=too-many-instance-attributes + config: WorkflowSource metadata: OpenMetadata client: Looker40SDK @@ -178,11 +202,15 @@ def __init__( self._explores_cache = {} self._repo_credentials: Optional[ReadersCredentials] = None self._reader_class: Optional[Type[Reader]] = None - self._project_parsers: Optional[Dict[str, LkmlParser]] = None + self._project_parsers: Optional[Dict[str, BulkLkmlParser]] = None self._main_lookml_repo: Optional[LookMLRepo] = None self._main__lookml_manifest: Optional[LookMLManifest] = None self._view_data_model: Optional[DashboardDataModel] = None + self._parsed_views: Optional[Dict[str, str]] = {} + self._unparsed_views: Optional[Dict[str, str]] = {} + self._derived_dependencies = nx.DiGraph() + self._added_lineage: Optional[Dict] = {} @classmethod @@ -260,7 +288,7 @@ def prepare(self): self._main__lookml_manifest = self.__read_manifest(credentials) @property - def parser(self) -> Optional[Dict[str, LkmlParser]]: + def parser(self) -> Optional[Dict[str, BulkLkmlParser]]: if self.repository_credentials: return self._project_parsers return None @@ -282,7 +310,7 @@ def parser(self, all_lookml_models: Sequence[LookmlModel]) -> None: """ if self.repository_credentials: all_projects: Set[str] = {model.project_name for model in all_lookml_models} - self._project_parsers: Dict[str, LkmlParser] = { + self._project_parsers: Dict[str, BulkLkmlParser] = { project_name: BulkLkmlParser( reader=self.reader(Path(self._main_lookml_repo.path)) ) @@ -325,7 +353,7 @@ def repository_credentials(self) -> Optional[ReadersCredentials]: """ if not self._repo_credentials: if self.service_connection.gitCredentials and isinstance( - self.service_connection.gitCredentials, ReadersCredentials + self.service_connection.gitCredentials, get_args(ReadersCredentials) ): self._repo_credentials = self.service_connection.gitCredentials @@ -487,9 +515,13 @@ def _get_explore_sql(self, explore: LookmlModelExplore) -> Optional[str]: try: project_parser = self.parser.get(explore.project_name) if project_parser: - return project_parser.parsed_files.get( + explore_sql = project_parser.parsed_files.get( Includes(get_path_from_link(explore.lookml_link)) ) + logger.debug( + f"Explore SQL for project {explore.project_name}: \n{explore_sql}" + ) + return explore_sql except Exception as err: logger.warning(f"Exception getting the model sql: {err}") @@ -544,6 +576,68 @@ def _process_view( ) ) + def replace_derived_references(self, sql_query): + """ + Replace all derived references with the parsed views sql query + will replace the derived references in the SQL query using regex + for e.g. It will replace ${view_name.SQL_TABLE_NAME} with the parsed view query for view_name + https://cloud.google.com/looker/docs/derived-tables#referencing_derived_tables_in_other_derived_tables + """ + try: + sql_query = re.sub( + DERIVED_REFERENCES, + # from `${view_name.SQL_TABLE_NAME}` we want the `view_name`. + # match.group(1) will give us the `view_name` + lambda match: f"({self._parsed_views.get(match.group(1), match.group(0))})", + sql_query, + ) + except Exception as e: + logger.warning( + f"Something went wrong while replacing derived view references: {e}" + ) + return sql_query + + def build_lineage_for_unparsed_views(self) -> Iterable[Either[AddLineageRequest]]: + """ + build lineage by parsing the unparsed views containing derived references + """ + try: + # Doing a reversed topological sort to process the views in the right order + for view_name in reversed( + list(nx.topological_sort(self._derived_dependencies)) + ): + if view_name in self._parsed_views: + # Skip if already processed + continue + sql_query = self.replace_derived_references( + self._unparsed_views[view_name] + ) + if view_references := find_derived_references(sql_query): + # There are still derived references in the view query + logger.debug( + f"Views {view_references} not found for {view_name}. Skipping." + ) + continue + self._parsed_views[view_name] = sql_query + del self._unparsed_views[view_name] + yield from self._build_lineage_for_view(view_name, sql_query) + + except Exception as err: + yield Either( + left=StackTraceError( + name="parse_unparsed_views", + error=f"Error parsing unparsed views: {err}", + stackTrace=traceback.format_exc(), + ) + ) + + def _add_dependency_edge(self, view_name: str, view_references: List[str]): + """ + Add a dependency edge between the view and the derived reference + """ + for dependent_view_name in view_references: + self._derived_dependencies.add_edge(view_name, dependent_view_name) + def add_view_lineage( self, view: LookMlView, explore: LookmlModelExplore ) -> Iterable[Either[AddLineageRequest]]: @@ -572,10 +666,13 @@ def add_view_lineage( if view.sql_table_name: sql_table_name = self._render_table_name(view.sql_table_name) - source_table_name = self._clean_table_name(sql_table_name) - # View to the source is only there if we are informing the dbServiceNames for db_service_name in db_service_names or []: + dialect = self._get_db_dialect(db_service_name) + source_table_name = self._clean_table_name(sql_table_name, dialect) + self._parsed_views[view.name] = source_table_name + + # View to the source is only there if we are informing the dbServiceNames yield self.build_lineage_request( source=source_table_name, db_service_name=db_service_name, @@ -586,25 +683,19 @@ def add_view_lineage( sql_query = view.derived_table.sql if not sql_query: return - for db_service_name in db_service_names or []: - db_service = self.metadata.get_by_name( - DatabaseService, db_service_name - ) - - lineage_parser = LineageParser( - sql_query, - ConnectionTypeDialectMapper.dialect_of( - db_service.connection.config.type.value - ), - timeout_seconds=30, - ) - if lineage_parser.source_tables: - for from_table_name in lineage_parser.source_tables: - yield self.build_lineage_request( - source=str(from_table_name), - db_service_name=db_service_name, - to_entity=self._view_data_model, - ) + if find_derived_references(sql_query): + sql_query = self.replace_derived_references(sql_query) + # If we still have derived references, we cannot process the view + if view_references := find_derived_references(sql_query): + self._add_dependency_edge(view.name, view_references) + logger.warning( + f"Not all references are replaced for view [{view.name}]. Parsing it later." + ) + return + logger.debug(f"Processing view [{view.name}] with SQL: \n[{sql_query}]") + yield from self._build_lineage_for_view(view.name, sql_query) + if self._unparsed_views: + self.build_lineage_for_unparsed_views() except Exception as err: yield Either( @@ -615,6 +706,33 @@ def add_view_lineage( ) ) + def _build_lineage_for_view( + self, view_name: str, sql_query: str + ) -> Iterable[Either[AddLineageRequest]]: + """ + Parse the SQL query and build lineage for the view. + """ + for db_service_name in self.get_db_service_names() or []: + lineage_parser = LineageParser( + sql_query, + self._get_db_dialect(db_service_name), + timeout_seconds=30, + ) + if lineage_parser.source_tables: + self._parsed_views[view_name] = sql_query + for from_table_name in lineage_parser.source_tables: + yield self.build_lineage_request( + source=str(from_table_name), + db_service_name=db_service_name, + to_entity=self._view_data_model, + ) + + def _get_db_dialect(self, db_service_name) -> Dialect: + db_service = self.metadata.get_by_name(DatabaseService, db_service_name) + return ConnectionTypeDialectMapper.dialect_of( + db_service.connection.config.type.value + ) + def get_dashboards_list(self) -> List[DashboardBase]: """ Get List of all dashboards @@ -718,7 +836,7 @@ def get_project_name(self, dashboard_details: LookerDashboard) -> Optional[str]: return None @staticmethod - def _clean_table_name(table_name: str) -> str: + def _clean_table_name(table_name: str, dialect: Dialect = Dialect.ANSI) -> str: """ sql_table_names might be renamed when defining an explore. E.g., customers as cust @@ -726,7 +844,10 @@ def _clean_table_name(table_name: str) -> str: :return: clean table name """ - return table_name.lower().split(" as ")[0].strip() + clean_table_name = table_name.lower().split(" as ")[0].strip() + if dialect == Dialect.BIGQUERY: + clean_table_name = clean_table_name.strip("`") + return clean_table_name @staticmethod def _render_table_name(table_name: str) -> str: diff --git a/ingestion/src/metadata/ingestion/source/dashboard/looker/parser.py b/ingestion/src/metadata/ingestion/source/dashboard/looker/parser.py index 008c914ed15d..b954e26d4166 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/looker/parser.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/looker/parser.py @@ -181,6 +181,9 @@ def get_view_from_cache(self, view_name: ViewName) -> Optional[LookMlView]: Otherwise, return None """ if view_name in self._views_cache: + logger.debug( + f"Found view [{view_name}] in cache: \n{self._views_cache[view_name]}" + ) return self._views_cache[view_name] return None diff --git a/ingestion/src/metadata/ingestion/source/dashboard/powerbi/client.py b/ingestion/src/metadata/ingestion/source/dashboard/powerbi/client.py index d1acc7901ac9..cfaa47379ef0 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/powerbi/client.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/powerbi/client.py @@ -47,7 +47,10 @@ logger = utils_logger() - +GETGROUPS_DEFAULT_PARAMS = {"$top": "1", "$skip": "0"} +API_RESPONSE_MESSAGE_KEY = "message" +AUTH_TOKEN_MAX_RETRIES = 5 +AUTH_TOKEN_RETRY_WAIT = 120 # Similar inner methods with mode client. That's fine. # pylint: disable=duplicate-code class PowerBiApiClient: @@ -59,6 +62,9 @@ class PowerBiApiClient: def __init__(self, config: PowerBIConnection): self.config = config + self.pagination_entity_per_page = min( + 100, self.config.pagination_entity_per_page + ) self.msal_client = msal.ConfidentialClientApplication( client_id=self.config.clientId, client_credential=self.config.clientSecret.get_secret_value(), @@ -82,42 +88,84 @@ def get_auth_token(self) -> Tuple[str, str]: """ logger.info("Generating PowerBi access token") - response_data = self.msal_client.acquire_token_silent( - scopes=self.config.scope, account=None - ) - + response_data = self.get_auth_token_from_cache() if not response_data: logger.info("Token does not exist in the cache. Getting a new token.") - response_data = self.msal_client.acquire_token_for_client( - scopes=self.config.scope - ) + response_data = self.generate_new_auth_token() + response_data = response_data or {} auth_response = PowerBiToken(**response_data) if not auth_response.access_token: raise InvalidSourceException( - "Failed to generate the PowerBi access token. Please check provided config" + f"Failed to generate the PowerBi access token. Please check provided config {response_data}" ) logger.info("PowerBi Access Token generated successfully") return auth_response.access_token, auth_response.expires_in + def generate_new_auth_token(self) -> Optional[dict]: + """generate new auth token""" + retry = AUTH_TOKEN_MAX_RETRIES + while retry: + try: + response_data = self.msal_client.acquire_token_for_client( + scopes=self.config.scope + ) + return response_data + except Exception as exc: + logger.debug(traceback.format_exc()) + logger.warning(f"Error generating new auth token: {exc}") + # wait for time and retry + retry -= 1 + if retry: + logger.warning( + f"Error generating new token: {exc}, " + f"sleep {AUTH_TOKEN_RETRY_WAIT} seconds retrying {retry} more times.." + ) + sleep(AUTH_TOKEN_RETRY_WAIT) + else: + logger.warning( + "Could not generate new token after maximum retries, " + "Please check provided configs" + ) + return None + + def get_auth_token_from_cache(self) -> Optional[dict]: + """fetch auth token from cache""" + retry = AUTH_TOKEN_MAX_RETRIES + while retry: + try: + response_data = self.msal_client.acquire_token_silent( + scopes=self.config.scope, account=None + ) + return response_data + except Exception as exc: + logger.debug(traceback.format_exc()) + logger.warning(f"Error getting token from cache: {exc}") + retry -= 1 + if retry: + logger.warning( + f"Error getting token from cache: {exc}, " + f"sleep {AUTH_TOKEN_RETRY_WAIT} seconds retrying {retry} more times.." + ) + sleep(AUTH_TOKEN_RETRY_WAIT) + else: + logger.warning( + "Could not get token from cache after maximum retries, " + "Please check provided configs" + ) + return None + def fetch_dashboards(self) -> Optional[List[PowerBIDashboard]]: """Get dashboards method Returns: List[PowerBIDashboard] """ - try: - if self.config.useAdminApis: - response_data = self.client.get("/myorg/admin/dashboards") - response = DashboardsResponse(**response_data) - return response.value - group = self.fetch_all_workspaces()[0] - return self.fetch_all_org_dashboards(group_id=group.id) - - except Exception as exc: # pylint: disable=broad-except - logger.debug(traceback.format_exc()) - logger.warning(f"Error fetching dashboards: {exc}") - - return None + if self.config.useAdminApis: + response_data = self.client.get("/myorg/admin/dashboards") + response = DashboardsResponse(**response_data) + return response.value + group = self.fetch_all_workspaces()[0] + return self.fetch_all_org_dashboards(group_id=group.id) def fetch_all_org_dashboards( self, group_id: str @@ -205,6 +253,7 @@ def fetch_dataset_tables( return None + # pylint: disable=too-many-branches,too-many-statements def fetch_all_workspaces(self) -> Optional[List[Group]]: """Method to fetch all powerbi workspace details Returns: @@ -213,22 +262,94 @@ def fetch_all_workspaces(self) -> Optional[List[Group]]: try: admin = "admin/" if self.config.useAdminApis else "" api_url = f"/myorg/{admin}groups" - entities_per_page = self.config.pagination_entity_per_page - params_data = {"$top": "1"} - response_data = self.client.get(api_url, data=params_data) - response = GroupsResponse(**response_data) - count = response.odata_count + entities_per_page = self.pagination_entity_per_page + failed_indexes = [] + params_data = GETGROUPS_DEFAULT_PARAMS + response = self.client.get(api_url, data=params_data) + if ( + not response + or API_RESPONSE_MESSAGE_KEY in response + or len(response) != len(GroupsResponse.__annotations__) + ): + logger.warning("Error fetching workspaces between results: (0, 1)") + if response and response.get(API_RESPONSE_MESSAGE_KEY): + logger.warning( + "Error message from API response: " + f"{str(response.get(API_RESPONSE_MESSAGE_KEY))}" + ) + failed_indexes.append(params_data) + count = 0 + else: + try: + response = GroupsResponse(**response) + count = response.odata_count + except Exception as exc: + logger.warning(f"Error processing GetGroups response: {exc}") + count = 0 indexes = math.ceil(count / entities_per_page) - workspaces = [] for index in range(indexes): params_data = { "$top": str(entities_per_page), "$skip": str(index * entities_per_page), } - response_data = self.client.get(api_url, data=params_data) - response = GroupsResponse(**response_data) - workspaces.extend(response.value) + response = self.client.get(api_url, data=params_data) + if ( + not response + or API_RESPONSE_MESSAGE_KEY in response + or len(response) != len(GroupsResponse.__annotations__) + ): + index_range = ( + int(params_data.get("$skip")), + int(params_data.get("$skip")) + int(params_data.get("$top")), + ) + logger.warning( + f"Error fetching workspaces between results: {str(index_range)}" + ) + if response and response.get(API_RESPONSE_MESSAGE_KEY): + logger.warning( + "Error message from API response: " + f"{str(response.get(API_RESPONSE_MESSAGE_KEY))}" + ) + failed_indexes.append(params_data) + continue + try: + response = GroupsResponse(**response) + workspaces.extend(response.value) + except Exception as exc: + logger.warning(f"Error processing GetGroups response: {exc}") + + if failed_indexes: + logger.info( + "Retrying one more time on failed indexes to get workspaces" + ) + for params_data in failed_indexes: + response = self.client.get(api_url, data=params_data) + if ( + not response + or API_RESPONSE_MESSAGE_KEY in response + or len(response) != len(GroupsResponse.__annotations__) + ): + index_range = ( + int(params_data.get("$skip")), + int(params_data.get("$skip")) + + int(params_data.get("$top")), + ) + logger.warning( + f"Workspaces between results {str(index_range)} " + "could not be fetched on multiple attempts" + ) + if response and response.get(API_RESPONSE_MESSAGE_KEY): + logger.warning( + "Error message from API response: " + f"{str(response.get(API_RESPONSE_MESSAGE_KEY))}" + ) + continue + try: + response = GroupsResponse(**response) + workspaces.extend(response.value) + except Exception as exc: + logger.warning(f"Error processing GetGroups response: {exc}") return workspaces except Exception as exc: # pylint: disable=broad-except logger.debug(traceback.format_exc()) diff --git a/ingestion/src/metadata/ingestion/source/dashboard/powerbi/metadata.py b/ingestion/src/metadata/ingestion/source/dashboard/powerbi/metadata.py index 685e699c281d..a918b46fd8a1 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/powerbi/metadata.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/powerbi/metadata.py @@ -92,6 +92,11 @@ def __init__( self.datamodel_file_mappings = [] def prepare(self): + """ + - Since we get all the required info i.e. reports, dashboards, charts, datasets + with workflow scan approach, we are populating bulk data for workspace. + - Some individual APIs are not able to yield data with details. + """ if self.service_connection.useAdminApis: groups = self.get_admin_workspace_data() else: @@ -185,24 +190,41 @@ def get_admin_workspace_data(self) -> Optional[List[Group]]: workspace_scan = self.client.api_client.initiate_workspace_scan( workspace_ids_chunk ) + if not workspace_scan: + logger.error( + f"Error initiating workspace scan for ids:{str(workspace_ids_chunk)}\n moving to next set of workspaces" + ) + count += 1 + continue # Keep polling the scan status endpoint to check if scan is succeeded workspace_scan_status = self.client.api_client.wait_for_scan_complete( scan_id=workspace_scan.id ) - if workspace_scan_status: - response = self.client.api_client.fetch_workspace_scan_result( - scan_id=workspace_scan.id + if not workspace_scan_status: + logger.error( + f"Max poll hit to scan status for scan_id: {workspace_scan.id}, moving to next set of workspaces" ) - groups.extend( - [ - active_workspace - for active_workspace in response.workspaces - if active_workspace.state == "Active" - ] + count += 1 + continue + + # Get scan result for successfull scan + response = self.client.api_client.fetch_workspace_scan_result( + scan_id=workspace_scan.id + ) + if not response: + logger.error( + f"Error getting workspace scan result for scan_id: {workspace_scan.id}" ) - else: - logger.error("Error in fetching dashboards and charts") + count += 1 + continue + groups.extend( + [ + active_workspace + for active_workspace in response.workspaces + if active_workspace.state == "Active" + ] + ) count += 1 else: logger.error("Unable to fetch any PowerBI workspaces") diff --git a/ingestion/src/metadata/ingestion/source/dashboard/redash/client.py b/ingestion/src/metadata/ingestion/source/dashboard/redash/client.py index 30cc9e9cd6c8..40a0e925aade 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/redash/client.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/redash/client.py @@ -13,6 +13,7 @@ """ from metadata.ingestion.ometa.client import REST, ClientConfig +from metadata.utils.helpers import clean_uri from metadata.utils.logger import utils_logger logger = utils_logger() @@ -28,8 +29,8 @@ class RedashApiClient: def __init__(self, config): self.config = config client_config = ClientConfig( - base_url=str(config.hostPort), - api_version="", + base_url=clean_uri(config.hostPort), + api_version="api", access_token=config.apiKey.get_secret_value(), auth_header="Authorization", auth_token_mode="Key", @@ -41,16 +42,11 @@ def dashboards(self, page=1, page_size=25): """GET api/dashboards""" params_data = {"page": page, "page_size": page_size} - return self.client.get(path="api/dashboards", data=params_data) + return self.client.get(path="/dashboards", data=params_data) - def get_dashboard(self, slug): - """GET api/dashboards/""" - - # The API changed from redash v9 onwards - # legacy=true allows us to get the results in the old way - return self.client.get( - f"api/dashboards/{slug}?legacy=true", - ) + def get_dashboard(self, dashboard_id: int): + """GET api/dashboards/""" + return self.client.get(f"/dashboards/{dashboard_id}") def paginate(self, resource, page=1, page_size=25, **kwargs): """Load all items of a paginated resource""" diff --git a/ingestion/src/metadata/ingestion/source/dashboard/redash/metadata.py b/ingestion/src/metadata/ingestion/source/dashboard/redash/metadata.py index 94ae4a0cb7dc..1067e1afcc76 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/redash/metadata.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/redash/metadata.py @@ -115,7 +115,7 @@ def get_dashboard_name(self, dashboard: dict) -> str: return dashboard["name"] def get_dashboard_details(self, dashboard: dict) -> dict: - return self.client.get_dashboard(dashboard["slug"]) + return self.client.get_dashboard(dashboard["id"]) def get_owner_ref(self, dashboard_details) -> Optional[EntityReferenceList]: """ @@ -160,9 +160,9 @@ def yield_dashboard( dashboard_request = CreateDashboardRequest( name=EntityName(str(dashboard_details["id"])), displayName=dashboard_details.get("name"), - description=Markdown(dashboard_description) - if dashboard_description - else None, + description=( + Markdown(dashboard_description) if dashboard_description else None + ), charts=[ FullyQualifiedEntityName( fqn.build( @@ -275,9 +275,11 @@ def yield_dashboard_chart( yield Either( right=CreateChartRequest( name=EntityName(str(widgets["id"])), - displayName=chart_display_name - if visualization and visualization["query"] - else "", + displayName=( + chart_display_name + if visualization and visualization["query"] + else "" + ), chartType=get_standard_chart_type( visualization["type"] if visualization else "" ), @@ -285,9 +287,11 @@ def yield_dashboard_chart( self.context.get().dashboard_service ), sourceUrl=SourceUrl(self.get_dashboard_url(dashboard_details)), - description=Markdown(visualization["description"]) - if visualization - else None, + description=( + Markdown(visualization["description"]) + if visualization + else None + ), ) ) except Exception as exc: diff --git a/ingestion/src/metadata/ingestion/source/database/bigquery/metadata.py b/ingestion/src/metadata/ingestion/source/database/bigquery/metadata.py index 078853aca599..87e5b36c96f9 100644 --- a/ingestion/src/metadata/ingestion/source/database/bigquery/metadata.py +++ b/ingestion/src/metadata/ingestion/source/database/bigquery/metadata.py @@ -34,9 +34,11 @@ from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema from metadata.generated.schema.entity.data.storedProcedure import StoredProcedureCode from metadata.generated.schema.entity.data.table import ( + ConstraintType, PartitionColumnDetails, PartitionIntervalTypes, Table, + TableConstraint, TablePartition, TableType, ) @@ -96,6 +98,7 @@ from metadata.ingestion.source.database.multi_db_source import MultiDBSource from metadata.utils import fqn from metadata.utils.credentials import GOOGLE_CREDENTIALS +from metadata.utils.execution_time_tracker import calculate_execution_time from metadata.utils.filters import filter_by_database, filter_by_schema from metadata.utils.logger import ingestion_logger from metadata.utils.sqlalchemy_utils import ( @@ -657,6 +660,42 @@ def _get_partition_column_name( ) return None + @calculate_execution_time() + def update_table_constraints( + self, + table_name, + schema_name, + db_name, + table_constraints, + foreign_columns, + columns, + ) -> List[TableConstraint]: + """ + From topology. + process the table constraints of all tables + """ + table_constraints = super().update_table_constraints( + table_name, + schema_name, + db_name, + table_constraints, + foreign_columns, + columns, + ) + try: + table = self.client.get_table(fqn._build(db_name, schema_name, table_name)) + if hasattr(table, "clustering_fields") and table.clustering_fields: + table_constraints.append( + TableConstraint( + constraintType=ConstraintType.CLUSTER_KEY, + columns=table.clustering_fields, + ) + ) + except Exception as exc: + logger.warning(f"Error getting clustering fields for {table_name}: {exc}") + logger.debug(traceback.format_exc()) + return table_constraints + def get_table_partition_details( self, table_name: str, schema_name: str, inspector: Inspector ) -> Tuple[bool, Optional[TablePartition]]: @@ -667,6 +706,33 @@ def get_table_partition_details( database = self.context.get().database table = self.client.get_table(fqn._build(database, schema_name, table_name)) columns = inspector.get_columns(table_name, schema_name, db_name=database) + if ( + hasattr(table, "external_data_configuration") + and hasattr(table.external_data_configuration, "hive_partitioning") + and table.external_data_configuration.hive_partitioning + ): + # Ingesting External Hive Partitioned Tables + from google.cloud.bigquery.external_config import ( # pylint: disable=import-outside-toplevel + HivePartitioningOptions, + ) + + partition_details: HivePartitioningOptions = ( + table.external_data_configuration.hive_partitioning + ) + return True, TablePartition( + columns=[ + PartitionColumnDetails( + columnName=self._get_partition_column_name( + columns=columns, + partition_field_name=field, + ), + interval=str(partition_details._properties.get("mode")), + intervalType=PartitionIntervalTypes.OTHER, + ) + for field in partition_details._properties.get("fields") + ] + ) + if table.time_partitioning is not None: if table.time_partitioning.field: table_partition = TablePartition( @@ -708,6 +774,30 @@ def get_table_partition_details( table_partition.interval = table.range_partitioning.range_.interval table_partition.columnName = table.range_partitioning.field return True, TablePartition(columns=[table_partition]) + if ( + hasattr(table, "_properties") + and table._properties.get("partitionDefinition") + and table._properties.get("partitionDefinition").get( + "partitionedColumn" + ) + ): + + return True, TablePartition( + columns=[ + PartitionColumnDetails( + columnName=self._get_partition_column_name( + columns=columns, + partition_field_name=field.get("field"), + ), + intervalType=PartitionIntervalTypes.OTHER, + ) + for field in table._properties.get("partitionDefinition").get( + "partitionedColumn" + ) + if field and field.get("field") + ] + ) + except Exception as exc: logger.debug(traceback.format_exc()) logger.warning( diff --git a/ingestion/src/metadata/ingestion/source/database/bigquery/profiler/profiler.py b/ingestion/src/metadata/ingestion/source/database/bigquery/profiler/profiler.py index 2cd0f225b31d..750ae61fb20d 100644 --- a/ingestion/src/metadata/ingestion/source/database/bigquery/profiler/profiler.py +++ b/ingestion/src/metadata/ingestion/source/database/bigquery/profiler/profiler.py @@ -24,6 +24,7 @@ def _compute_system_metrics( return self.system_metrics_computer.get_system_metrics( table=runner.dataset, usage_location=self.service_connection_config.usageLocation, + runner=runner, ) def initialize_system_metrics_computer(self) -> BigQuerySystemMetricsComputer: diff --git a/ingestion/src/metadata/ingestion/source/database/bigquery/profiler/system.py b/ingestion/src/metadata/ingestion/source/database/bigquery/profiler/system.py index bd04c112a682..64b9cf9c45ba 100644 --- a/ingestion/src/metadata/ingestion/source/database/bigquery/profiler/system.py +++ b/ingestion/src/metadata/ingestion/source/database/bigquery/profiler/system.py @@ -29,8 +29,8 @@ def get_kwargs(self, **kwargs): runner: QueryRunner = kwargs.get("runner") return { "table": runner.table_name, - "database": runner.session.get_bind().url.database, - "schema": runner.schema_name, + "project_id": runner.session.get_bind().url.host, + "dataset_id": runner.schema_name, "usage_location": kwargs.get("usage_location"), } diff --git a/ingestion/src/metadata/ingestion/source/database/common_db_source.py b/ingestion/src/metadata/ingestion/source/database/common_db_source.py index 663b21628cbc..f5345aaafdd8 100644 --- a/ingestion/src/metadata/ingestion/source/database/common_db_source.py +++ b/ingestion/src/metadata/ingestion/source/database/common_db_source.py @@ -681,7 +681,7 @@ def _get_foreign_constraints( foreign_constraint = self._prepare_foreign_constraints( supports_database, column, table_name, schema_name, db_name, columns ) - if foreign_constraint: + if foreign_constraint and foreign_constraint not in foreign_constraints: foreign_constraints.append(foreign_constraint) return foreign_constraints @@ -705,7 +705,11 @@ def update_table_constraints( ) if foreign_table_constraints: if table_constraints: - table_constraints.extend(foreign_table_constraints) + table_constraints.extend( + constraint + for constraint in foreign_table_constraints + if constraint not in table_constraints + ) else: table_constraints = foreign_table_constraints return table_constraints diff --git a/ingestion/src/metadata/ingestion/source/database/databricks/connection.py b/ingestion/src/metadata/ingestion/source/database/databricks/connection.py index 14d3d3e392e9..7bae5d4b7f8a 100644 --- a/ingestion/src/metadata/ingestion/source/database/databricks/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/databricks/connection.py @@ -38,9 +38,9 @@ test_connection_steps, ) from metadata.ingestion.ometa.ometa_api import OpenMetadata -from metadata.ingestion.source.database.databricks.client import DatabricksClient from metadata.ingestion.source.database.databricks.queries import ( DATABRICKS_GET_CATALOGS, + DATABRICKS_SQL_STATEMENT_TEST, ) from metadata.utils.constants import THREE_MIN from metadata.utils.logger import ingestion_logger @@ -81,7 +81,6 @@ def test_connection( Test connection. This can be executed either as part of a metadata workflow or during an Automation Workflow """ - client = DatabricksClient(service_connection) def test_database_query(engine: Engine, statement: str): """ @@ -106,7 +105,13 @@ def test_database_query(engine: Engine, statement: str): engine=connection, statement=DATABRICKS_GET_CATALOGS, ), - "GetQueries": client.test_query_api_access, + "GetQueries": partial( + test_database_query, + engine=connection, + statement=DATABRICKS_SQL_STATEMENT_TEST.format( + query_history=service_connection.queryHistoryTable + ), + ), } return test_connection_steps( diff --git a/ingestion/src/metadata/ingestion/source/database/databricks/lineage.py b/ingestion/src/metadata/ingestion/source/database/databricks/lineage.py index a77cb780e56d..eb4b74b5d495 100644 --- a/ingestion/src/metadata/ingestion/source/database/databricks/lineage.py +++ b/ingestion/src/metadata/ingestion/source/database/databricks/lineage.py @@ -11,12 +11,10 @@ """ Databricks lineage module """ -import traceback -from datetime import datetime -from typing import Iterator -from metadata.generated.schema.type.basic import DateTime -from metadata.generated.schema.type.tableQuery import TableQuery +from metadata.ingestion.source.database.databricks.queries import ( + DATABRICKS_SQL_STATEMENT, +) from metadata.ingestion.source.database.databricks.query_parser import ( DatabricksQueryParserSource, ) @@ -31,23 +29,13 @@ class DatabricksLineageSource(DatabricksQueryParserSource, LineageSource): Databricks Lineage Legacy Source """ - def yield_table_query(self) -> Iterator[TableQuery]: - data = self.client.list_query_history( - start_date=self.start, - end_date=self.end, + sql_stmt = DATABRICKS_SQL_STATEMENT + + filters = """ + AND ( + lower(statement_text) LIKE '%%create%%select%%' + OR lower(statement_text) LIKE '%%insert%%into%%select%%' + OR lower(statement_text) LIKE '%%update%%' + OR lower(statement_text) LIKE '%%merge%%' ) - for row in data or []: - try: - if self.client.is_query_valid(row): - yield TableQuery( - dialect=self.dialect.value, - query=row.get("query_text"), - userName=row.get("user_name"), - startTime=str(row.get("query_start_time_ms")), - endTime=str(row.get("execution_end_time_ms")), - analysisDate=DateTime(datetime.now()), - serviceName=self.config.serviceName, - ) - except Exception as exc: - logger.debug(traceback.format_exc()) - logger.warning(f"Error processing query_dict {row}: {exc}") + """ diff --git a/ingestion/src/metadata/ingestion/source/database/databricks/metadata.py b/ingestion/src/metadata/ingestion/source/database/databricks/metadata.py index 3bc7b99f6d0d..5166ca1f56de 100644 --- a/ingestion/src/metadata/ingestion/source/database/databricks/metadata.py +++ b/ingestion/src/metadata/ingestion/source/database/databricks/metadata.py @@ -113,11 +113,6 @@ class MAP(String): ) -def format_schema_name(schema): - # Adds back quotes(``) if hyphen(-) in schema name - return f"`{schema}`" if "-" in schema else schema - - # This method is from hive dialect originally but # is overridden to optimize DESCRIBE query execution def _get_table_columns(self, connection, table_name, schema, db_name): @@ -158,7 +153,6 @@ def _get_table_columns(self, connection, table_name, schema, db_name): def _get_column_rows(self, connection, table_name, schema, db_name): # get columns and strip whitespace - schema = format_schema_name(schema=schema) table_columns = _get_table_columns( # pylint: disable=protected-access self, connection, table_name, schema, db_name ) @@ -212,11 +206,10 @@ def get_columns(self, connection, table_name, schema=None, **kw): "system_data_type": raw_col_type, } if col_type in {"array", "struct", "map"}: - col_name = f"`{col_name}`" if "." in col_name else col_name try: rows = dict( connection.execute( - f"DESCRIBE TABLE {kw.get('db_name')}.{schema}.{table_name} {col_name}" + f"DESCRIBE TABLE `{kw.get('db_name')}`.`{schema}`.`{table_name}` `{col_name}`" ).fetchall() ) col_info["system_data_type"] = rows["data_type"] @@ -393,8 +386,7 @@ def get_table_type(self, connection, database, schema, table): database_name=database, schema_name=schema, table_name=table ) else: - schema = format_schema_name(schema=schema) - query = f"DESCRIBE TABLE EXTENDED {schema}.{table}" + query = f"DESCRIBE TABLE EXTENDED `{schema}`.`{table}`" rows = get_table_comment_result( self, connection=connection, @@ -761,7 +753,6 @@ def get_table_description( ) -> str: description = None try: - schema_name = format_schema_name(schema=schema_name) query = DATABRICKS_GET_TABLE_COMMENTS.format( database_name=self.context.get().database, schema_name=schema_name, @@ -816,9 +807,7 @@ def get_owner_ref(self, table_name: str) -> Optional[EntityReferenceList]: try: query = DATABRICKS_GET_TABLE_COMMENTS.format( database_name=self.context.get().database, - schema_name=format_schema_name( - schema=self.context.get().database_schema - ), + schema_name=self.context.get().database_schema, table_name=table_name, ) result = self.inspector.dialect.get_table_comment_result( diff --git a/ingestion/src/metadata/ingestion/source/database/databricks/queries.py b/ingestion/src/metadata/ingestion/source/database/databricks/queries.py index ca057716609e..25cdcedfc105 100644 --- a/ingestion/src/metadata/ingestion/source/database/databricks/queries.py +++ b/ingestion/src/metadata/ingestion/source/database/databricks/queries.py @@ -14,6 +14,30 @@ import textwrap +DATABRICKS_SQL_STATEMENT = textwrap.dedent( + """ + SELECT + statement_type AS query_type, + statement_text AS query_text, + executed_by AS user_name, + start_time AS start_time, + null AS database_name, + null AS schema_name, + end_time AS end_time, + total_duration_ms/1000 AS duration + from {query_history} + WHERE statement_text NOT LIKE '/* {{"app": "OpenMetadata", %%}} */%%' + AND statement_text NOT LIKE '/* {{"app": "dbt", %%}} */%%' + AND start_time between to_timestamp('{start_time}') and to_timestamp('{end_time}') + {filters} + LIMIT {result_limit} + """ +) + +DATABRICKS_SQL_STATEMENT_TEST = """ + SELECT statement_text from {query_history} LIMIT 1 +""" + DATABRICKS_VIEW_DEFINITIONS = textwrap.dedent( """ select @@ -25,27 +49,27 @@ ) DATABRICKS_GET_TABLE_COMMENTS = ( - "DESCRIBE TABLE EXTENDED {database_name}.{schema_name}.{table_name}" + "DESCRIBE TABLE EXTENDED `{database_name}`.`{schema_name}`.`{table_name}`" ) DATABRICKS_GET_CATALOGS = "SHOW CATALOGS" DATABRICKS_GET_CATALOGS_TAGS = textwrap.dedent( - """SELECT * FROM {database_name}.information_schema.catalog_tags;""" + """SELECT * FROM `{database_name}`.information_schema.catalog_tags;""" ) DATABRICKS_GET_SCHEMA_TAGS = textwrap.dedent( """ SELECT * - FROM {database_name}.information_schema.schema_tags""" + FROM `{database_name}`.information_schema.schema_tags""" ) DATABRICKS_GET_TABLE_TAGS = textwrap.dedent( """ SELECT * - FROM {database_name}.information_schema.table_tags + FROM `{database_name}`.information_schema.table_tags """ ) @@ -53,8 +77,8 @@ """ SELECT * - FROM {database_name}.information_schema.column_tags + FROM `{database_name}`.information_schema.column_tags """ ) -DATABRICKS_DDL = "SHOW CREATE TABLE {table_name}" +DATABRICKS_DDL = "SHOW CREATE TABLE `{table_name}`" diff --git a/ingestion/src/metadata/ingestion/source/database/databricks/query_parser.py b/ingestion/src/metadata/ingestion/source/database/databricks/query_parser.py index 00628bfbddaa..c67b06aa30ed 100644 --- a/ingestion/src/metadata/ingestion/source/database/databricks/query_parser.py +++ b/ingestion/src/metadata/ingestion/source/database/databricks/query_parser.py @@ -22,7 +22,6 @@ ) from metadata.ingestion.api.steps import InvalidSourceException from metadata.ingestion.ometa.ometa_api import OpenMetadata -from metadata.ingestion.source.database.databricks.client import DatabricksClient from metadata.ingestion.source.database.query_parser_source import QueryParserSource from metadata.utils.logger import ingestion_logger @@ -36,18 +35,6 @@ class DatabricksQueryParserSource(QueryParserSource, ABC): filters: str - def _init_super( - self, - config: WorkflowSource, - metadata: OpenMetadata, - ): - super().__init__(config, metadata, False) - - # pylint: disable=super-init-not-called - def __init__(self, config: WorkflowSource, metadata: OpenMetadata): - self._init_super(config=config, metadata=metadata) - self.client = DatabricksClient(self.service_connection) - @classmethod def create( cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None @@ -61,7 +48,16 @@ def create( ) return cls(config, metadata) - def prepare(self): + def get_sql_statement(self, start_time, end_time): """ - By default, there's nothing to prepare + returns sql statement to fetch query logs. + + Override if we have specific parameters """ + return self.sql_stmt.format( + start_time=start_time, + end_time=end_time, + filters=self.get_filters(), + result_limit=self.source_config.resultLimit, + query_history=self.service_connection.queryHistoryTable, + ) diff --git a/ingestion/src/metadata/ingestion/source/database/databricks/usage.py b/ingestion/src/metadata/ingestion/source/database/databricks/usage.py index 0e5364a465d8..fedbab2da486 100644 --- a/ingestion/src/metadata/ingestion/source/database/databricks/usage.py +++ b/ingestion/src/metadata/ingestion/source/database/databricks/usage.py @@ -11,12 +11,10 @@ """ Databricks usage module """ -import traceback -from datetime import datetime -from typing import Iterable -from metadata.generated.schema.type.basic import DateTime -from metadata.generated.schema.type.tableQuery import TableQueries, TableQuery +from metadata.ingestion.source.database.databricks.queries import ( + DATABRICKS_SQL_STATEMENT, +) from metadata.ingestion.source.database.databricks.query_parser import ( DatabricksQueryParserSource, ) @@ -31,36 +29,8 @@ class DatabricksUsageSource(DatabricksQueryParserSource, UsageSource): Databricks Usage Source """ - def yield_table_queries(self) -> Iterable[TableQuery]: - """ - Method to yield TableQueries - """ - queries = [] - data = self.client.list_query_history( - start_date=self.start, - end_date=self.end, - ) - for row in data or []: - try: - if self.client.is_query_valid(row): - queries.append( - TableQuery( - dialect=self.dialect.value, - query=row.get("query_text"), - userName=row.get("user_name"), - startTime=str(row.get("query_start_time_ms")), - endTime=str(row.get("execution_end_time_ms")), - analysisDate=DateTime(datetime.now()), - serviceName=self.config.serviceName, - duration=row.get("duration") - if row.get("duration") - else None, - ) - ) - except Exception as err: - logger.debug(traceback.format_exc()) - logger.warning( - f"Failed to process query {row.get('query_text')} due to: {err}" - ) + sql_stmt = DATABRICKS_SQL_STATEMENT - yield TableQueries(queries=queries) + filters = """ + AND statement_type NOT IN ('SHOW', 'DESCRIBE', 'USE') + """ diff --git a/ingestion/src/metadata/ingestion/source/database/db2/connection.py b/ingestion/src/metadata/ingestion/source/database/db2/connection.py index b25ac3efc757..c9efc348f730 100644 --- a/ingestion/src/metadata/ingestion/source/database/db2/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/db2/connection.py @@ -50,7 +50,7 @@ def get_connection(connection: Db2Connection) -> Engine: "w", encoding=UTF_8, ) as file: - file.write(connection.license) + file.write(connection.license.encode(UTF_8).decode("unicode-escape")) return create_generic_db_connection( connection=connection, diff --git a/ingestion/src/metadata/ingestion/source/database/db2/metadata.py b/ingestion/src/metadata/ingestion/source/database/db2/metadata.py index 794cb3b85903..44baf862e60d 100644 --- a/ingestion/src/metadata/ingestion/source/database/db2/metadata.py +++ b/ingestion/src/metadata/ingestion/source/database/db2/metadata.py @@ -13,6 +13,7 @@ from typing import Iterable, Optional from ibm_db_sa.base import ischema_names +from ibm_db_sa.reflection import DB2Reflector, OS390Reflector from sqlalchemy.engine.reflection import Inspector from sqlalchemy.engine.row import LegacyRow from sqlalchemy.sql.sqltypes import BOOLEAN @@ -26,6 +27,7 @@ from metadata.ingestion.api.steps import InvalidSourceException from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.database.common_db_source import CommonDbSourceService +from metadata.ingestion.source.database.db2.utils import get_unique_constraints from metadata.utils.logger import ingestion_logger logger = ingestion_logger() @@ -34,6 +36,10 @@ ischema_names.update({"BOOLEAN": BOOLEAN}) +DB2Reflector.get_unique_constraints = get_unique_constraints +OS390Reflector.get_unique_constraints = get_unique_constraints + + class Db2Source(CommonDbSourceService): """ Implements the necessary methods to extract diff --git a/ingestion/src/metadata/ingestion/source/database/db2/utils.py b/ingestion/src/metadata/ingestion/source/database/db2/utils.py new file mode 100644 index 000000000000..657942aca471 --- /dev/null +++ b/ingestion/src/metadata/ingestion/source/database/db2/utils.py @@ -0,0 +1,63 @@ +# Copyright 2021 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Module to define overriden dialect methods +""" +from sqlalchemy import and_, join, sql +from sqlalchemy.engine import reflection + + +@reflection.cache +def get_unique_constraints( + self, connection, table_name, schema=None, **kw +): # pylint: disable=unused-argument + """Small Method to override the Dialect default as it is not filtering properly the Schema and Table Name.""" + current_schema = self.denormalize_name(schema or self.default_schema_name) + table_name = self.denormalize_name(table_name) + syskeycol = self.sys_keycoluse + sysconst = self.sys_tabconst + query = ( + sql.select(syskeycol.c.constname, syskeycol.c.colname) + .select_from( + join( + syskeycol, + sysconst, + and_( + syskeycol.c.constname == sysconst.c.constname, + syskeycol.c.tabschema == sysconst.c.tabschema, + syskeycol.c.tabname == sysconst.c.tabname, + ), + ) + ) + .where( + and_( + sysconst.c.tabname == table_name, + sysconst.c.tabschema == current_schema, + sysconst.c.type == "U", + ) + ) + .order_by(syskeycol.c.constname) + ) + unique_consts = [] + curr_const = None + for r in connection.execute(query): + if curr_const == r[0]: + unique_consts[-1]["column_names"].append(self.normalize_name(r[1])) + else: + curr_const = r[0] + unique_consts.append( + { + "name": self.normalize_name(curr_const), + "column_names": [self.normalize_name(r[1])], + } + ) + return unique_consts diff --git a/ingestion/src/metadata/ingestion/source/database/dbt/dbt_service.py b/ingestion/src/metadata/ingestion/source/database/dbt/dbt_service.py index 5021f4c5fb22..40c5e96d9a80 100644 --- a/ingestion/src/metadata/ingestion/source/database/dbt/dbt_service.py +++ b/ingestion/src/metadata/ingestion/source/database/dbt/dbt_service.py @@ -15,7 +15,7 @@ from abc import ABC, abstractmethod from typing import Iterable, List -from dbt_artifacts_parser.parser import ( +from collate_dbt_artifacts_parser.parser import ( parse_catalog, parse_manifest, parse_run_results, @@ -122,6 +122,11 @@ class DbtServiceTopology(ServiceTopology): processor="process_dbt_descriptions", nullable=True, ), + NodeStage( + type_=DataModelLink, + processor="process_dbt_owners", + nullable=True, + ), ], ) process_dbt_tests: Annotated[ @@ -293,6 +298,12 @@ def process_dbt_descriptions(self, data_model_link: DataModelLink): Method to process DBT descriptions using patch APIs """ + @abstractmethod + def process_dbt_owners(self, data_model_link: DataModelLink): + """ + Method to process DBT owners using patch APIs + """ + def get_dbt_tests(self) -> dict: """ Prepare the DBT tests diff --git a/ingestion/src/metadata/ingestion/source/database/dbt/metadata.py b/ingestion/src/metadata/ingestion/source/database/dbt/metadata.py index 5c1b2cf81e31..a422fd6e4902 100644 --- a/ingestion/src/metadata/ingestion/source/database/dbt/metadata.py +++ b/ingestion/src/metadata/ingestion/source/database/dbt/metadata.py @@ -15,7 +15,7 @@ import traceback from copy import deepcopy from datetime import datetime -from typing import Any, Iterable, List, Optional, Union +from typing import Any, Iterable, List, Optional from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest from metadata.generated.schema.api.tests.createTestCase import CreateTestCaseRequest @@ -61,7 +61,9 @@ from metadata.ingestion.lineage.models import ConnectionTypeDialectMapper from metadata.ingestion.lineage.sql_lineage import get_lineage_by_query from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification +from metadata.ingestion.models.patch_request import PatchedEntity, PatchRequest from metadata.ingestion.models.table_metadata import ColumnDescription +from metadata.ingestion.ometa.client import APIError from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.database.column_type_parser import ColumnTypeParser from metadata.ingestion.source.database.database_service import DataModelLink @@ -97,7 +99,7 @@ from metadata.utils.entity_link import get_table_fqn from metadata.utils.logger import ingestion_logger from metadata.utils.tag_utils import get_ometa_tag_and_classification, get_tag_labels -from metadata.utils.time_utils import convert_timestamp_to_milliseconds +from metadata.utils.time_utils import datetime_to_timestamp logger = ingestion_logger() @@ -325,11 +327,14 @@ def add_dbt_tests( None, ) - def _add_dbt_freshness_test_from_sources( + def add_dbt_sources( self, key: str, manifest_node, manifest_entities, dbt_objects: DbtObjects - ): - # in dbt manifest sources node name is table/view name (not test name like with test nodes) - # so in order for the test creation to be named precisely I am amending manifest node name within it's deepcopy + ) -> None: + """ + Method to append dbt test cases based on sources file for later processing + In dbt manifest sources node name is table/view name (not test name like with test nodes) + So in order for the test creation to be named precisely I am amending manifest node name within it's deepcopy + """ manifest_node_new = deepcopy(manifest_node) manifest_node_new.name = manifest_node_new.name + "_freshness" @@ -349,17 +354,57 @@ def _add_dbt_freshness_test_from_sources( DbtCommonEnum.RESULTS.value ] = freshness_test_result - def add_dbt_sources( - self, key: str, manifest_node, manifest_entities, dbt_objects: DbtObjects - ) -> None: - """ - Method to append dbt test cases based on sources file for later processing - """ - self._add_dbt_freshness_test_from_sources( - key, manifest_node, manifest_entities, dbt_objects - ) + def _get_table_entity(self, table_fqn) -> Optional[Table]: + def search_table(fqn_search_string: str) -> Optional[Table]: + table_entities = get_entity_from_es_result( + entity_list=self.metadata.es_search_from_fqn( + entity_type=Table, + fqn_search_string=fqn_search_string, + fields="sourceHash", + ), + fetch_multiple_entities=True, + ) + logger.debug( + f"Found table entities from {fqn_search_string}: {table_entities}" + ) + return next(iter(filter(None, table_entities)), None) - # pylint: disable=too-many-locals, too-many-branches, too-many-statements + try: + table_entity = search_table(table_fqn) + if table_entity: + return table_entity + + if self.source_config.searchAcrossDatabases: + logger.warning( + f"Table {table_fqn} not found under service: {self.config.serviceName}." + "Trying to find table across services" + ) + _, database_name, schema_name, table_name = fqn.split(table_fqn) + table_fqn = fqn.build( + self.metadata, + entity_type=Table, + service_name="*", + database_name=database_name, + schema_name=schema_name, + table_name=table_name, + ) + table_entity = search_table(table_fqn) + if table_entity: + return table_entity + + logger.warning( + f"Unable to find the table '{table_fqn}' in OpenMetadata. " + "Please check if the table exists and is ingested in OpenMetadata. " + "Also, ensure the name, database, and schema of the manifest node" + "match the table present in OpenMetadata." + ) + except Exception as exc: + logger.debug(traceback.format_exc()) + logger.warning(f"Failed to get table entity from OpenMetadata: {exc}") + + return None + + # pylint: disable=too-many-locals, too-many-branches def yield_data_models( self, dbt_objects: DbtObjects ) -> Iterable[Either[DataModelLink]]: @@ -470,28 +515,19 @@ def yield_data_models( dbt_compiled_query = get_dbt_compiled_query(manifest_node) dbt_raw_query = get_dbt_raw_query(manifest_node) - # Get the table entity from ES table_fqn = fqn.build( self.metadata, entity_type=Table, - service_name="*", + service_name=self.config.serviceName, database_name=get_corrected_name(manifest_node.database), schema_name=get_corrected_name(manifest_node.schema_), table_name=model_name, ) - table_entity: Optional[ - Union[Table, List[Table]] - ] = get_entity_from_es_result( - entity_list=self.metadata.es_search_from_fqn( - entity_type=Table, - fqn_search_string=table_fqn, - fields="sourceHash", - ), - fetch_multiple_entities=False, - ) - - if table_entity: + if table_entity := self._get_table_entity(table_fqn=table_fqn): + logger.debug( + f"Using Table Entity for datamodel: {table_entity}" + ) data_model_link = DataModelLink( table_entity=table_entity, datamodel=DataModel( @@ -522,13 +558,7 @@ def yield_data_models( ) yield Either(right=data_model_link) self.context.get().data_model_links.append(data_model_link) - else: - logger.warning( - f"Unable to find the table '{table_fqn}' in OpenMetadata" - "Please check if the table exists and is ingested in OpenMetadata" - "Also name, database, schema of the manifest node matches with the table present " - "in OpenMetadata" - ) + except Exception as exc: yield Either( left=StackTraceError( @@ -572,22 +602,14 @@ def parse_upstream_nodes(self, manifest_entities, dbt_node): parent_fqn = fqn.build( self.metadata, entity_type=Table, - service_name="*", + service_name=self.config.serviceName, database_name=get_corrected_name(parent_node.database), schema_name=get_corrected_name(parent_node.schema_), table_name=table_name, ) # check if the parent table exists in OM before adding it to the upstream list - parent_table_entity: Optional[ - Union[Table, List[Table]] - ] = get_entity_from_es_result( - entity_list=self.metadata.es_search_from_fqn( - entity_type=Table, fqn_search_string=parent_fqn - ), - fetch_multiple_entities=False, - ) - if parent_table_entity: + if self._get_table_entity(table_fqn=parent_fqn): upstream_nodes.append(parent_fqn) except Exception as exc: # pylint: disable=broad-except logger.debug(traceback.format_exc()) @@ -600,22 +622,14 @@ def parse_upstream_nodes(self, manifest_entities, dbt_node): parent_fqn = fqn.build( self.metadata, entity_type=Table, - service_name="*", + service_name=self.config.serviceName, database_name=get_corrected_name(dbt_node.database), schema_name=get_corrected_name(dbt_node.schema_), table_name=dbt_node.name, ) # check if the parent table exists in OM before adding it to the upstream list - parent_table_entity: Optional[ - Union[Table, List[Table]] - ] = get_entity_from_es_result( - entity_list=self.metadata.es_search_from_fqn( - entity_type=Table, fqn_search_string=parent_fqn - ), - fetch_multiple_entities=False, - ) - if parent_table_entity: + if self._get_table_entity(table_fqn=parent_fqn): upstream_nodes.append(parent_fqn) return upstream_nodes @@ -707,14 +721,8 @@ def create_dbt_lineage( for upstream_node in data_model_link.datamodel.upstream: try: - from_es_result = self.metadata.es_search_from_fqn( - entity_type=Table, - fqn_search_string=upstream_node, - ) - from_entity: Optional[ - Union[Table, List[Table]] - ] = get_entity_from_es_result( - entity_list=from_es_result, fetch_multiple_entities=False + from_entity: Optional[Table] = self._get_table_entity( + table_fqn=upstream_node ) if from_entity and to_entity: yield Either( @@ -886,6 +894,47 @@ def process_dbt_descriptions(self, data_model_link: DataModelLink): f"to update dbt description: {exc}" ) + def process_dbt_owners( + self, data_model_link: DataModelLink + ) -> Iterable[Either[PatchedEntity]]: + """ + Method to process DBT owners + """ + table_entity: Table = data_model_link.table_entity + if table_entity: + logger.debug( + f"Processing DBT owners for: {table_entity.fullyQualifiedName.root}" + ) + try: + data_model = data_model_link.datamodel + if ( + data_model.resourceType != DbtCommonEnum.SOURCE.value + and self.source_config.dbtUpdateOwners + ): + logger.debug( + f"Overwriting owners with DBT owners: {table_entity.fullyQualifiedName.root}" + ) + if data_model.owners: + new_entity = deepcopy(table_entity) + new_entity.owners = data_model.owners + yield Either( + right=PatchRequest( + original_entity=table_entity, + new_entity=new_entity, + override_metadata=True, + ) + ) + + except Exception as exc: # pylint: disable=broad-except + yield Either( + left=StackTraceError( + name=str(table_entity.fullyQualifiedName.root), + error=f"Failed to parse the node" + f"{table_entity.fullyQualifiedName.root} to update dbt owner: {exc}", + stackTrace=traceback.format_exc(), + ) + ) + def create_dbt_tests_definition( self, dbt_test: dict ) -> Iterable[Either[CreateTestDefinitionRequest]]: @@ -947,6 +996,7 @@ def create_dbt_test_case( self.metadata, entity_link_str ) table_fqn = get_table_fqn(entity_link_str) + logger.debug(f"Table fqn found: {table_fqn}") source_elements = table_fqn.split(fqn.FQN_SEPARATOR) test_case_fqn = fqn.build( self.metadata, @@ -982,6 +1032,7 @@ def create_dbt_test_case( owners=None, ) ) + logger.debug(f"Test case Already Exists: {test_case_fqn}") except Exception as err: # pylint: disable=broad-except yield Either( left=StackTraceError( @@ -1043,7 +1094,7 @@ def add_dbt_test_result(self, dbt_test: dict): # Create the test case result object test_case_result = TestCaseResult( timestamp=Timestamp( - convert_timestamp_to_milliseconds(dbt_timestamp.timestamp()) + datetime_to_timestamp(dbt_timestamp, milliseconds=True) ), testCaseStatus=test_case_status, testResultValue=[ @@ -1071,13 +1122,20 @@ def add_dbt_test_result(self, dbt_test: dict): else None, test_case_name=manifest_node.name, ) - self.metadata.add_test_case_results( - test_results=test_case_result, - test_case_fqn=test_case_fqn, - ) + + logger.debug(f"Adding test case results to {test_case_fqn} ") + try: + self.metadata.add_test_case_results( + test_results=test_case_result, + test_case_fqn=test_case_fqn, + ) + except APIError as err: + if err.code != 409: + raise APIError(err) from err + except Exception as err: # pylint: disable=broad-except logger.debug(traceback.format_exc()) - logger.error( + logger.debug( f"Failed to capture tests results for node: {manifest_node.name} {err}" ) diff --git a/ingestion/src/metadata/ingestion/source/database/lineage_source.py b/ingestion/src/metadata/ingestion/source/database/lineage_source.py index 581467faf779..204d57b55077 100644 --- a/ingestion/src/metadata/ingestion/source/database/lineage_source.py +++ b/ingestion/src/metadata/ingestion/source/database/lineage_source.py @@ -13,11 +13,12 @@ """ import csv import os +import time import traceback from abc import ABC -from concurrent.futures import ThreadPoolExecutor, as_completed +from concurrent.futures import ThreadPoolExecutor from functools import partial -from typing import Callable, Iterable, Iterator, Union +from typing import Any, Callable, Iterable, Iterator, List, Union from metadata.generated.schema.api.data.createQuery import CreateQueryRequest from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest @@ -25,8 +26,12 @@ from metadata.generated.schema.type.tableQuery import TableQuery from metadata.ingestion.api.models import Either from metadata.ingestion.lineage.models import ConnectionTypeDialectMapper, Dialect -from metadata.ingestion.lineage.sql_lineage import get_lineage_by_query +from metadata.ingestion.lineage.sql_lineage import ( + get_lineage_by_graph, + get_lineage_by_query, +) from metadata.ingestion.models.ometa_lineage import OMetaLineageRequest +from metadata.ingestion.models.topology import Queue from metadata.ingestion.source.database.query_parser_source import QueryParserSource from metadata.ingestion.source.models import TableView from metadata.utils import fqn @@ -36,6 +41,9 @@ logger = ingestion_logger() +CHUNK_SIZE = 200 + + class LineageSource(QueryParserSource, ABC): """ This is the base source to handle Lineage-only ingestion. @@ -96,27 +104,57 @@ def get_table_query(self) -> Iterator[TableQuery]: ) yield from self.yield_table_query() - def generate_lineage_in_thread(self, producer_fn: Callable, processor_fn: Callable): - with ThreadPoolExecutor(max_workers=self.source_config.threads) as executor: - futures = [] + def generate_lineage_in_thread( + self, + producer_fn: Callable[[], Iterable[Any]], + processor_fn: Callable[[Any], Iterable[Any]], + chunk_size: int = CHUNK_SIZE, + ): + """ + Optimized multithreaded lineage generation with improved error handling and performance. - for produced_input in producer_fn(): - futures.append(executor.submit(processor_fn, produced_input)) + Args: + producer_fn: Function that yields input items + processor_fn: Function to process each input item + chunk_size: Optional batching to reduce thread creation overhead + """ - # Handle remaining futures after the loop - for future in as_completed( - futures, timeout=self.source_config.parsingTimeoutLimit - ): - try: - results = future.result( - timeout=self.source_config.parsingTimeoutLimit - ) - yield from results - except Exception as exc: - logger.debug(traceback.format_exc()) - logger.warning( - f"Error processing result for {produced_input}: {exc}" - ) + def chunk_generator(): + temp_chunk = [] + for chunk in producer_fn(): + temp_chunk.append(chunk) + if len(temp_chunk) >= chunk_size: + yield temp_chunk + temp_chunk = [] + + if temp_chunk: + yield temp_chunk + + thread_pool = ThreadPoolExecutor(max_workers=self.source_config.threads) + queue = Queue() + + futures = [ + thread_pool.submit( + processor_fn, + chunk, + queue, + ) + for chunk in chunk_generator() + ] + while True: + if queue.has_tasks(): + yield from queue.process() + + else: + if not futures: + break + + for i, future in enumerate(futures): + if future.done(): + future.result() + futures.pop(i) + + time.sleep(0.01) def yield_table_query(self) -> Iterator[TableQuery]: """ @@ -158,33 +196,45 @@ def _query_already_processed(self, table_query: TableQuery) -> bool: return fqn.get_query_checksum(table_query.query) in checksums or {} def query_lineage_generator( - self, table_query: TableQuery + self, table_queries: List[TableQuery], queue: Queue ) -> Iterable[Either[Union[AddLineageRequest, CreateQueryRequest]]]: - if not self._query_already_processed(table_query): - lineages: Iterable[Either[AddLineageRequest]] = get_lineage_by_query( - self.metadata, - query=table_query.query, - service_name=table_query.serviceName, - database_name=table_query.databaseName, - schema_name=table_query.databaseSchema, - dialect=self.dialect, - timeout_seconds=self.source_config.parsingTimeoutLimit, - ) + if self.graph is None and self.source_config.enableTempTableLineage: + import networkx as nx + + # Create a directed graph + self.graph = nx.DiGraph() + + for table_query in table_queries or []: + if not self._query_already_processed(table_query): + lineages: Iterable[Either[AddLineageRequest]] = get_lineage_by_query( + self.metadata, + query=table_query.query, + service_name=table_query.serviceName, + database_name=table_query.databaseName, + schema_name=table_query.databaseSchema, + dialect=self.dialect, + timeout_seconds=self.source_config.parsingTimeoutLimit, + graph=self.graph, + ) + + for lineage_request in lineages or []: + queue.put(lineage_request) - for lineage_request in lineages or []: - yield lineage_request - - # If we identified lineage properly, ingest the original query - if lineage_request.right: - yield Either( - right=CreateQueryRequest( - query=SqlQuery(table_query.query), - query_type=table_query.query_type, - duration=table_query.duration, - processedLineage=True, - service=FullyQualifiedEntityName(self.config.serviceName), + # If we identified lineage properly, ingest the original query + if lineage_request.right: + queue.put( + Either( + right=CreateQueryRequest( + query=SqlQuery(table_query.query), + query_type=table_query.query_type, + duration=table_query.duration, + processedLineage=True, + service=FullyQualifiedEntityName( + self.config.serviceName + ), + ) + ) ) - ) def yield_query_lineage( self, @@ -197,28 +247,33 @@ def yield_query_lineage( self.dialect = ConnectionTypeDialectMapper.dialect_of(connection_type) producer_fn = self.get_table_query processor_fn = self.query_lineage_generator - yield from self.generate_lineage_in_thread(producer_fn, processor_fn) + yield from self.generate_lineage_in_thread( + producer_fn, processor_fn, CHUNK_SIZE + ) def view_lineage_generator( - self, view: TableView + self, views: List[TableView], queue: Queue ) -> Iterable[Either[AddLineageRequest]]: try: - for lineage in get_view_lineage( - view=view, - metadata=self.metadata, - service_name=self.config.serviceName, - connection_type=self.service_connection.type.value, - timeout_seconds=self.source_config.parsingTimeoutLimit, - ): - if lineage.right is not None: - yield Either( - right=OMetaLineageRequest( - lineage_request=lineage.right, - override_lineage=self.source_config.overrideViewLineage, + for view in views: + for lineage in get_view_lineage( + view=view, + metadata=self.metadata, + service_name=self.config.serviceName, + connection_type=self.service_connection.type.value, + timeout_seconds=self.source_config.parsingTimeoutLimit, + ): + if lineage.right is not None: + queue.put( + Either( + right=OMetaLineageRequest( + lineage_request=lineage.right, + override_lineage=self.source_config.overrideViewLineage, + ) + ) ) - ) - else: - yield lineage + else: + queue.put(lineage) except Exception as exc: logger.debug(traceback.format_exc()) logger.warning(f"Error processing view {view}: {exc}") @@ -253,6 +308,7 @@ def _iter( if self.source_config.processQueryLineage: if hasattr(self.service_connection, "supportsLineageExtraction"): yield from self.yield_query_lineage() or [] + yield from get_lineage_by_graph(graph=self.graph) else: logger.warning( f"Lineage extraction is not supported for {str(self.service_connection.type.value)} connection" diff --git a/ingestion/src/metadata/ingestion/source/database/mariadb/lineage.py b/ingestion/src/metadata/ingestion/source/database/mariadb/lineage.py index afbfcc398f3b..074ce5b8c6a0 100644 --- a/ingestion/src/metadata/ingestion/source/database/mariadb/lineage.py +++ b/ingestion/src/metadata/ingestion/source/database/mariadb/lineage.py @@ -14,7 +14,7 @@ from typing import Optional from metadata.generated.schema.entity.services.connections.database.mariaDBConnection import ( - MariadbConnection, + MariaDBConnection, ) from metadata.generated.schema.metadataIngestion.workflow import ( Source as WorkflowSource, @@ -38,8 +38,8 @@ def create( ): """Create class instance""" config: WorkflowSource = WorkflowSource.model_validate(config_dict) - connection: MariadbConnection = config.serviceConnection.root.config - if not isinstance(connection, MariadbConnection): + connection: MariaDBConnection = config.serviceConnection.root.config + if not isinstance(connection, MariaDBConnection): raise InvalidSourceException( f"Expected MariadbConnection, but got {connection}" ) diff --git a/ingestion/src/metadata/ingestion/source/database/mssql/queries.py b/ingestion/src/metadata/ingestion/source/database/mssql/queries.py index d9bab948e080..0ed3f06da060 100644 --- a/ingestion/src/metadata/ingestion/source/database/mssql/queries.py +++ b/ingestion/src/metadata/ingestion/source/database/mssql/queries.py @@ -20,8 +20,8 @@ db.NAME database_name, t.text query_text, s.last_execution_time start_time, - DATEADD(s, s.total_elapsed_time/1000, s.last_execution_time) end_time, - s.total_elapsed_time/1000 duration, + DATEADD(s, s.last_elapsed_time/1000000, s.last_execution_time) end_time, + s.last_elapsed_time/1000000 duration, NULL schema_name, NULL query_type, NULL user_name, @@ -209,7 +209,7 @@ WITH SP_HISTORY (start_time, end_time, procedure_name, query_text) AS ( select s.last_execution_time start_time, - DATEADD(s, s.total_elapsed_time/1000, s.last_execution_time) end_time, + DATEADD(s, s.last_elapsed_time/1000000, s.last_execution_time) end_time, OBJECT_NAME(object_id, database_id) as procedure_name, text as query_text from sys.dm_exec_procedure_stats s @@ -222,8 +222,8 @@ db.NAME database_name, t.text query_text, s.last_execution_time start_time, - DATEADD(s, s.total_elapsed_time/1000, s.last_execution_time) end_time, - s.total_elapsed_time/1000 duration, + DATEADD(s, s.last_elapsed_time/1000000, s.last_execution_time) end_time, + s.last_elapsed_time/1000000 duration, case when t.text LIKE '%%MERGE%%' then 'MERGE' when t.text LIKE '%%UPDATE%%' then 'UPDATE' diff --git a/ingestion/src/metadata/ingestion/source/database/postgres/metadata.py b/ingestion/src/metadata/ingestion/source/database/postgres/metadata.py index 845277a088c9..42a37bb49ebc 100644 --- a/ingestion/src/metadata/ingestion/source/database/postgres/metadata.py +++ b/ingestion/src/metadata/ingestion/source/database/postgres/metadata.py @@ -67,7 +67,6 @@ get_columns, get_etable_owner, get_foreign_keys, - get_json_fields_and_type, get_table_comment, get_table_owner, get_view_definition, @@ -147,7 +146,6 @@ Inspector.get_all_table_ddls = get_all_table_ddls Inspector.get_table_ddl = get_table_ddl Inspector.get_table_owner = get_etable_owner -Inspector.get_json_fields_and_type = get_json_fields_and_type PGDialect.get_foreign_keys = get_foreign_keys diff --git a/ingestion/src/metadata/ingestion/source/database/postgres/queries.py b/ingestion/src/metadata/ingestion/source/database/postgres/queries.py index e6cf7df513cf..03a66ede0a31 100644 --- a/ingestion/src/metadata/ingestion/source/database/postgres/queries.py +++ b/ingestion/src/metadata/ingestion/source/database/postgres/queries.py @@ -193,7 +193,7 @@ """ POSTGRES_GET_SERVER_VERSION = """ -show server_version +show server_version_num """ POSTGRES_FETCH_FK = """ @@ -212,91 +212,6 @@ ORDER BY 1 """ -POSTGRES_GET_JSON_FIELDS = """ - WITH RECURSIVE json_hierarchy AS ( - SELECT - key AS path, - json_typeof(value) AS type, - value, - json_build_object() AS properties, - key AS title - FROM - {table_name} tbd, - LATERAL json_each({column_name}::json) - ), - build_hierarchy AS ( - SELECT - path, - type, - title, - CASE - WHEN type = 'object' THEN - json_build_object( - 'title', title, - 'type', 'object', - 'properties', ( - SELECT json_object_agg( - key, - json_build_object( - 'title', key, - 'type', json_typeof(value), - 'properties', ( - CASE - WHEN json_typeof(value) = 'object' THEN - ( - SELECT json_object_agg( - key, - json_build_object( - 'title', key, - 'type', json_typeof(value), - 'properties', - json_build_object() - ) - ) - FROM json_each(value::json) AS sub_key_value - ) - ELSE json_build_object() - END - ) - ) - ) - FROM json_each(value::json) AS key_value - ) - ) - WHEN type = 'array' THEN - json_build_object( - 'title', title, - 'type', 'array', - 'properties', json_build_object() - ) - ELSE - json_build_object( - 'title', title, - 'type', type - ) - END AS hierarchy - FROM - json_hierarchy - ), - aggregate_hierarchy AS ( - select - json_build_object( - 'title','{column_name}', - 'type','object', - 'properties', - json_object_agg( - path, - hierarchy - )) AS result - FROM - build_hierarchy - ) - SELECT - result - FROM - aggregate_hierarchy; -""" - POSTGRES_GET_STORED_PROCEDURES = """ SELECT proname AS procedure_name, nspname AS schema_name, diff --git a/ingestion/src/metadata/ingestion/source/database/postgres/usage.py b/ingestion/src/metadata/ingestion/source/database/postgres/usage.py index 579590b4ec59..522a89a9e2ee 100644 --- a/ingestion/src/metadata/ingestion/source/database/postgres/usage.py +++ b/ingestion/src/metadata/ingestion/source/database/postgres/usage.py @@ -15,6 +15,11 @@ from datetime import datetime from typing import Iterable +from sqlalchemy.exc import OperationalError + +from metadata.generated.schema.entity.services.ingestionPipelines.status import ( + StackTraceError, +) from metadata.generated.schema.type.basic import DateTime from metadata.generated.schema.type.tableQuery import TableQueries, TableQuery from metadata.ingestion.source.connections import get_connection @@ -67,6 +72,16 @@ def process_table_query(self) -> Iterable[TableQueries]: logger.error(str(err)) if queries: yield TableQueries(queries=queries) + + except OperationalError as err: + self.status.failed( + StackTraceError( + name="Usage", + error=f"Source Usage failed due to - {err}", + stackTrace=traceback.format_exc(), + ) + ) + except Exception as err: if query: logger.debug( diff --git a/ingestion/src/metadata/ingestion/source/database/postgres/utils.py b/ingestion/src/metadata/ingestion/source/database/postgres/utils.py index 246519c00e26..6222dfa493ae 100644 --- a/ingestion/src/metadata/ingestion/source/database/postgres/utils.py +++ b/ingestion/src/metadata/ingestion/source/database/postgres/utils.py @@ -13,7 +13,6 @@ """ Postgres SQLAlchemy util methods """ -import json import re import traceback from typing import Dict, Optional, Tuple @@ -24,18 +23,15 @@ from sqlalchemy.engine import reflection from sqlalchemy.sql import sqltypes -from metadata.generated.schema.entity.data.table import Column from metadata.ingestion.source.database.postgres.queries import ( POSTGRES_COL_IDENTITY, POSTGRES_FETCH_FK, - POSTGRES_GET_JSON_FIELDS, POSTGRES_GET_SERVER_VERSION, POSTGRES_SQL_COLUMNS, POSTGRES_TABLE_COMMENTS, POSTGRES_TABLE_OWNERS, POSTGRES_VIEW_DEFINITIONS, ) -from metadata.parsers.json_schema_parser import parse_json_schema from metadata.utils.logger import utils_logger from metadata.utils.sqlalchemy_utils import ( get_table_comment_wrapper, @@ -45,7 +41,7 @@ logger = utils_logger() -OLD_POSTGRES_VERSION = "13.0" +OLD_POSTGRES_VERSION = "130000" def get_etable_owner( @@ -190,28 +186,6 @@ def get_table_comment( ) -@reflection.cache -def get_json_fields_and_type( - self, table_name, column_name, schema=None, **kw -): # pylint: disable=unused-argument - try: - query = POSTGRES_GET_JSON_FIELDS.format( - table_name=table_name, column_name=column_name - ) - cursor = self.engine.execute(query) - result = cursor.fetchone() - if result: - parsed_column = parse_json_schema(json.dumps(result[0]), Column) - if parsed_column: - return parsed_column[0].children - except Exception as err: - logger.warning( - f"Unable to parse the json fields for {table_name}.{column_name} - {err}" - ) - logger.debug(traceback.format_exc()) - return None - - @reflection.cache def get_columns( # pylint: disable=too-many-locals self, connection, table_name, schema=None, **kw @@ -523,9 +497,6 @@ def get_postgres_version(engine) -> Optional[str]: results = engine.execute(POSTGRES_GET_SERVER_VERSION) for res in results: version_string = str(res[0]) - opening_parenthesis_index = version_string.find("(") - if opening_parenthesis_index != -1: - return version_string[:opening_parenthesis_index].strip() return version_string except Exception as err: logger.warning(f"Unable to fetch the Postgres Version - {err}") diff --git a/ingestion/src/metadata/ingestion/source/database/query_parser_source.py b/ingestion/src/metadata/ingestion/source/database/query_parser_source.py index 2c4477a6969a..c897f2877e2a 100644 --- a/ingestion/src/metadata/ingestion/source/database/query_parser_source.py +++ b/ingestion/src/metadata/ingestion/source/database/query_parser_source.py @@ -67,6 +67,7 @@ def __init__( self.engine = ( get_ssl_connection(self.service_connection) if get_engine else None ) + self.graph = None @property def name(self) -> str: diff --git a/ingestion/src/metadata/ingestion/source/database/redshift/metadata.py b/ingestion/src/metadata/ingestion/source/database/redshift/metadata.py index d1c87bd07289..244d96acce2a 100644 --- a/ingestion/src/metadata/ingestion/source/database/redshift/metadata.py +++ b/ingestion/src/metadata/ingestion/source/database/redshift/metadata.py @@ -11,7 +11,6 @@ """ Redshift source ingestion """ - import re import traceback from typing import Iterable, List, Optional, Tuple @@ -57,6 +56,9 @@ CommonDbSourceService, TableNameAndType, ) +from metadata.ingestion.source.database.external_table_lineage_mixin import ( + ExternalTableLineageMixin, +) from metadata.ingestion.source.database.incremental_metadata_extraction import ( IncrementalConfig, ) @@ -69,6 +71,7 @@ ) from metadata.ingestion.source.database.redshift.models import RedshiftStoredProcedure from metadata.ingestion.source.database.redshift.queries import ( + REDSHIFT_EXTERNAL_TABLE_LOCATION, REDSHIFT_GET_ALL_RELATION_INFO, REDSHIFT_GET_DATABASE_NAMES, REDSHIFT_GET_STORED_PROCEDURES, @@ -121,12 +124,13 @@ RedshiftDialect._get_all_relation_info = ( # pylint: disable=protected-access _get_all_relation_info ) - Inspector.get_all_table_ddls = get_all_table_ddls Inspector.get_table_ddl = get_table_ddl -class RedshiftSource(LifeCycleQueryMixin, CommonDbSourceService, MultiDBSource): +class RedshiftSource( + ExternalTableLineageMixin, LifeCycleQueryMixin, CommonDbSourceService, MultiDBSource +): """ Implements the necessary methods to extract Database metadata from Redshift Source @@ -146,6 +150,7 @@ def __init__( self.incremental_table_processor: Optional[ RedshiftIncrementalTableProcessor ] = None + self.external_location_map = {} if self.incremental.enabled: logger.info( @@ -168,6 +173,14 @@ def create( ) return cls(config, metadata, incremental_config) + def get_location_path(self, table_name: str, schema_name: str) -> Optional[str]: + """ + Method to fetch the location path of the table + """ + return self.external_location_map.get( + (self.context.get().database, schema_name, table_name) + ) + def get_partition_details(self) -> None: """ Populate partition details @@ -275,15 +288,23 @@ def _set_incremental_table_processor(self, database: str): for schema_name, table_name in self.incremental_table_processor.get_deleted() ) + def set_external_location_map(self, database_name: str) -> None: + self.external_location_map.clear() + results = self.engine.execute( + REDSHIFT_EXTERNAL_TABLE_LOCATION.format(database_name=database_name) + ).all() + self.external_location_map = { + (database_name, row.schemaname, row.tablename): row.location + for row in results + } + def get_database_names(self) -> Iterable[str]: if not self.config.serviceConnection.root.config.ingestAllDatabases: + configured_db = self.config.serviceConnection.root.config.database self.get_partition_details() - - self._set_incremental_table_processor( - self.config.serviceConnection.root.config.database - ) - - yield self.config.serviceConnection.root.config.database + self._set_incremental_table_processor(configured_db) + self.set_external_location_map(configured_db) + yield configured_db else: for new_database in self.get_database_names_raw(): database_fqn = fqn.build( @@ -307,9 +328,8 @@ def get_database_names(self) -> Iterable[str]: try: self.set_inspector(database_name=new_database) self.get_partition_details() - self._set_incremental_table_processor(new_database) - + self.set_external_location_map(new_database) yield new_database except Exception as exc: logger.debug(traceback.format_exc()) diff --git a/ingestion/src/metadata/ingestion/source/database/redshift/queries.py b/ingestion/src/metadata/ingestion/source/database/redshift/queries.py index 50ef72085cf3..114ba4e33dd5 100644 --- a/ingestion/src/metadata/ingestion/source/database/redshift/queries.py +++ b/ingestion/src/metadata/ingestion/source/database/redshift/queries.py @@ -203,6 +203,11 @@ """ ) +REDSHIFT_EXTERNAL_TABLE_LOCATION = """ + SELECT schemaname, tablename, location + FROM svv_external_tables + where redshift_database_name='{database_name}' +""" REDSHIFT_PARTITION_DETAILS = """ select "schema", "table", diststyle @@ -328,7 +333,7 @@ pid as query_session_id, starttime as query_start_time, endtime as query_end_time, - b.usename as query_user_name + cast(b.usename as varchar) as query_user_name from STL_QUERY q join pg_catalog.pg_user b on b.usesysid = q.userid diff --git a/ingestion/src/metadata/ingestion/source/database/salesforce/metadata.py b/ingestion/src/metadata/ingestion/source/database/salesforce/metadata.py index 0e68053ab90c..e4c46c7532a5 100644 --- a/ingestion/src/metadata/ingestion/source/database/salesforce/metadata.py +++ b/ingestion/src/metadata/ingestion/source/database/salesforce/metadata.py @@ -12,7 +12,7 @@ Salesforce source ingestion """ import traceback -from typing import Any, Iterable, Optional, Tuple +from typing import Any, Iterable, List, Optional, Tuple from metadata.generated.schema.api.data.createDatabase import CreateDatabaseRequest from metadata.generated.schema.api.data.createDatabaseSchema import ( @@ -231,6 +231,29 @@ def get_table_description( ) return table_description if table_description else object_label + def get_table_column_description(self, table_name: str) -> Optional[List]: + """ + Method to get the all columns' (field) description for Salesforce with the Tooling API. + """ + all_column_description = None + try: + result = self.client.toolingexecute( + f"query/?q=SELECT+Description+FROM+FieldDefinition+WHERE+" + f"EntityDefinition.QualifiedApiName='{table_name}'" + ) + all_column_description = result["records"] + except KeyError as err: + logger.warning( + "Unable to get required key from Tooling API response for " + f"table [{table_name}]: {err}" + ) + except Exception as exc: + logger.debug(traceback.format_exc()) + logger.warning( + f"Unable to get column description with Tooling API for table [{table_name}]: {exc}" + ) + return all_column_description + def yield_table( self, table_name_and_type: Tuple[str, TableType] ) -> Iterable[Either[CreateTableRequest]]: @@ -245,7 +268,7 @@ def yield_table( f"sobjects/{table_name}/describe/", params=None, ) - columns = self.get_columns(salesforce_objects.get("fields", [])) + columns = self.get_columns(table_name, salesforce_objects.get("fields", [])) table_request = CreateTableRequest( name=EntityName(table_name), tableType=table_type, @@ -278,12 +301,26 @@ def yield_table( ) ) - def get_columns(self, salesforce_fields): + def get_columns(self, table_name: str, salesforce_fields: List): """ Method to handle column details """ row_order = 1 columns = [] + column_description_mapping = {} + all_column_description = self.get_table_column_description(table_name) + if all_column_description: + for item in all_column_description: + try: + if item.get("Description") is not None: + column_name = item["attributes"]["url"].split(".")[-1] + column_description_mapping.update( + {column_name: item["Description"]} + ) + except Exception as ex: + logger.debug( + f"Error creating column description mapping: {str(ex)}" + ) for column in salesforce_fields: col_constraint = None if column["nillable"]: @@ -292,11 +329,15 @@ def get_columns(self, salesforce_fields): col_constraint = Constraint.NOT_NULL if column["unique"]: col_constraint = Constraint.UNIQUE + if column_description_mapping.get(column["name"]): + column_description = column_description_mapping[column["name"]] + else: + column_description = column["label"] columns.append( Column( name=column["name"], - description=column["label"], + description=column_description, dataType=self.column_type(column["type"].upper()), dataTypeDisplay=column["type"], constraint=col_constraint, diff --git a/ingestion/src/metadata/ingestion/source/database/sample_data.py b/ingestion/src/metadata/ingestion/source/database/sample_data.py index f809f045206b..69cdb59897e0 100644 --- a/ingestion/src/metadata/ingestion/source/database/sample_data.py +++ b/ingestion/src/metadata/ingestion/source/database/sample_data.py @@ -1548,9 +1548,7 @@ def ingest_test_suite(self) -> Iterable[Either[OMetaTestSuiteSample]]: test_suite=CreateTestSuiteRequest( name=test_suite["testSuiteName"], description=test_suite["testSuiteDescription"], - executableEntityReference=test_suite[ - "executableEntityReference" - ], + basicEntityReference=test_suite["executableEntityReference"], ) ) ) diff --git a/ingestion/src/metadata/ingestion/source/database/snowflake/connection.py b/ingestion/src/metadata/ingestion/source/database/snowflake/connection.py index e32479fe3987..6c98a201b319 100644 --- a/ingestion/src/metadata/ingestion/source/database/snowflake/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/snowflake/connection.py @@ -193,10 +193,18 @@ def test_connection( engine_wrapper=engine_wrapper, ), "GetQueries": partial( - test_query, statement=SNOWFLAKE_TEST_GET_QUERIES, engine=engine + test_query, + statement=SNOWFLAKE_TEST_GET_QUERIES.format( + account_usage=service_connection.accountUsageSchema + ), + engine=engine, ), "GetTags": partial( - test_query, statement=SNOWFLAKE_TEST_FETCH_TAG, engine=engine + test_query, + statement=SNOWFLAKE_TEST_FETCH_TAG.format( + account_usage=service_connection.accountUsageSchema + ), + engine=engine, ), } diff --git a/ingestion/src/metadata/ingestion/source/database/snowflake/lineage.py b/ingestion/src/metadata/ingestion/source/database/snowflake/lineage.py index 2d85ed8772e2..1337635fe14f 100644 --- a/ingestion/src/metadata/ingestion/source/database/snowflake/lineage.py +++ b/ingestion/src/metadata/ingestion/source/database/snowflake/lineage.py @@ -47,6 +47,8 @@ class SnowflakeLineageSource( AND ( QUERY_TYPE IN ('MERGE', 'UPDATE','CREATE_TABLE_AS_SELECT') OR (QUERY_TYPE = 'INSERT' and query_text ILIKE '%%insert%%into%%select%%') + OR (QUERY_TYPE = 'ALTER' and query_text ILIKE '%%alter%%table%%swap%%') + OR (QUERY_TYPE = 'CREATE_TABLE' and query_text ILIKE '%%clone%%') ) """ @@ -60,6 +62,7 @@ def get_stored_procedure_queries_dict(self) -> Dict[str, List[QueryByProcedure]] start, _ = get_start_and_end(self.source_config.queryLogDuration) query = self.stored_procedure_query.format( start_date=start, + account_usage=self.service_connection.accountUsageSchema, ) queries_dict = self.procedure_queries_dict( query=query, diff --git a/ingestion/src/metadata/ingestion/source/database/snowflake/metadata.py b/ingestion/src/metadata/ingestion/source/database/snowflake/metadata.py index da5a7034a26d..f6f026d39137 100644 --- a/ingestion/src/metadata/ingestion/source/database/snowflake/metadata.py +++ b/ingestion/src/metadata/ingestion/source/database/snowflake/metadata.py @@ -418,6 +418,7 @@ def yield_tag( SNOWFLAKE_FETCH_ALL_TAGS.format( database_name=self.context.get().database, schema_name=schema_name, + account_usage=self.service_connection.accountUsageSchema, ) ) @@ -431,6 +432,7 @@ def yield_tag( SNOWFLAKE_FETCH_ALL_TAGS.format( database_name=f'"{self.context.get().database}"', schema_name=f'"{self.context.get().database_schema}"', + account_usage=self.service_connection.accountUsageSchema, ) ) except Exception as inner_exc: @@ -635,6 +637,7 @@ def _get_stored_procedures_internal( query.format( database_name=self.context.get().database, schema_name=self.context.get().database_schema, + account_usage=self.service_connection.accountUsageSchema, ) ).all() for row in results: diff --git a/ingestion/src/metadata/ingestion/source/database/snowflake/queries.py b/ingestion/src/metadata/ingestion/source/database/snowflake/queries.py index 671e2ebfa8a4..55b2bf909c4a 100644 --- a/ingestion/src/metadata/ingestion/source/database/snowflake/queries.py +++ b/ingestion/src/metadata/ingestion/source/database/snowflake/queries.py @@ -25,7 +25,7 @@ start_time "start_time", end_time "end_time", total_elapsed_time "duration" - from snowflake.account_usage.query_history + from {account_usage}.query_history WHERE query_text NOT LIKE '/* {{"app": "OpenMetadata", %%}} */%%' AND query_text NOT LIKE '/* {{"app": "dbt", %%}} */%%' AND start_time between to_timestamp_ltz('{start_time}') and to_timestamp_ltz('{end_time}') @@ -39,7 +39,7 @@ SNOWFLAKE_FETCH_ALL_TAGS = textwrap.dedent( """ select TAG_NAME, TAG_VALUE, OBJECT_DATABASE, OBJECT_SCHEMA, OBJECT_NAME, COLUMN_NAME - from snowflake.account_usage.tag_references + from {account_usage}.tag_references where OBJECT_DATABASE = '{database_name}' and OBJECT_SCHEMA = '{schema_name}' """ @@ -234,11 +234,11 @@ """ SNOWFLAKE_TEST_FETCH_TAG = """ -select TAG_NAME from snowflake.account_usage.tag_references limit 1 +select TAG_NAME from {account_usage}.tag_references limit 1 """ SNOWFLAKE_TEST_GET_QUERIES = """ -SELECT query_text from snowflake.account_usage.query_history limit 1 +SELECT query_text from {account_usage}.query_history limit 1 """ SNOWFLAKE_TEST_GET_TABLES = """ @@ -296,9 +296,10 @@ ARGUMENT_SIGNATURE AS signature, COMMENT as comment, 'StoredProcedure' as procedure_type -FROM INFORMATION_SCHEMA.PROCEDURES +FROM {account_usage}.PROCEDURES WHERE PROCEDURE_CATALOG = '{database_name}' AND PROCEDURE_SCHEMA = '{schema_name}' + AND DELETED IS NULL """ ) @@ -312,9 +313,10 @@ ARGUMENT_SIGNATURE AS signature, COMMENT as comment, 'UDF' as procedure_type -FROM INFORMATION_SCHEMA.FUNCTIONS +FROM {account_usage}.FUNCTIONS WHERE FUNCTION_CATALOG = '{database_name}' AND FUNCTION_SCHEMA = '{schema_name}' + AND DELETED IS NULL """ ) @@ -334,9 +336,11 @@ SESSION_ID, START_TIME, END_TIME - FROM SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY SP + FROM {account_usage}.QUERY_HISTORY SP WHERE QUERY_TYPE = 'CALL' AND START_TIME >= '{start_date}' + AND QUERY_TEXT <> '' + AND QUERY_TEXT IS NOT NULL ), Q_HISTORY AS ( SELECT @@ -349,11 +353,15 @@ USER_NAME, SCHEMA_NAME, DATABASE_NAME - FROM SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY SP + FROM {account_usage}.QUERY_HISTORY SP WHERE QUERY_TYPE <> 'CALL' AND QUERY_TEXT NOT LIKE '/* {{"app": "OpenMetadata", %%}} */%%' AND QUERY_TEXT NOT LIKE '/* {{"app": "dbt", %%}} */%%' AND START_TIME >= '{start_date}' + AND ( + QUERY_TYPE IN ('MERGE', 'UPDATE','CREATE_TABLE_AS_SELECT') + OR (QUERY_TYPE = 'INSERT' and query_text ILIKE '%%insert%%into%%select%%') + ) ) SELECT Q.QUERY_TYPE AS QUERY_TYPE, diff --git a/ingestion/src/metadata/ingestion/source/database/snowflake/query_parser.py b/ingestion/src/metadata/ingestion/source/database/snowflake/query_parser.py index bbc528fc4c44..363495bd0f87 100644 --- a/ingestion/src/metadata/ingestion/source/database/snowflake/query_parser.py +++ b/ingestion/src/metadata/ingestion/source/database/snowflake/query_parser.py @@ -60,6 +60,7 @@ def get_sql_statement(self, start_time: datetime, end_time: datetime) -> str: end_time=end_time, result_limit=self.config.sourceConfig.config.resultLimit, filters=self.get_filters(), + account_usage=self.service_connection.accountUsageSchema, ) def check_life_cycle_query( diff --git a/ingestion/src/metadata/ingestion/source/database/sql_column_handler.py b/ingestion/src/metadata/ingestion/source/database/sql_column_handler.py index 0f1ef012f9a3..9b65620d3402 100644 --- a/ingestion/src/metadata/ingestion/source/database/sql_column_handler.py +++ b/ingestion/src/metadata/ingestion/source/database/sql_column_handler.py @@ -94,6 +94,12 @@ def _process_col_type(self, column: dict, schema: str) -> Tuple: parsed_string["name"] = column["name"] else: col_type = ColumnTypeParser.get_column_type(column["type"]) + # For arrays, we'll get the item type if possible, or parse the string representation of the column + # if SQLAlchemy does not provide any further information + if col_type == "ARRAY" and getattr(column["type"], "item_type"): + arr_data_type = ColumnTypeParser.get_column_type( + column["type"].item_type + ) if col_type == "ARRAY" and re.match( r"(?:\w*)(?:\()(\w*)(?:.*)", str(column["type"]) ): @@ -196,26 +202,6 @@ def _process_complex_col_type(self, parsed_string: dict, column: dict) -> Column ] return Column(**parsed_string) - @calculate_execution_time() - def process_json_type_column_fields( # pylint: disable=too-many-locals - self, schema_name: str, table_name: str, column_name: str, inspector: Inspector - ) -> Optional[List[Column]]: - """ - Parse fields column with json data types - """ - try: - if hasattr(inspector, "get_json_fields_and_type"): - result = inspector.get_json_fields_and_type( - table_name, column_name, schema_name - ) - return result - - except NotImplementedError: - logger.debug( - "Cannot parse json fields for table column [{schema_name}.{table_name}.{col_name}]: NotImplementedError" - ) - return None - @calculate_execution_time() def get_columns_and_constraints( # pylint: disable=too-many-locals self, schema_name: str, table_name: str, db_name: str, inspector: Inspector @@ -240,12 +226,16 @@ def get_columns_and_constraints( # pylint: disable=too-many-locals if len(col) == 1: column_level_unique_constraints.add(col[0]) else: - table_constraints.append( - TableConstraint( - constraintType=ConstraintType.UNIQUE, - columns=col, + if not any( + tc.constraintType == ConstraintType.UNIQUE and tc.columns == col + for tc in table_constraints + ): + table_constraints.append( + TableConstraint( + constraintType=ConstraintType.UNIQUE, + columns=col, + ) ) - ) if len(pk_columns) > 1: table_constraints.append( TableConstraint( @@ -292,11 +282,6 @@ def get_columns_and_constraints( # pylint: disable=too-many-locals ) col_data_length = 1 if col_data_length is None else col_data_length - if col_type == "JSON": - children = self.process_json_type_column_fields( - schema_name, table_name, column.get("name"), inspector - ) - om_column = Column( name=ColumnName( root=column["name"] diff --git a/ingestion/src/metadata/ingestion/source/database/stored_procedures_mixin.py b/ingestion/src/metadata/ingestion/source/database/stored_procedures_mixin.py index 5473f8b17009..7dba89daefd6 100644 --- a/ingestion/src/metadata/ingestion/source/database/stored_procedures_mixin.py +++ b/ingestion/src/metadata/ingestion/source/database/stored_procedures_mixin.py @@ -38,10 +38,11 @@ from metadata.ingestion.api.status import Status from metadata.ingestion.lineage.models import ConnectionTypeDialectMapper from metadata.ingestion.lineage.sql_lineage import get_lineage_by_query +from metadata.ingestion.models.topology import Queue from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.utils.logger import ingestion_logger from metadata.utils.stored_procedures import get_procedure_name_from_call -from metadata.utils.time_utils import convert_timestamp_to_milliseconds +from metadata.utils.time_utils import datetime_to_timestamp logger = ingestion_logger() @@ -176,8 +177,6 @@ def _yield_procedure_lineage( timeout_seconds=self.source_config.parsingTimeoutLimit, lineage_source=LineageSource.QueryLineage, ): - print("&& " * 100) - print(either_lineage) if ( either_lineage.left is None and either_lineage.right.edge.lineageDetails @@ -200,8 +199,8 @@ def yield_procedure_query( query_type=query_by_procedure.query_type, duration=query_by_procedure.query_duration, queryDate=Timestamp( - root=convert_timestamp_to_milliseconds( - int(query_by_procedure.query_start_time.timestamp()) + root=datetime_to_timestamp( + query_by_procedure.query_start_time, True ) ), triggeredBy=EntityReference( @@ -214,37 +213,49 @@ def yield_procedure_query( ) def procedure_lineage_processor( - self, procedure_and_query: ProcedureAndQuery + self, procedure_and_queries: List[ProcedureAndQuery], queue: Queue ) -> Iterable[Either[Union[AddLineageRequest, CreateQueryRequest]]]: - - try: - yield from self._yield_procedure_lineage( - query_by_procedure=procedure_and_query.query_by_procedure, - procedure=procedure_and_query.procedure, - ) - except Exception as exc: - logger.debug(traceback.format_exc()) - logger.warning( - f"Could not get lineage for store procedure '{procedure_and_query.procedure.fullyQualifiedName}' due to [{exc}]." - ) - try: - yield from self.yield_procedure_query( - query_by_procedure=procedure_and_query.query_by_procedure, - procedure=procedure_and_query.procedure, - ) - except Exception as exc: - logger.debug(traceback.format_exc()) - logger.warning( - f"Could not get query for store procedure '{procedure_and_query.procedure.fullyQualifiedName}' due to [{exc}]." - ) + for procedure_and_query in procedure_and_queries: + try: + for lineage in self._yield_procedure_lineage( + query_by_procedure=procedure_and_query.query_by_procedure, + procedure=procedure_and_query.procedure, + ): + queue.put(lineage) + except Exception as exc: + logger.debug(traceback.format_exc()) + logger.warning( + f"Could not get lineage for store procedure '{procedure_and_query.procedure.fullyQualifiedName}' due to [{exc}]." + ) + try: + for lineage in self.yield_procedure_query( + query_by_procedure=procedure_and_query.query_by_procedure, + procedure=procedure_and_query.procedure, + ): + queue.put(lineage) + except Exception as exc: + logger.debug(traceback.format_exc()) + logger.warning( + f"Could not get query for store procedure '{procedure_and_query.procedure.fullyQualifiedName}' due to [{exc}]." + ) def procedure_lineage_generator(self) -> Iterable[ProcedureAndQuery]: query = { "query": { "bool": { "must": [ - {"term": {"service.name.keyword": self.service_name}}, - {"term": {"deleted": False}}, + { + "bool": { + "should": [ + { + "term": { + "service.name.keyword": self.service_name + } + } + ] + } + }, + {"bool": {"should": [{"term": {"deleted": False}}]}}, ] } } @@ -256,7 +267,9 @@ def procedure_lineage_generator(self) -> Iterable[ProcedureAndQuery]: queries_dict = self.get_stored_procedure_queries_dict() # Then for each procedure, iterate over all its queries for procedure in ( - self.metadata.paginate_es(entity=StoredProcedure, query_filter=query_filter) + self.metadata.paginate_es( + entity=StoredProcedure, query_filter=query_filter, size=10 + ) or [] ): if procedure: diff --git a/ingestion/src/metadata/ingestion/source/database/trino/connection.py b/ingestion/src/metadata/ingestion/source/database/trino/connection.py index 6dbae4ac9c39..a87dd792b6bd 100644 --- a/ingestion/src/metadata/ingestion/source/database/trino/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/trino/connection.py @@ -39,6 +39,7 @@ create_generic_db_connection, get_connection_args_common, init_empty_connection_arguments, + init_empty_connection_options, ) from metadata.ingestion.connections.secrets import connection_with_options_secrets from metadata.ingestion.connections.test_connections import ( @@ -135,6 +136,10 @@ def get_connection(connection: TrinoConnection) -> Engine: # here we are creating a copy of connection, because we need to dynamically # add auth params to connectionArguments, which we do no intend to store # in original connection object and in OpenMetadata database + from trino.sqlalchemy.dialect import TrinoDialect + + TrinoDialect.is_disconnect = _is_disconnect + connection_copy = deepcopy(connection) if connection_copy.verify: connection_copy.connectionArguments = ( @@ -183,3 +188,11 @@ def test_connection( queries=queries, timeout_seconds=timeout_seconds, ) + + +# pylint: disable=unused-argument +def _is_disconnect(self, e, connection, cursor): + """is_disconnect method for the Databricks dialect""" + if "JWT expired" in str(e): + return True + return False diff --git a/ingestion/src/metadata/ingestion/source/database/unitycatalog/lineage.py b/ingestion/src/metadata/ingestion/source/database/unitycatalog/lineage.py index 3189102eec3e..7ba075113605 100644 --- a/ingestion/src/metadata/ingestion/source/database/unitycatalog/lineage.py +++ b/ingestion/src/metadata/ingestion/source/database/unitycatalog/lineage.py @@ -11,6 +11,7 @@ """ Databricks Unity Catalog Lineage Source Module """ +import traceback from typing import Iterable, Optional from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest @@ -27,6 +28,7 @@ EntitiesEdge, LineageDetails, ) +from metadata.generated.schema.type.entityLineage import Source as LineageSource from metadata.generated.schema.type.entityReference import EntityReference from metadata.ingestion.api.models import Either from metadata.ingestion.api.steps import InvalidSourceException, Source @@ -109,9 +111,59 @@ def _get_lineage_details( ) ) if col_lineage: - return LineageDetails(columnsLineage=col_lineage) + return LineageDetails( + columnsLineage=col_lineage, source=LineageSource.QueryLineage + ) return None + def _handle_upstream_table( + self, + table_streams: LineageTableStreams, + table: Table, + databricks_table_fqn: str, + ) -> Iterable[Either[AddLineageRequest]]: + for upstream_table in table_streams.upstream_tables: + try: + if not upstream_table.name: + continue + from_entity_fqn = fqn.build( + metadata=self.metadata, + entity_type=Table, + database_name=upstream_table.catalog_name, + schema_name=upstream_table.schema_name, + table_name=upstream_table.name, + service_name=self.config.serviceName, + ) + + from_entity = self.metadata.get_by_name( + entity=Table, fqn=from_entity_fqn + ) + if from_entity: + lineage_details = self._get_lineage_details( + from_table=from_entity, + to_table=table, + databricks_table_fqn=databricks_table_fqn, + ) + yield Either( + left=None, + right=AddLineageRequest( + edge=EntitiesEdge( + toEntity=EntityReference(id=table.id, type="table"), + fromEntity=EntityReference( + id=from_entity.id, type="table" + ), + lineageDetails=lineage_details, + ) + ), + ) + except Exception: + logger.debug( + "Error while processing lineage for " + f"{upstream_table.catalog_name}.{upstream_table.schema_name}.{upstream_table.name}" + f" -> {databricks_table_fqn}" + ) + logger.debug(traceback.format_exc()) + def _iter(self, *_, **__) -> Iterable[Either[AddLineageRequest]]: """ Based on the query logs, prepare the lineage @@ -128,37 +180,9 @@ def _iter(self, *_, **__) -> Iterable[Either[AddLineageRequest]]: table_streams: LineageTableStreams = self.client.get_table_lineage( databricks_table_fqn ) - for upstream_table in table_streams.upstream_tables: - from_entity_fqn = fqn.build( - metadata=self.metadata, - entity_type=Table, - database_name=upstream_table.catalog_name, - schema_name=upstream_table.schema_name, - table_name=upstream_table.name, - service_name=self.config.serviceName, - ) - - from_entity = self.metadata.get_by_name( - entity=Table, fqn=from_entity_fqn - ) - if from_entity: - lineage_details = self._get_lineage_details( - from_table=from_entity, - to_table=table, - databricks_table_fqn=databricks_table_fqn, - ) - yield Either( - left=None, - right=AddLineageRequest( - edge=EntitiesEdge( - toEntity=EntityReference(id=table.id, type="table"), - fromEntity=EntityReference( - id=from_entity.id, type="table" - ), - lineageDetails=lineage_details, - ) - ), - ) + yield from self._handle_upstream_table( + table_streams, table, databricks_table_fqn + ) def test_connection(self) -> None: test_connection_fn = get_test_connection_fn(self.service_connection) diff --git a/ingestion/src/metadata/ingestion/source/database/unitycatalog/query_parser.py b/ingestion/src/metadata/ingestion/source/database/unitycatalog/query_parser.py index 5a6b7933a28d..f2ac03b99f98 100644 --- a/ingestion/src/metadata/ingestion/source/database/unitycatalog/query_parser.py +++ b/ingestion/src/metadata/ingestion/source/database/unitycatalog/query_parser.py @@ -44,6 +44,13 @@ class UnityCatalogQueryParserSource( filters: str + def _init_super( + self, + config: WorkflowSource, + metadata: OpenMetadata, + ): + super().__init__(config, metadata, False) + # pylint: disable=super-init-not-called def __init__(self, config: WorkflowSource, metadata: OpenMetadata): self._init_super(config=config, metadata=metadata) diff --git a/ingestion/src/metadata/ingestion/source/database/unitycatalog/service_spec.py b/ingestion/src/metadata/ingestion/source/database/unitycatalog/service_spec.py index ca9f17b254e5..892f88ef7b74 100644 --- a/ingestion/src/metadata/ingestion/source/database/unitycatalog/service_spec.py +++ b/ingestion/src/metadata/ingestion/source/database/unitycatalog/service_spec.py @@ -11,6 +11,9 @@ from metadata.profiler.interface.sqlalchemy.unity_catalog.profiler_interface import ( UnityCatalogProfilerInterface, ) +from metadata.profiler.interface.sqlalchemy.unity_catalog.sampler_interface import ( + UnityCatalogSamplerInterface, +) from metadata.utils.service_spec.default import DefaultDatabaseSpec ServiceSpec = DefaultDatabaseSpec( @@ -19,4 +22,5 @@ usage_source_class=UnitycatalogUsageSource, profiler_class=UnityCatalogProfilerInterface, test_suite_class=UnityCatalogTestSuiteInterface, + sampler_class=UnityCatalogSamplerInterface, ) diff --git a/ingestion/src/metadata/ingestion/source/database/unitycatalog/usage.py b/ingestion/src/metadata/ingestion/source/database/unitycatalog/usage.py index 7a310f736a3f..c533454be8cb 100644 --- a/ingestion/src/metadata/ingestion/source/database/unitycatalog/usage.py +++ b/ingestion/src/metadata/ingestion/source/database/unitycatalog/usage.py @@ -11,17 +11,22 @@ """ unity catalog usage module """ +import traceback +from datetime import datetime +from typing import Iterable -from metadata.ingestion.source.database.databricks.usage import DatabricksUsageSource +from metadata.generated.schema.type.basic import DateTime +from metadata.generated.schema.type.tableQuery import TableQueries, TableQuery from metadata.ingestion.source.database.unitycatalog.query_parser import ( UnityCatalogQueryParserSource, ) +from metadata.ingestion.source.database.usage_source import UsageSource from metadata.utils.logger import ingestion_logger logger = ingestion_logger() -class UnitycatalogUsageSource(UnityCatalogQueryParserSource, DatabricksUsageSource): +class UnitycatalogUsageSource(UnityCatalogQueryParserSource, UsageSource): """ UnityCatalog Usage Source @@ -29,3 +34,37 @@ class UnitycatalogUsageSource(UnityCatalogQueryParserSource, DatabricksUsageSour DatabricksUsageSource as both the sources would call the same API for fetching Usage Queries """ + + def yield_table_queries(self) -> Iterable[TableQuery]: + """ + Method to yield TableQueries + """ + queries = [] + data = self.client.list_query_history( + start_date=self.start, + end_date=self.end, + ) + for row in data or []: + try: + if self.client.is_query_valid(row): + queries.append( + TableQuery( + dialect=self.dialect.value, + query=row.get("query_text"), + userName=row.get("user_name"), + startTime=str(row.get("query_start_time_ms")), + endTime=str(row.get("execution_end_time_ms")), + analysisDate=DateTime(datetime.now()), + serviceName=self.config.serviceName, + duration=row.get("duration") + if row.get("duration") + else None, + ) + ) + except Exception as err: + logger.debug(traceback.format_exc()) + logger.warning( + f"Failed to process query {row.get('query_text')} due to: {err}" + ) + + yield TableQueries(queries=queries) diff --git a/ingestion/src/metadata/ingestion/source/database/usage_source.py b/ingestion/src/metadata/ingestion/source/database/usage_source.py index 6209d601364f..65d5f2635876 100644 --- a/ingestion/src/metadata/ingestion/source/database/usage_source.py +++ b/ingestion/src/metadata/ingestion/source/database/usage_source.py @@ -153,7 +153,7 @@ def yield_table_queries(self) -> Iterable[TableQuery]: if query: logger.debug( ( - f"###### USAGE QUERY #######\n{mask_query(query, self.dialect.value)}" + f"###### USAGE QUERY #######\n{mask_query(query, self.dialect.value) or query}" "\n##########################" ) ) diff --git a/ingestion/src/metadata/ingestion/source/messaging/kafka/metadata.py b/ingestion/src/metadata/ingestion/source/messaging/kafka/metadata.py index f6f11d070fd9..72873ace2dcf 100644 --- a/ingestion/src/metadata/ingestion/source/messaging/kafka/metadata.py +++ b/ingestion/src/metadata/ingestion/source/messaging/kafka/metadata.py @@ -22,21 +22,19 @@ from metadata.ingestion.api.steps import InvalidSourceException from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.messaging.common_broker_source import CommonBrokerSource -from metadata.utils.ssl_manager import SSLManager +from metadata.utils.ssl_manager import SSLManager, check_ssl_and_init class KafkaSource(CommonBrokerSource): def __init__(self, config: WorkflowSource, metadata: OpenMetadata): self.ssl_manager = None - service_connection = cast(KafkaConnection, config.serviceConnection.root.config) - if service_connection.schemaRegistrySSL: - self.ssl_manager = SSLManager( - ca=service_connection.schemaRegistrySSL.root.caCertificate, - key=service_connection.schemaRegistrySSL.root.sslKey, - cert=service_connection.schemaRegistrySSL.root.sslCertificate, - ) - service_connection = self.ssl_manager.setup_ssl( - config.serviceConnection.root.config + self.service_connection = cast( + KafkaConnection, config.serviceConnection.root.config + ) + self.ssl_manager: SSLManager = check_ssl_and_init(self.service_connection) + if self.ssl_manager: + self.service_connection = self.ssl_manager.setup_ssl( + self.service_connection ) super().__init__(config, metadata) diff --git a/ingestion/src/metadata/ingestion/source/pipeline/airbyte/client.py b/ingestion/src/metadata/ingestion/source/pipeline/airbyte/client.py index 63a6817cf84a..ab342831deaf 100644 --- a/ingestion/src/metadata/ingestion/source/pipeline/airbyte/client.py +++ b/ingestion/src/metadata/ingestion/source/pipeline/airbyte/client.py @@ -20,6 +20,7 @@ from metadata.ingestion.ometa.client import REST, APIError, ClientConfig from metadata.utils.constants import AUTHORIZATION_HEADER, NO_ACCESS_TOKEN from metadata.utils.credentials import generate_http_basic_token +from metadata.utils.helpers import clean_uri class AirbyteClient: @@ -30,8 +31,8 @@ class AirbyteClient: def __init__(self, config: AirbyteConnection): self.config = config client_config: ClientConfig = ClientConfig( - base_url=str(self.config.hostPort), - api_version="api/v1", + base_url=clean_uri(self.config.hostPort), + api_version=self.config.apiVersion, auth_header=AUTHORIZATION_HEADER, auth_token=lambda: (NO_ACCESS_TOKEN, 0), ) diff --git a/ingestion/src/metadata/ingestion/source/pipeline/dbtcloud/metadata.py b/ingestion/src/metadata/ingestion/source/pipeline/dbtcloud/metadata.py index f0df49e3123a..6420276e73ec 100644 --- a/ingestion/src/metadata/ingestion/source/pipeline/dbtcloud/metadata.py +++ b/ingestion/src/metadata/ingestion/source/pipeline/dbtcloud/metadata.py @@ -276,8 +276,16 @@ def yield_pipeline_status( Get Pipeline Status """ try: - task_status = [ - TaskStatus( + + pipeline_fqn = fqn.build( + metadata=self.metadata, + entity_type=Pipeline, + service_name=self.context.get().pipeline_service, + pipeline_name=self.context.get().pipeline, + ) + + for task in self.client.get_runs(job_id=int(pipeline_details.id)) or []: + task_status = TaskStatus( name=str(task.id), executionStatus=STATUS_MAP.get(task.state, StatusType.Pending), startTime=( @@ -303,37 +311,21 @@ def yield_pipeline_status( else None ), ) - for task in self.client.get_runs(job_id=int(pipeline_details.id)) or [] - ] - pipeline_status = PipelineStatus( - executionStatus=STATUS_MAP.get( - pipeline_details.state, StatusType.Pending - ), - taskStatus=task_status, - timestamp=Timestamp( - datetime_to_ts( - datetime.strptime( - pipeline_details.created_at, "%Y-%m-%d %H:%M:%S.%f%z" - ) - if pipeline_details.created_at - else None - ) - ), - ) + pipeline_status = PipelineStatus( + executionStatus=task_status.executionStatus, + taskStatus=[task_status], + timestamp=task_status.endTime + if task_status.endTime + else task_status.startTime, + ) - pipeline_fqn = fqn.build( - metadata=self.metadata, - entity_type=Pipeline, - service_name=self.context.get().pipeline_service, - pipeline_name=self.context.get().pipeline, - ) - yield Either( - right=OMetaPipelineStatus( - pipeline_fqn=pipeline_fqn, - pipeline_status=pipeline_status, + yield Either( + right=OMetaPipelineStatus( + pipeline_fqn=pipeline_fqn, + pipeline_status=pipeline_status, + ) ) - ) except Exception as exc: yield Either( diff --git a/ingestion/src/metadata/ingestion/source/pipeline/kafkaconnect/metadata.py b/ingestion/src/metadata/ingestion/source/pipeline/kafkaconnect/metadata.py index 620f2416033e..ce37ca3f285c 100644 --- a/ingestion/src/metadata/ingestion/source/pipeline/kafkaconnect/metadata.py +++ b/ingestion/src/metadata/ingestion/source/pipeline/kafkaconnect/metadata.py @@ -95,7 +95,7 @@ def yield_pipeline( sourceUrl=connection_url, tasks=[ Task( - name=task.id, + name=str(task.id), ) for task in pipeline_details.tasks or [] ], @@ -205,7 +205,7 @@ def yield_pipeline_lineage_details( metadata=self.metadata, entity_type=Topic, service_name=self.service_connection.messagingServiceName, - topic_name=topic.name, + topic_name=str(topic.name), ) topic_entity = self.metadata.get_by_name(entity=Topic, fqn=topic_fqn) @@ -279,7 +279,7 @@ def yield_pipeline_status( try: task_status = [ TaskStatus( - name=task.id, + name=str(task.id), executionStatus=STATUS_MAP.get(task.state, StatusType.Pending), ) for task in pipeline_details.tasks or [] diff --git a/ingestion/src/metadata/ingestion/source/pipeline/kafkaconnect/models.py b/ingestion/src/metadata/ingestion/source/pipeline/kafkaconnect/models.py index 55109d02d712..9de270f18a2c 100644 --- a/ingestion/src/metadata/ingestion/source/pipeline/kafkaconnect/models.py +++ b/ingestion/src/metadata/ingestion/source/pipeline/kafkaconnect/models.py @@ -27,7 +27,7 @@ class KafkaConnectTasks(BaseModel): default="UNASSIGNED", description="State of the task (e.g., RUNNING, STOPPED)" ) worker_id: Optional[str] = Field( - ..., description="ID of the worker running the task" + default=None, description="ID of the worker running the task" ) @@ -43,15 +43,15 @@ class KafkaConnectPipelineDetails(BaseModel): default="UNASSIGNED", description="State of the connector (e.g., RUNNING, STOPPED)", ) - tasks: Optional[List[KafkaConnectTasks]] - topics: Optional[List[KafkaConnectTopics]] - conn_type: Optional[str] = Field(..., alias="type") + tasks: Optional[List[KafkaConnectTasks]] = [] + topics: Optional[List[KafkaConnectTopics]] = [] + conn_type: Optional[str] = Field(default="UNKNOWN", alias="type") class KafkaConnectDatasetDetails(BaseModel): - table: Optional[str] - database: Optional[str] - container_name: Optional[str] + table: Optional[str] = None + database: Optional[str] = None + container_name: Optional[str] = None @property def dataset_type(self): diff --git a/ingestion/src/metadata/ingestion/source/pipeline/nifi/connection.py b/ingestion/src/metadata/ingestion/source/pipeline/nifi/connection.py index 097b8cc1dbb4..680a8cff54c5 100644 --- a/ingestion/src/metadata/ingestion/source/pipeline/nifi/connection.py +++ b/ingestion/src/metadata/ingestion/source/pipeline/nifi/connection.py @@ -17,8 +17,10 @@ from metadata.generated.schema.entity.automations.workflow import ( Workflow as AutomationWorkflow, ) +from metadata.generated.schema.entity.services.connections.pipeline.nifi.basicAuth import ( + NifiBasicAuth, +) from metadata.generated.schema.entity.services.connections.pipeline.nifiConnection import ( - BasicAuthentication, NifiConnection, ) from metadata.generated.schema.entity.services.connections.testConnectionResult import ( @@ -34,7 +36,7 @@ def get_connection(connection: NifiConnection) -> NifiClient: """ Create connection """ - if isinstance(connection.nifiConfig, BasicAuthentication): + if isinstance(connection.nifiConfig, NifiBasicAuth): return NifiClient( host_port=connection.hostPort, username=connection.nifiConfig.username, diff --git a/ingestion/src/metadata/ingestion/source/storage/s3/metadata.py b/ingestion/src/metadata/ingestion/source/storage/s3/metadata.py index 777944bc204b..6220433d72b5 100644 --- a/ingestion/src/metadata/ingestion/source/storage/s3/metadata.py +++ b/ingestion/src/metadata/ingestion/source/storage/s3/metadata.py @@ -12,6 +12,7 @@ import json import secrets import traceback +from copy import deepcopy from datetime import datetime, timedelta from enum import Enum from typing import Dict, Iterable, List, Optional, Tuple @@ -293,15 +294,26 @@ def _generate_container_details( ) # if we have a sample file to fetch a schema from if sample_key: - columns = self._get_columns( - container_name=bucket_name, - sample_key=sample_key, - metadata_entry=metadata_entry, - config_source=S3Config( - securityConfig=self.service_connection.awsConfig - ), - client=self.s3_client, - ) + try: + columns = self._get_columns( + container_name=bucket_name, + sample_key=sample_key, + metadata_entry=metadata_entry, + config_source=S3Config( + securityConfig=self.service_connection.awsConfig + ), + client=self.s3_client, + ) + except Exception as err: + logger.warning() + self.status.failed( + error=StackTraceError( + name=f"{bucket_name}/{sample_key}", + error=f"Error extracting columns from [{bucket_name}/{sample_key}] due to: [{err}]", + stackTrace=traceback.format_exc(), + ) + ) + return None if columns: prefix = ( f"{KEY_SEPARATOR}{metadata_entry.dataPath.strip(KEY_SEPARATOR)}" @@ -325,6 +337,45 @@ def _generate_container_details( ) return None + def _generate_structured_containers_by_depth( + self, + bucket_response: S3BucketResponse, + metadata_entry: MetadataEntry, + parent: Optional[EntityReference] = None, + ) -> Iterable[S3ContainerDetails]: + try: + prefix = self._get_sample_file_prefix(metadata_entry=metadata_entry) + if prefix: + response = self.s3_client.list_objects_v2( + Bucket=bucket_response.name, Prefix=prefix + ) + # total depth is depth of prefix + depth of the metadata entry + total_depth = metadata_entry.depth + len(prefix[:-1].split("/")) + candidate_keys = { + "/".join(entry.get("Key").split("/")[:total_depth]) + "/" + for entry in response[S3_CLIENT_ROOT_RESPONSE] + if entry + and entry.get("Key") + and len(entry.get("Key").split("/")) > total_depth + } + for key in candidate_keys: + metadata_entry_copy = deepcopy(metadata_entry) + metadata_entry_copy.dataPath = key.strip(KEY_SEPARATOR) + structured_container: Optional[ + S3ContainerDetails + ] = self._generate_container_details( + bucket_response=bucket_response, + metadata_entry=metadata_entry_copy, + parent=parent, + ) + if structured_container: + yield structured_container + except Exception as err: + logger.warning( + f"Error while generating structured containers by depth for {metadata_entry.dataPath} - {err}" + ) + logger.debug(traceback.format_exc()) + def _generate_structured_containers( self, bucket_response: S3BucketResponse, @@ -336,15 +387,22 @@ def _generate_structured_containers( f"Extracting metadata from path {metadata_entry.dataPath.strip(KEY_SEPARATOR)} " f"and generating structured container" ) - structured_container: Optional[ - S3ContainerDetails - ] = self._generate_container_details( - bucket_response=bucket_response, - metadata_entry=metadata_entry, - parent=parent, - ) - if structured_container: - yield structured_container + if metadata_entry.depth == 0: + structured_container: Optional[ + S3ContainerDetails + ] = self._generate_container_details( + bucket_response=bucket_response, + metadata_entry=metadata_entry, + parent=parent, + ) + if structured_container: + yield structured_container + else: + yield from self._generate_structured_containers_by_depth( + bucket_response=bucket_response, + metadata_entry=metadata_entry, + parent=parent, + ) def is_valid_unstructured_file(self, accepted_extensions: List, key: str) -> bool: # Split the string into a list of values @@ -413,7 +471,7 @@ def _yield_nested_unstructured_containers( candidate_keys = [ entry["Key"] for entry in response[S3_CLIENT_ROOT_RESPONSE] - if entry and entry.get("Key") + if entry and entry.get("Key") and not entry.get("Key").endswith("/") ] for key in candidate_keys: if self.is_valid_unstructured_file(metadata_entry.unstructuredFormats, key): @@ -622,7 +680,7 @@ def _get_sample_file_path( candidate_keys = [ entry["Key"] for entry in response[S3_CLIENT_ROOT_RESPONSE] - if entry and entry.get("Key") + if entry and entry.get("Key") and not entry.get("Key").endswith("/") ] # pick a random key out of the candidates if any were returned if candidate_keys: diff --git a/ingestion/src/metadata/profiler/interface/sqlalchemy/unity_catalog/sampler_interface.py b/ingestion/src/metadata/profiler/interface/sqlalchemy/unity_catalog/sampler_interface.py new file mode 100644 index 000000000000..12a4ae3eaacb --- /dev/null +++ b/ingestion/src/metadata/profiler/interface/sqlalchemy/unity_catalog/sampler_interface.py @@ -0,0 +1,29 @@ +# Copyright 2021 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Interfaces with database for all database engine +supporting sqlalchemy abstraction layer +""" +from metadata.ingestion.source.database.databricks.connection import ( + get_connection as databricks_get_connection, +) +from metadata.sampler.sqlalchemy.sampler import SQASampler + + +class UnityCatalogSamplerInterface(SQASampler): + def get_client(self): + """client is the session for SQA""" + self.connection = databricks_get_connection(self.service_connection_config) + self.client = super().get_client() + self.set_catalog(self.client) + + return self.client diff --git a/ingestion/src/metadata/profiler/metrics/window/first_quartile.py b/ingestion/src/metadata/profiler/metrics/window/first_quartile.py index e24e94ec75ce..2b20bd3af6ec 100644 --- a/ingestion/src/metadata/profiler/metrics/window/first_quartile.py +++ b/ingestion/src/metadata/profiler/metrics/window/first_quartile.py @@ -56,14 +56,14 @@ def fn(self): # col fullname is only needed for MySQL and SQLite return self._compute_sqa_fn( column(self.col.name, self.col.type), - self.col.table.fullname if self.col.table is not None else None, + self.col.table.name if self.col.table is not None else None, 0.25, ) if is_concatenable(self.col.type): return self._compute_sqa_fn( LenFn(column(self.col.name, self.col.type)), - self.col.table.fullname if self.col.table is not None else None, + self.col.table.name if self.col.table is not None else None, 0.25, ) diff --git a/ingestion/src/metadata/profiler/metrics/window/median.py b/ingestion/src/metadata/profiler/metrics/window/median.py index 8e2d1dcffe07..c12878de9184 100644 --- a/ingestion/src/metadata/profiler/metrics/window/median.py +++ b/ingestion/src/metadata/profiler/metrics/window/median.py @@ -56,14 +56,14 @@ def fn(self): # col fullname is only needed for MySQL and SQLite return self._compute_sqa_fn( column(self.col.name, self.col.type), - self.col.table.fullname if self.col.table is not None else None, + self.col.table.name if self.col.table is not None else None, 0.5, ) if is_concatenable(self.col.type): return self._compute_sqa_fn( LenFn(column(self.col.name, self.col.type)), - self.col.table.fullname if self.col.table is not None else None, + self.col.table.name if self.col.table is not None else None, 0.5, ) diff --git a/ingestion/src/metadata/profiler/metrics/window/third_quartile.py b/ingestion/src/metadata/profiler/metrics/window/third_quartile.py index 86ebf26bf83d..8f0479a097c5 100644 --- a/ingestion/src/metadata/profiler/metrics/window/third_quartile.py +++ b/ingestion/src/metadata/profiler/metrics/window/third_quartile.py @@ -56,14 +56,14 @@ def fn(self): # col fullname is only needed for MySQL and SQLite return self._compute_sqa_fn( column(self.col.name, self.col.type), - self.col.table.fullname if self.col.table is not None else None, + self.col.table.name if self.col.table is not None else None, 0.75, ) if is_concatenable(self.col.type): return self._compute_sqa_fn( LenFn(column(self.col.name, self.col.type)), - self.col.table.fullname if self.col.table is not None else None, + self.col.table.name if self.col.table is not None else None, 0.75, ) diff --git a/ingestion/src/metadata/profiler/orm/converter/common.py b/ingestion/src/metadata/profiler/orm/converter/common.py index 2d1e41e13dc4..4c2f1d0997a7 100644 --- a/ingestion/src/metadata/profiler/orm/converter/common.py +++ b/ingestion/src/metadata/profiler/orm/converter/common.py @@ -76,7 +76,9 @@ def map_types(self, col: Column, table_service_type): """returns an ORM type""" if col.arrayDataType: - return self._TYPE_MAP.get(col.dataType)(item_type=col.arrayDataType) + return self._TYPE_MAP.get(col.dataType)( + item_type=self._TYPE_MAP.get(col.arrayDataType) + ) return self.return_custom_type(col, table_service_type) def return_custom_type(self, col: Column, _): diff --git a/ingestion/src/metadata/profiler/processor/core.py b/ingestion/src/metadata/profiler/processor/core.py index 289b2aa8f09c..64b268f71e91 100644 --- a/ingestion/src/metadata/profiler/processor/core.py +++ b/ingestion/src/metadata/profiler/processor/core.py @@ -550,12 +550,12 @@ def get_profile(self) -> CreateTableProfileRequest: createDateTime=raw_create_date, sizeInByte=self._table_results.get("sizeInBytes"), profileSample=( - self.profiler_interface.sampler.sample_config.profile_sample + self.profiler_interface.sampler.sample_config.profileSample if self.profiler_interface.sampler.sample_config else None ), profileSampleType=( - self.profiler_interface.sampler.sample_config.profile_sample_type + self.profiler_interface.sampler.sample_config.profileSampleType if self.profiler_interface.sampler.sample_config else None ), diff --git a/ingestion/src/metadata/profiler/source/database/base/profiler_source.py b/ingestion/src/metadata/profiler/source/database/base/profiler_source.py index f14dfb206a46..c9bec0aa2170 100644 --- a/ingestion/src/metadata/profiler/source/database/base/profiler_source.py +++ b/ingestion/src/metadata/profiler/source/database/base/profiler_source.py @@ -150,9 +150,9 @@ def create_profiler_interface( database_entity=database_entity, table_config=config, default_sample_config=SampleConfig( - profile_sample=self.source_config.profileSample, - profile_sample_type=self.source_config.profileSampleType, - sampling_method_type=self.source_config.samplingMethodType, + profileSample=self.source_config.profileSample, + profileSampleType=self.source_config.profileSampleType, + samplingMethodType=self.source_config.samplingMethodType, ), ) diff --git a/ingestion/src/metadata/profiler/source/fetcher/config.py b/ingestion/src/metadata/profiler/source/fetcher/config.py index ef44e9eca1e2..006f1d5181c3 100644 --- a/ingestion/src/metadata/profiler/source/fetcher/config.py +++ b/ingestion/src/metadata/profiler/source/fetcher/config.py @@ -42,3 +42,7 @@ def tableFilterPattern(self) -> Optional[FilterPattern]: @property def useFqnForFiltering(self) -> Optional[bool]: ... + + @property + def includeViews(self) -> Optional[bool]: + ... diff --git a/ingestion/src/metadata/profiler/source/fetcher/fetcher_strategy.py b/ingestion/src/metadata/profiler/source/fetcher/fetcher_strategy.py index 0fd55f84ccdc..60c04e3a8b81 100644 --- a/ingestion/src/metadata/profiler/source/fetcher/fetcher_strategy.py +++ b/ingestion/src/metadata/profiler/source/fetcher/fetcher_strategy.py @@ -17,6 +17,7 @@ from typing import Iterable, Iterator, Optional, cast from metadata.generated.schema.entity.data.database import Database +from metadata.generated.schema.entity.data.table import TableType from metadata.generated.schema.entity.services.ingestionPipelines.status import ( StackTraceError, ) @@ -115,13 +116,13 @@ def __init__( super().__init__(config, metadata, global_profiler_config, status) self.source_config = cast( EntityFilterConfigInterface, self.source_config - ) # Satisfy typchecker + ) # Satisfy typechecker def _filter_databases(self, databases: Iterable[Database]) -> Iterable[Database]: """Filter databases based on the filter pattern Args: - database (Database): Database to filter + databases (Database): Database to filter Returns: bool @@ -192,6 +193,21 @@ def _filter_tables(self, table: Table) -> bool: return False + def _filter_views(self, table: Table) -> bool: + """Filter the tables based on include views configuration""" + # If we include views, nothing to filter + if self.source_config.includeViews: + return False + + # Otherwise, filter out views + if table.tableType == TableType.View: + self.status.filter( + table.name.root, f"We are not including views {table.name.root}" + ) + return True + + return False + def _filter_column_metrics_computation(self): """Filter""" @@ -242,6 +258,7 @@ def _filter_entities(self, tables: Iterable[Table]) -> Iterable[Table]: not self.source_config.classificationFilterPattern or not self.filter_classifications(table) ) + and not self._filter_views(table) ] return tables diff --git a/ingestion/src/metadata/sampler/config.py b/ingestion/src/metadata/sampler/config.py index e5c5239f6ae6..69b9dc75e554 100644 --- a/ingestion/src/metadata/sampler/config.py +++ b/ingestion/src/metadata/sampler/config.py @@ -119,9 +119,9 @@ def get_profile_sample_config( try: if config and config.profileSample: return SampleConfig( - profile_sample=config.profileSample, - profile_sample_type=config.profileSampleType, - sampling_method_type=config.samplingMethodType, + profileSample=config.profileSample, + profileSampleType=config.profileSampleType, + samplingMethodType=config.samplingMethodType, ) except AttributeError: pass diff --git a/ingestion/src/metadata/sampler/models.py b/ingestion/src/metadata/sampler/models.py index 26aa4d9ac95f..e76aa40f58bb 100644 --- a/ingestion/src/metadata/sampler/models.py +++ b/ingestion/src/metadata/sampler/models.py @@ -101,6 +101,6 @@ def __str__(self): class SampleConfig(ConfigModel): """Profile Sample Config""" - profile_sample: Optional[Union[float, int]] = None - profile_sample_type: Optional[ProfileSampleType] = ProfileSampleType.PERCENTAGE - sampling_method_type: Optional[SamplingMethodType] = None + profileSample: Optional[Union[float, int]] = None + profileSampleType: Optional[ProfileSampleType] = ProfileSampleType.PERCENTAGE + samplingMethodType: Optional[SamplingMethodType] = None diff --git a/ingestion/src/metadata/sampler/nosql/sampler.py b/ingestion/src/metadata/sampler/nosql/sampler.py index dc873c5c10d5..16ed1229f62b 100644 --- a/ingestion/src/metadata/sampler/nosql/sampler.py +++ b/ingestion/src/metadata/sampler/nosql/sampler.py @@ -82,10 +82,10 @@ def _fetch_sample_data(self, columns: List[SQALikeColumn]) -> TableData: def _get_limit(self) -> Optional[int]: num_rows = self.client.item_count(self.raw_dataset) - if self.sample_config.profile_sample_type == ProfileSampleType.PERCENTAGE: - limit = num_rows * (self.sample_config.profile_sample / 100) - elif self.sample_config.profile_sample_type == ProfileSampleType.ROWS: - limit = self.sample_config.profile_sample + if self.sample_config.profileSampleType == ProfileSampleType.PERCENTAGE: + limit = num_rows * (self.sample_config.profileSample or 100 / 100) + elif self.sample_config.profileSampleType == ProfileSampleType.ROWS: + limit = self.sample_config.profileSample else: limit = SAMPLE_DATA_DEFAULT_COUNT return limit diff --git a/ingestion/src/metadata/sampler/pandas/sampler.py b/ingestion/src/metadata/sampler/pandas/sampler.py index 11f1095e9578..b221837daed5 100644 --- a/ingestion/src/metadata/sampler/pandas/sampler.py +++ b/ingestion/src/metadata/sampler/pandas/sampler.py @@ -123,13 +123,13 @@ def _get_sampled_dataframe(self): """ random.shuffle(self.raw_dataset) # we'll shuffle the list of dataframes # sampling data based on profiler config (if any) - if self.sample_config.profile_sample_type == ProfileSampleType.PERCENTAGE: + if self.sample_config.profileSampleType == ProfileSampleType.PERCENTAGE: try: - profile_sample = (self.sample_config.profile_sample or 100) / 100 + profile_sample = (self.sample_config.profileSample or 100) / 100 except TypeError: # if the profile sample is not a number or is None # we'll set it to 100 - profile_sample = self.sample_config.profile_sample = 100 + profile_sample = self.sample_config.profileSample = 100 return [ df.sample( frac=profile_sample, @@ -141,7 +141,7 @@ def _get_sampled_dataframe(self): # we'll distribute the sample size equally among the dataframes sample_rows_per_chunk: int = math.floor( - (self.sample_config.profile_sample or 100) / len(self.raw_dataset) + (self.sample_config.profileSample or 100) / len(self.raw_dataset) ) num_rows = sum(len(df) for df in self.raw_dataset) @@ -187,9 +187,9 @@ def get_dataset(self, **__): if self.partition_details: self._table = self._partitioned_table() - if not self.sample_config.profile_sample or ( - self.sample_config.profile_sample == 100 - and self.sample_config.profile_sample_type == ProfileSampleType.PERCENTAGE + if not self.sample_config.profileSample or ( + self.sample_config.profileSample == 100 + and self.sample_config.profileSampleType == ProfileSampleType.PERCENTAGE ): return self.raw_dataset return self._get_sampled_dataframe() diff --git a/ingestion/src/metadata/sampler/partition.py b/ingestion/src/metadata/sampler/partition.py index e62ad94a0496..c2eb76f9833e 100644 --- a/ingestion/src/metadata/sampler/partition.py +++ b/ingestion/src/metadata/sampler/partition.py @@ -49,7 +49,7 @@ def validate_athena_injected_partitioning( column_partitions: Optional[List[PartitionColumnDetails]] = table_partitions.columns if not column_partitions: - raise RuntimeError("Table parition is set but no columns are defined.") + raise RuntimeError("Table partition is set but no columns are defined.") for column_partition in column_partitions: if column_partition.intervalType == PartitionIntervalTypes.INJECTED: @@ -163,6 +163,7 @@ def _handle_bigquery_partition( partitionIntegerRangeStart=1, partitionIntegerRangeEnd=10000, ) + # TODO: Allow External Hive Partitioning for profiler raise TypeError( f"Unsupported partition type {partition.intervalType}. Skipping table" ) diff --git a/ingestion/src/metadata/sampler/sampler_interface.py b/ingestion/src/metadata/sampler/sampler_interface.py index 03a3d1d6dc98..fe363816d01c 100644 --- a/ingestion/src/metadata/sampler/sampler_interface.py +++ b/ingestion/src/metadata/sampler/sampler_interface.py @@ -76,9 +76,6 @@ def __init__( self._columns: Optional[List[SQALikeColumn]] = None self.sample_config = sample_config - if not self.sample_config.profile_sample: - self.sample_config.profile_sample = 100 - self.entity = entity self.include_columns = include_columns self.exclude_columns = exclude_columns diff --git a/ingestion/src/metadata/sampler/sqlalchemy/bigquery/sampler.py b/ingestion/src/metadata/sampler/sqlalchemy/bigquery/sampler.py index 502143c11468..cd82565506b6 100644 --- a/ingestion/src/metadata/sampler/sqlalchemy/bigquery/sampler.py +++ b/ingestion/src/metadata/sampler/sqlalchemy/bigquery/sampler.py @@ -54,7 +54,6 @@ def __init__( sample_query: Optional[str] = None, storage_config: DataStorageConfig = None, sample_data_count: Optional[int] = SAMPLE_DATA_DEFAULT_COUNT, - table_type: TableType = None, **kwargs, ): super().__init__( @@ -68,7 +67,7 @@ def __init__( sample_data_count=sample_data_count, **kwargs, ) - self.raw_dataset_type: TableType = table_type + self.raw_dataset_type: Optional[TableType] = entity.tableType def set_tablesample(self, selectable: SqaTable): """Set the TABLESAMPLE clause for BigQuery @@ -76,11 +75,11 @@ def set_tablesample(self, selectable: SqaTable): selectable (Table): Table object """ if ( - self.sample_config.profile_sample_type == ProfileSampleType.PERCENTAGE + self.sample_config.profileSampleType == ProfileSampleType.PERCENTAGE and self.raw_dataset_type != TableType.View ): return selectable.tablesample( - text(f"{self.sample_config.profile_sample or 100} PERCENT") + text(f"{self.sample_config.profileSample or 100} PERCENT") ) return selectable @@ -116,7 +115,7 @@ def get_sample_query(self, *, column=None) -> Query: """get query for sample data""" # TABLESAMPLE SYSTEM is not supported for views if ( - self.sample_config.profile_sample_type == ProfileSampleType.PERCENTAGE + self.sample_config.profileSampleType == ProfileSampleType.PERCENTAGE and self.raw_dataset_type != TableType.View ): return self._base_sample_query(column).cte( diff --git a/ingestion/src/metadata/sampler/sqlalchemy/mssql/sampler.py b/ingestion/src/metadata/sampler/sqlalchemy/mssql/sampler.py index 0d5b7dc5b53a..637cd35e7bae 100644 --- a/ingestion/src/metadata/sampler/sqlalchemy/mssql/sampler.py +++ b/ingestion/src/metadata/sampler/sqlalchemy/mssql/sampler.py @@ -17,7 +17,7 @@ from sqlalchemy import Table, text from sqlalchemy.sql.selectable import CTE -from metadata.generated.schema.entity.data.table import ProfileSampleType +from metadata.generated.schema.entity.data.table import ProfileSampleType, TableType from metadata.sampler.sqlalchemy.sampler import SQASampler @@ -32,14 +32,16 @@ def set_tablesample(self, selectable: Table): Args: selectable (Table): _description_ """ - if self.sample_config.profile_sample_type == ProfileSampleType.PERCENTAGE: + if self.entity.tableType != TableType.View: + if self.sample_config.profileSampleType == ProfileSampleType.PERCENTAGE: + return selectable.tablesample( + text(f"{self.sample_config.profileSample or 100} PERCENT") + ) + return selectable.tablesample( - text(f"{self.sample_config.profile_sample or 100} PERCENT") + text(f"{int(self.sample_config.profileSample or 100)} ROWS") ) - - return selectable.tablesample( - text(f"{int(self.sample_config.profile_sample or 100)} ROWS") - ) + return selectable def get_sample_query(self, *, column=None) -> CTE: """get query for sample data""" diff --git a/ingestion/src/metadata/sampler/sqlalchemy/postgres/sampler.py b/ingestion/src/metadata/sampler/sqlalchemy/postgres/sampler.py index e2f15f1e59a9..1bdbe08e6e75 100644 --- a/ingestion/src/metadata/sampler/sqlalchemy/postgres/sampler.py +++ b/ingestion/src/metadata/sampler/sqlalchemy/postgres/sampler.py @@ -66,7 +66,7 @@ def __init__( self.sampling_method_type = SamplingMethodType.BERNOULLI if ( sample_config - and sample_config.sampling_method_type == SamplingMethodType.SYSTEM + and sample_config.samplingMethodType == SamplingMethodType.SYSTEM ): self.sampling_fn = func.system @@ -75,15 +75,15 @@ def set_tablesample(self, selectable: SqaTable): Args: selectable (Table): _description_ """ - if self.sample_config.profile_sample_type == ProfileSampleType.PERCENTAGE: + if self.sample_config.profileSampleType == ProfileSampleType.PERCENTAGE: return selectable.tablesample( - self.sampling_fn(self.sample_config.profile_sample or 100) + self.sampling_fn(self.sample_config.profileSample or 100) ) return selectable def get_sample_query(self, *, column=None) -> Query: - if self.sample_config.profile_sample_type == ProfileSampleType.PERCENTAGE: + if self.sample_config.profileSampleType == ProfileSampleType.PERCENTAGE: return self._base_sample_query(column).cte( f"{self.raw_dataset.__tablename__}_rnd" ) diff --git a/ingestion/src/metadata/sampler/sqlalchemy/sampler.py b/ingestion/src/metadata/sampler/sqlalchemy/sampler.py index cf5113208dd0..4d28f29f4a95 100644 --- a/ingestion/src/metadata/sampler/sqlalchemy/sampler.py +++ b/ingestion/src/metadata/sampler/sqlalchemy/sampler.py @@ -12,6 +12,7 @@ Helper module to handle data sampling for the profiler """ +import hashlib import traceback from typing import List, Optional, Union, cast @@ -32,6 +33,7 @@ from metadata.profiler.orm.functions.random_num import RandomNumFn from metadata.profiler.processor.handle_partition import build_partition_predicate from metadata.sampler.sampler_interface import SamplerInterface +from metadata.utils.constants import UTF_8 from metadata.utils.helpers import is_safe_sql_query from metadata.utils.logger import profiler_interface_registry_logger @@ -109,17 +111,28 @@ def _base_sample_query(self, column: Optional[Column], label=None): query = self.get_partitioned_query(query) return query + def get_sampler_table_name(self) -> str: + """Get the base name of the SQA table for sampling. + We use MD5 as a hashing algorithm to generate a unique name for the table + keeping its length controlled. Otherwise, we ended up having issues + with names getting truncated when we add the suffixes to the identifiers + such as _sample, or _rnd. + """ + encoded_name = self.raw_dataset.__tablename__.encode(UTF_8) + hash_object = hashlib.md5(encoded_name) + return hash_object.hexdigest() + def get_sample_query(self, *, column=None) -> Query: """get query for sample data""" - if self.sample_config.profile_sample_type == ProfileSampleType.PERCENTAGE: + if self.sample_config.profileSampleType == ProfileSampleType.PERCENTAGE: rnd = self._base_sample_query( column, (ModuloFn(RandomNumFn(), 100)).label(RANDOM_LABEL), - ).cte(f"{self.raw_dataset.__tablename__}_rnd") + ).cte(f"{self.get_sampler_table_name()}_rnd") session_query = self.client.query(rnd) return session_query.where( - rnd.c.random <= self.sample_config.profile_sample - ).cte(f"{self.raw_dataset.__tablename__}_sample") + rnd.c.random <= self.sample_config.profileSample + ).cte(f"{self.get_sampler_table_name()}_sample") table_query = self.client.query(self.raw_dataset) session_query = self._base_sample_query( @@ -128,8 +141,8 @@ def get_sample_query(self, *, column=None) -> Query: ) return ( session_query.order_by(RANDOM_LABEL) - .limit(self.sample_config.profile_sample) - .cte(f"{self.raw_dataset.__tablename__}_rnd") + .limit(self.sample_config.profileSample) + .cte(f"{self.get_sampler_table_name()}_rnd") ) def get_dataset(self, column=None, **__) -> Union[DeclarativeMeta, AliasedClass]: @@ -140,13 +153,10 @@ def get_dataset(self, column=None, **__) -> Union[DeclarativeMeta, AliasedClass] if self.sample_query: return self._rdn_sample_from_user_query() - if not self.sample_config.profile_sample or ( - int(self.sample_config.profile_sample) == 100 - and self.sample_config.profile_sample_type == ProfileSampleType.PERCENTAGE - ): + if not self.sample_config.profileSample: if self.partition_details: partitioned = self._partitioned_table() - return partitioned.cte(f"{self.raw_dataset.__tablename__}_partitioned") + return partitioned.cte(f"{self.get_sampler_table_name()}_partitioned") return self.raw_dataset @@ -165,23 +175,23 @@ def fetch_sample_data(self, columns: Optional[List[Column]] = None) -> TableData return self._fetch_sample_data_from_user_query() # Add new RandomNumFn column - rnd = self.get_sample_query() + ds = self.get_dataset() if not columns: - sqa_columns = [col for col in inspect(rnd).c if col.name != RANDOM_LABEL] + sqa_columns = [col for col in inspect(ds).c if col.name != RANDOM_LABEL] else: # we can't directly use columns as it is bound to self.raw_dataset and not the rnd table. # If we use it, it will result in a cross join between self.raw_dataset and rnd table names = [col.name for col in columns] sqa_columns = [ col - for col in inspect(rnd).c + for col in inspect(ds).c if col.name != RANDOM_LABEL and col.name in names ] try: sqa_sample = ( self.client.query(*sqa_columns) - .select_from(rnd) + .select_from(ds) .limit(self.sample_limit) .all() ) @@ -227,7 +237,7 @@ def _rdn_sample_from_user_query(self) -> Query: stmt = stmt.columns(*list(inspect(self.raw_dataset).c)) return self.client.query(stmt.subquery()).cte( - f"{self.raw_dataset.__tablename__}_user_sampled" + f"{self.get_sampler_table_name()}_user_sampled" ) def _partitioned_table(self) -> Query: diff --git a/ingestion/src/metadata/sampler/sqlalchemy/snowflake/sampler.py b/ingestion/src/metadata/sampler/sqlalchemy/snowflake/sampler.py index 714d507c7296..f4dbaa45896e 100644 --- a/ingestion/src/metadata/sampler/sqlalchemy/snowflake/sampler.py +++ b/ingestion/src/metadata/sampler/sqlalchemy/snowflake/sampler.py @@ -68,7 +68,7 @@ def __init__( self.sampling_method_type = func.bernoulli if ( sample_config - and sample_config.sampling_method_type == SamplingMethodType.SYSTEM + and sample_config.samplingMethodType == SamplingMethodType.SYSTEM ): self.sampling_method_type = func.system @@ -77,13 +77,13 @@ def set_tablesample(self, selectable: Table): Args: selectable (Table): _description_ """ - if self.sample_config.profile_sample_type == ProfileSampleType.PERCENTAGE: + if self.sample_config.profileSampleType == ProfileSampleType.PERCENTAGE: return selectable.tablesample( - self.sampling_method_type(self.sample_config.profile_sample or 100) + self.sampling_method_type(self.sample_config.profileSample or 100) ) return selectable.tablesample( - func.ROW(text(f"{self.sample_config.profile_sample or 100} ROWS")) + func.ROW(text(f"{self.sample_config.profileSample or 100} ROWS")) ) def get_sample_query(self, *, column=None) -> CTE: diff --git a/ingestion/src/metadata/sampler/sqlalchemy/trino/sampler.py b/ingestion/src/metadata/sampler/sqlalchemy/trino/sampler.py index 333e0bf01aa7..4eb6cc40c518 100644 --- a/ingestion/src/metadata/sampler/sqlalchemy/trino/sampler.py +++ b/ingestion/src/metadata/sampler/sqlalchemy/trino/sampler.py @@ -41,7 +41,7 @@ def _base_sample_query(self, column, label=None): return self.client.query(entity, label).where( or_( *[ - text(f"is_nan({cols.name}) = False") + text(f'is_nan("{cols.name}") = False') for cols in sqa_columns if type(cols.type) in FLOAT_SET ] diff --git a/ingestion/src/metadata/utils/constants.py b/ingestion/src/metadata/utils/constants.py index 49a0e7c5eef5..62655909f832 100644 --- a/ingestion/src/metadata/utils/constants.py +++ b/ingestion/src/metadata/utils/constants.py @@ -12,12 +12,17 @@ """ Define constants useful for the metadata ingestion """ +from metadata.generated.schema.entity.data.apiCollection import APICollection +from metadata.generated.schema.entity.data.apiEndpoint import APIEndpoint from metadata.generated.schema.entity.data.chart import Chart from metadata.generated.schema.entity.data.container import Container from metadata.generated.schema.entity.data.dashboard import Dashboard from metadata.generated.schema.entity.data.dashboardDataModel import DashboardDataModel from metadata.generated.schema.entity.data.database import Database from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema +from metadata.generated.schema.entity.data.glossary import Glossary +from metadata.generated.schema.entity.data.glossaryTerm import GlossaryTerm +from metadata.generated.schema.entity.data.metric import Metric from metadata.generated.schema.entity.data.mlmodel import MlModel from metadata.generated.schema.entity.data.pipeline import Pipeline from metadata.generated.schema.entity.data.searchIndex import SearchIndex @@ -42,12 +47,6 @@ from metadata.generated.schema.entity.services.connections.database.domoDatabaseConnection import ( DomoDatabaseType, ) -from metadata.generated.schema.entity.services.connections.database.dorisConnection import ( - DorisType, -) -from metadata.generated.schema.entity.services.connections.database.druidConnection import ( - DruidType, -) from metadata.generated.schema.entity.services.connections.database.dynamoDBConnection import ( DynamoDBType, ) @@ -69,9 +68,6 @@ from metadata.generated.schema.entity.services.connections.database.sasConnection import ( SasType, ) -from metadata.generated.schema.entity.services.connections.database.unityCatalogConnection import ( - DatabricksType, -) from metadata.generated.schema.entity.services.dashboardService import DashboardService from metadata.generated.schema.entity.services.databaseService import DatabaseService from metadata.generated.schema.entity.services.messagingService import MessagingService @@ -131,6 +127,8 @@ "metadataService": MetadataService, "searchService": SearchService, # Data Asset Entities + "apiCollection": APICollection, + "apiEndpoint": APIEndpoint, "table": Table, "storedProcedure": StoredProcedure, "database": Database, @@ -149,6 +147,10 @@ # Domain "domain": Domain, "dataProduct": DataProduct, + # Governance + "metric": Metric, + "glossary": Glossary, + "glossaryTerm": GlossaryTerm, } ENTITY_REFERENCE_TYPE_MAP = { @@ -161,11 +163,8 @@ DatalakeType.Datalake.value, BigtableType.BigTable.value, CouchbaseType.Couchbase.value, - DatabricksType.UnityCatalog.value, DeltaLakeType.DeltaLake.value, DomoDatabaseType.DomoDatabase.value, - DorisType.Doris.value, - DruidType.Druid.value, DynamoDBType.DynamoDB.value, GlueType.Glue.value, IcebergType.Iceberg.value, diff --git a/ingestion/src/metadata/utils/db_utils.py b/ingestion/src/metadata/utils/db_utils.py index ef3a7c8d6088..afd1e4832940 100644 --- a/ingestion/src/metadata/utils/db_utils.py +++ b/ingestion/src/metadata/utils/db_utils.py @@ -69,6 +69,10 @@ def get_view_lineage( fqn=table_fqn, ) + if not view_definition: + logger.warning(f"View definition for view {table_fqn} not available") + return + try: connection_type = str(connection_type) dialect = ConnectionTypeDialectMapper.dialect_of(connection_type) diff --git a/ingestion/src/metadata/utils/elasticsearch.py b/ingestion/src/metadata/utils/elasticsearch.py index 0f0ab014b057..8490309ec18f 100644 --- a/ingestion/src/metadata/utils/elasticsearch.py +++ b/ingestion/src/metadata/utils/elasticsearch.py @@ -18,6 +18,8 @@ from metadata.generated.schema.analytics.reportData import ReportData from metadata.generated.schema.entity.classification.tag import Tag +from metadata.generated.schema.entity.data.apiCollection import APICollection +from metadata.generated.schema.entity.data.apiEndpoint import APIEndpoint from metadata.generated.schema.entity.data.chart import Chart from metadata.generated.schema.entity.data.container import Container from metadata.generated.schema.entity.data.dashboard import Dashboard @@ -26,6 +28,7 @@ from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema from metadata.generated.schema.entity.data.glossary import Glossary from metadata.generated.schema.entity.data.glossaryTerm import GlossaryTerm +from metadata.generated.schema.entity.data.metric import Metric from metadata.generated.schema.entity.data.mlmodel import MlModel from metadata.generated.schema.entity.data.pipeline import Pipeline from metadata.generated.schema.entity.data.query import Query @@ -33,6 +36,7 @@ from metadata.generated.schema.entity.data.storedProcedure import StoredProcedure from metadata.generated.schema.entity.data.table import Table from metadata.generated.schema.entity.data.topic import Topic +from metadata.generated.schema.entity.services.apiService import ApiService from metadata.generated.schema.entity.services.databaseService import DatabaseService from metadata.generated.schema.entity.teams.team import Team from metadata.generated.schema.entity.teams.user import User @@ -42,6 +46,9 @@ T = TypeVar("T", bound=BaseModel) ES_INDEX_MAP = { + ApiService.__name__: "api_service_search_index", + APICollection.__name__: "api_collection_search_index", + APIEndpoint.__name__: "api_endpoint_search_index", Table.__name__: "table_search_index", Database.__name__: "database_search_index", DatabaseSchema.__name__: "database_schema_search_index", @@ -56,12 +63,13 @@ Topic.__name__: "topic_search_index", Pipeline.__name__: "pipeline_search_index", Glossary.__name__: "glossary_search_index", - GlossaryTerm.__name__: "glossary_search_index", + GlossaryTerm.__name__: "glossary_term_search_index", MlModel.__name__: "mlmodel_search_index", Tag.__name__: "tag_search_index", Container.__name__: "container_search_index", Query.__name__: "query_search_index", ReportData.__name__: "entity_report_data_index", + Metric.__name__: "metric_search_index", "web_analytic_user_activity_report": "web_analytic_user_activity_report_data_index", "web_analytic_entity_view_report": "web_analytic_entity_view_report_data_index", } diff --git a/ingestion/src/metadata/utils/logger.py b/ingestion/src/metadata/utils/logger.py index f437bb4d3fcd..08d648d8c2fd 100644 --- a/ingestion/src/metadata/utils/logger.py +++ b/ingestion/src/metadata/utils/logger.py @@ -28,7 +28,6 @@ from metadata.generated.schema.type.queryParserData import QueryParserData from metadata.generated.schema.type.tableQuery import TableQueries from metadata.ingestion.api.models import Entity -from metadata.ingestion.lineage.masker import mask_query from metadata.ingestion.models.delete_entity import DeleteEntity from metadata.ingestion.models.life_cycle import OMetaLifeCycleData from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification @@ -284,19 +283,13 @@ def _(record: PatchRequest) -> str: @get_log_name.register def _(record: TableQueries) -> str: """Get the log of the TableQuery""" - queries = "\n------\n".join( - mask_query(query.query, query.dialect) for query in record.queries - ) - return f"Table Queries [{queries}]" + return f"Table Queries [{len(record.queries)}]" @get_log_name.register def _(record: QueryParserData) -> str: """Get the log of the ParsedData""" - queries = "\n------\n".join( - mask_query(query.sql, query.dialect) for query in record.parsedData - ) - return f"Usage ParsedData [{queries}]" + return f"Usage ParsedData [{len(record.parsedData)}]" def redacted_config(config: Dict[str, Union[str, dict]]) -> Dict[str, Union[str, dict]]: diff --git a/ingestion/src/metadata/utils/ssl_manager.py b/ingestion/src/metadata/utils/ssl_manager.py index 1e6d7a0af5ec..01e7d591a61a 100644 --- a/ingestion/src/metadata/utils/ssl_manager.py +++ b/ingestion/src/metadata/utils/ssl_manager.py @@ -17,7 +17,7 @@ import tempfile import traceback from functools import singledispatch, singledispatchmethod -from typing import Optional, Union, cast +from typing import List, Optional, Union, cast from pydantic import SecretStr @@ -66,7 +66,9 @@ class SSLManager: "SSL Manager to manage SSL certificates for service connections" - def __init__(self, ca=None, key=None, cert=None): + def __init__( + self, ca=None, key=None, cert=None, *args, **kwargs + ): # pylint: disable=keyword-arg-before-vararg self.temp_files = [] self.ca_file_path = None self.cert_file_path = None @@ -77,6 +79,14 @@ def __init__(self, ca=None, key=None, cert=None): self.cert_file_path = self.create_temp_file(cert) if key: self.key_file_path = self.create_temp_file(key) + if args: + for arg in args: + if arg: + setattr(self, f"{arg}", self.create_temp_file(arg)) + if kwargs: + for dict_key, value in kwargs.items(): + if value: + setattr(self, f"{dict_key}", self.create_temp_file(value)) def create_temp_file(self, content: SecretStr): with tempfile.NamedTemporaryFile(delete=False) as temp_file: @@ -197,8 +207,15 @@ def _(self, connection: MongoDBConnection): return connection @setup_ssl.register(KafkaConnection) - def _(self, connection): + def _(self, connection) -> KafkaConnection: connection = cast(KafkaConnection, connection) + if connection.consumerConfigSSL: + connection.consumerConfig = { + **connection.consumerConfig, + "ssl.ca.location": getattr(self, "ca_consumer_config", None), + "ssl.key.location": getattr(self, "key_consumer_config", None), + "ssl.certificate.location": getattr(self, "cert_consumer_config", None), + } connection.schemaRegistryConfig["ssl.ca.location"] = self.ca_file_path connection.schemaRegistryConfig["ssl.key.location"] = self.key_file_path connection.schemaRegistryConfig[ @@ -208,7 +225,9 @@ def _(self, connection): @singledispatch -def check_ssl_and_init(_) -> Optional[SSLManager]: +def check_ssl_and_init( + _, *args, **kwargs # pylint: disable=unused-argument +) -> Optional[Union[SSLManager, List[SSLManager]]]: return None @@ -274,6 +293,38 @@ def _(connection): return None +@check_ssl_and_init.register(KafkaConnection) +def _(connection, *args, **kwargs): + + service_connection: KafkaConnection = cast(KafkaConnection, connection) + ssl_consumer_config: Optional[ + verifySSLConfig.SslConfig + ] = service_connection.consumerConfigSSL + ssl_schema_registry: Optional[ + verifySSLConfig.SslConfig + ] = service_connection.schemaRegistrySSL + + ssl_consumer_config_dict = {} + + if ssl_consumer_config: + ssl_consumer_config_dict = { + "ca_consumer_config": ssl_consumer_config.root.caCertificate, + "cert_consumer_config": ssl_consumer_config.root.sslCertificate, + "key_consumer_config": ssl_consumer_config.root.sslKey, + } + ssl_schema_registry_dict = {} + + if ssl_schema_registry: + ssl_schema_registry_dict = { + "ca_schema_registry": ssl_schema_registry.root.caCertificate, + "cert_schema_registry": ssl_schema_registry.root.sslCertificate, + "key_schema_registry": ssl_schema_registry.root.sslKey, + } + if ssl_consumer_config_dict or ssl_schema_registry_dict: + return SSLManager(**ssl_consumer_config_dict, **ssl_schema_registry_dict) + return None + + @check_ssl_and_init.register(PostgresConnection) @check_ssl_and_init.register(RedshiftConnection) @check_ssl_and_init.register(GreenplumConnection) diff --git a/ingestion/src/metadata/workflow/data_quality.py b/ingestion/src/metadata/workflow/data_quality.py index b2b5b878b3b9..ea08be6107c4 100644 --- a/ingestion/src/metadata/workflow/data_quality.py +++ b/ingestion/src/metadata/workflow/data_quality.py @@ -11,15 +11,10 @@ """ Workflow definition for the Data Quality """ -import traceback from typing import Optional from metadata.data_quality.processor.test_case_runner import TestCaseRunner from metadata.data_quality.source.test_suite import TestSuiteSource -from metadata.generated.schema.entity.services.connections.serviceConnection import ( - ServiceConnection, -) -from metadata.generated.schema.entity.services.databaseService import DatabaseService from metadata.generated.schema.tests.testSuite import ServiceType, TestSuite from metadata.ingestion.api.steps import Processor, Sink from metadata.ingestion.ometa.utils import model_str @@ -63,44 +58,11 @@ def _get_sink(self) -> Sink: def _get_test_runner_processor(self) -> Processor: return TestCaseRunner.create(self.config.model_dump(), self.metadata) - def _retrieve_service_connection_if_needed(self, service_type: ServiceType) -> None: - """Get service object from source config `entityFullyQualifiedName`""" - if ( - not self.config.source.serviceConnection - and not self.metadata.config.forceEntityOverwriting - ): - fully_qualified_name = ( - self.config.source.sourceConfig.config.entityFullyQualifiedName.root - ) - try: - service_name = fqn.split(fully_qualified_name)[0] - except IndexError as exc: - logger.debug(traceback.format_exc()) - raise IndexError( - f"Could not retrieve service name from entity fully qualified name {fully_qualified_name}: {exc}" - ) - try: - service: DatabaseService = self.metadata.get_by_name( - DatabaseService, service_name - ) - if not service: - raise ConnectionError( - f"Could not retrieve service with name `{service_name}`. " - "Typically caused by the `entityFullyQualifiedName` does not exists in OpenMetadata " - "or the JWT Token is invalid." - ) - - self.config.source.serviceConnection = ServiceConnection( - service.connection - ) - - except Exception as exc: - logger.debug(traceback.format_exc()) - logger.error( - f"Error getting service connection for service name [{service_name}]" - f" using the secrets manager provider [{self.metadata.config.secretsManagerProvider}]: {exc}" - ) - raise exc + def _retrieve_service_connection_if_needed(self, _: ServiceType) -> None: + """A test suite might require multiple connections (e.g., for logical test suites) + We'll skip this step and get the connections at runtime if they are not informed + in the YAML already. + """ def _get_ingestion_pipeline_service(self) -> Optional[T]: """ diff --git a/ingestion/tests/integration/datalake/conftest.py b/ingestion/tests/integration/datalake/conftest.py index febbc3fbb1f6..491606a58718 100644 --- a/ingestion/tests/integration/datalake/conftest.py +++ b/ingestion/tests/integration/datalake/conftest.py @@ -209,8 +209,11 @@ def run_ingestion(metadata, ingestion_config): @pytest.fixture(scope="class") def run_test_suite_workflow(run_ingestion, ingestion_config): workflow_config = deepcopy(DATA_QUALITY_CONFIG) - workflow_config["source"]["serviceConnection"] = ingestion_config["source"][ - "serviceConnection" + workflow_config["source"]["sourceConfig"]["config"]["serviceConnections"] = [ + { + "serviceName": ingestion_config["source"]["serviceName"], + "serviceConnection": ingestion_config["source"]["serviceConnection"], + } ] ingestion_workflow = TestSuiteWorkflow.create(workflow_config) ingestion_workflow.execute() @@ -229,8 +232,11 @@ def run_sampled_test_suite_workflow(metadata, run_ingestion, ingestion_config): ), ) workflow_config = deepcopy(DATA_QUALITY_CONFIG) - workflow_config["source"]["serviceConnection"] = ingestion_config["source"][ - "serviceConnection" + workflow_config["source"]["sourceConfig"]["config"]["serviceConnections"] = [ + { + "serviceName": ingestion_config["source"]["serviceName"], + "serviceConnection": ingestion_config["source"]["serviceConnection"], + } ] ingestion_workflow = TestSuiteWorkflow.create(workflow_config) ingestion_workflow.execute() @@ -259,8 +265,11 @@ def run_partitioned_test_suite_workflow(metadata, run_ingestion, ingestion_confi ), ) workflow_config = deepcopy(DATA_QUALITY_CONFIG) - workflow_config["source"]["serviceConnection"] = ingestion_config["source"][ - "serviceConnection" + workflow_config["source"]["sourceConfig"]["config"]["serviceConnections"] = [ + { + "serviceName": ingestion_config["source"]["serviceName"], + "serviceConnection": ingestion_config["source"]["serviceConnection"], + } ] ingestion_workflow = TestSuiteWorkflow.create(workflow_config) ingestion_workflow.execute() diff --git a/ingestion/tests/integration/integration_base.py b/ingestion/tests/integration/integration_base.py index 11fe6ee449aa..ab65a16089ae 100644 --- a/ingestion/tests/integration/integration_base.py +++ b/ingestion/tests/integration/integration_base.py @@ -162,7 +162,7 @@ "serviceConnection": {{ "config": {service_config} }}, - "sourceConfig": {{"config": {{"type":"Profiler"}}}} + "sourceConfig": {{"config": {{"type":"Profiler", "profileSample": 100}}}} }}, "processor": {{"type": "orm-profiler", "config": {{}}}}, "sink": {{"type": "metadata-rest", "config": {{}}}}, @@ -371,7 +371,7 @@ def get_create_test_suite( return CreateTestSuiteRequest( name=TestSuiteEntityName(name), description=Markdown(description), - executableEntityReference=FullyQualifiedEntityName(executable_entity_reference), + basicEntityReference=FullyQualifiedEntityName(executable_entity_reference), ) diff --git a/ingestion/tests/integration/ometa/test_ometa_custom_properties_api.py b/ingestion/tests/integration/ometa/test_ometa_custom_properties_api.py index 960a15adae92..cd8e7515fa6b 100644 --- a/ingestion/tests/integration/ometa/test_ometa_custom_properties_api.py +++ b/ingestion/tests/integration/ometa/test_ometa_custom_properties_api.py @@ -93,7 +93,7 @@ "description": "Rating of a table", "propertyType": {"name": "enum"}, "customPropertyConfig": { - "config": {"values": ["Good", "Average", "Bad"], "multiSelect": False}, + "config": {"values": ["Average", "Bad", "Good"], "multiSelect": False}, }, }, { diff --git a/ingestion/tests/integration/ometa/test_ometa_test_suite.py b/ingestion/tests/integration/ometa/test_ometa_test_suite.py index c9391066a7bb..5e4f7711cad3 100644 --- a/ingestion/tests/integration/ometa/test_ometa_test_suite.py +++ b/ingestion/tests/integration/ometa/test_ometa_test_suite.py @@ -100,7 +100,7 @@ def setUpClass(cls) -> None: description=Markdown( root="This is a test suite for the integration tests" ), - executableEntityReference=FullyQualifiedEntityName( + basicEntityReference=FullyQualifiedEntityName( "sample_data.ecommerce_db.shopify.dim_address" ), ) diff --git a/ingestion/tests/integration/ometa/test_ometa_user_api.py b/ingestion/tests/integration/ometa/test_ometa_user_api.py index fa716b9efbbe..e746a7d65e10 100644 --- a/ingestion/tests/integration/ometa/test_ometa_user_api.py +++ b/ingestion/tests/integration/ometa/test_ometa_user_api.py @@ -62,7 +62,9 @@ def setUpClass(cls) -> None: cls.user_1: User = cls.metadata.create_or_update( data=CreateUserRequest( - name="random.user.es", email="random.user.es@getcollate.io" + name="random.user.es", + email="random.user.es@getcollate.io", + description="test", ), ) @@ -198,3 +200,8 @@ def test_es_search_from_name(self): self.assertIsNone( self.metadata.get_reference_by_name(name="Organization", is_owner=True) ) + + # description should not affect in search + self.assertIsNone( + self.metadata.get_reference_by_name(name="test", is_owner=True) + ) diff --git a/ingestion/tests/integration/orm_profiler/test_orm_profiler_e2e.py b/ingestion/tests/integration/orm_profiler/test_orm_profiler_e2e.py index 2d1d976ec72e..d6ca5e8a7e0e 100644 --- a/ingestion/tests/integration/orm_profiler/test_orm_profiler_e2e.py +++ b/ingestion/tests/integration/orm_profiler/test_orm_profiler_e2e.py @@ -549,8 +549,7 @@ def test_workflow_values_partition(ingest, metadata, service_name): profile = metadata.get_latest_table_profile(table.fullyQualifiedName).profile assert profile.rowCount == 4.0 - # If we don't have any sample, default to 100 - assert profile.profileSample == 100.0 + assert profile.profileSample == None workflow_config["processor"] = { "type": "orm-profiler", diff --git a/ingestion/tests/integration/postgres/test_data_quality.py b/ingestion/tests/integration/postgres/test_data_quality.py index f37c9560ae75..36b1cb9a0ccc 100644 --- a/ingestion/tests/integration/postgres/test_data_quality.py +++ b/ingestion/tests/integration/postgres/test_data_quality.py @@ -9,6 +9,7 @@ from metadata.data_quality.api.models import TestCaseDefinition from metadata.generated.schema.entity.services.databaseService import DatabaseService from metadata.generated.schema.metadataIngestion.testSuitePipeline import ( + ServiceConnections, TestSuiteConfigType, TestSuitePipeline, ) @@ -55,9 +56,14 @@ def run_data_quality_workflow( config=TestSuitePipeline( type=TestSuiteConfigType.TestSuite, entityFullyQualifiedName=f"{db_service.fullyQualifiedName.root}.dvdrental.public.customer", + serviceConnections=[ + ServiceConnections( + serviceName=db_service.name.root, + serviceConnection=db_service.connection, + ) + ], ) ), - serviceConnection=db_service.connection, ), processor=Processor( type="orm-test-runner", diff --git a/ingestion/tests/integration/test_suite/test_workflow.py b/ingestion/tests/integration/test_suite/test_workflow.py index 240493bb104d..42e45887e3cc 100644 --- a/ingestion/tests/integration/test_suite/test_workflow.py +++ b/ingestion/tests/integration/test_suite/test_workflow.py @@ -134,7 +134,7 @@ def setUpClass(cls) -> None: cls.test_suite = cls.metadata.create_or_update_executable_test_suite( data=CreateTestSuiteRequest( name="test-suite", - executableEntityReference=cls.table_with_suite.fullyQualifiedName.root, + basicEntityReference=cls.table_with_suite.fullyQualifiedName.root, ) ) diff --git a/ingestion/tests/unit/data_quality/source/test_test_suite.py b/ingestion/tests/unit/data_quality/source/test_test_suite.py index 9359a6e3b1e0..b153162e689d 100644 --- a/ingestion/tests/unit/data_quality/source/test_test_suite.py +++ b/ingestion/tests/unit/data_quality/source/test_test_suite.py @@ -5,7 +5,11 @@ from metadata.data_quality.source.test_suite import TestSuiteSource from metadata.generated.schema.entity.data.table import Table +from metadata.generated.schema.entity.services.connections.database.postgresConnection import ( + PostgresConnection, +) from metadata.generated.schema.entity.services.databaseService import ( + DatabaseConnection, DatabaseServiceType, ) from metadata.generated.schema.metadataIngestion.workflow import ( @@ -72,6 +76,19 @@ def test_source_config(parameters, expected, monkeypatch): }, } monkeypatch.setattr(TestSuiteSource, "test_connection", Mock()) + monkeypatch.setattr( + TestSuiteSource, + "_get_table_service_connection", + Mock( + return_value=DatabaseConnection( + config=PostgresConnection( + username="foo", + hostPort="localhost:5432", + database="postgres", + ) + ) + ), + ) mock_metadata = Mock(spec=OpenMetadata) mock_metadata.get_by_name.return_value = Table( @@ -98,7 +115,7 @@ def test_source_config(parameters, expected, monkeypatch): ), ] mock_metadata.get_by_id.return_value = TestSuite( - name="test_suite", executable=True, id=UUID(int=0) + name="test_suite", basic=True, id=UUID(int=0) ) source = TestSuiteSource( diff --git a/ingestion/tests/unit/metadata/utils/test_entity_link.py b/ingestion/tests/unit/metadata/utils/test_entity_link.py index 840b22f56c3f..bd0bb95a9437 100644 --- a/ingestion/tests/unit/metadata/utils/test_entity_link.py +++ b/ingestion/tests/unit/metadata/utils/test_entity_link.py @@ -16,7 +16,11 @@ import pytest from antlr4.error.Errors import ParseCancellationException -from metadata.utils.entity_link import get_decoded_column, get_table_or_column_fqn +from metadata.utils.entity_link import ( + get_decoded_column, + get_table_fqn, + get_table_or_column_fqn, +) @pytest.mark.parametrize( @@ -154,3 +158,16 @@ def test_invalid_get_table_or_column_fqn(entity_link, error): """ with pytest.raises(error): get_table_or_column_fqn(entity_link) + + +@pytest.mark.parametrize( + "entity_link,expected", + [ + ( + "<#E::table::red.dev.dbt_jaffle.customers::columns::a>", + "red.dev.dbt_jaffle.customers", + ), + ], +) +def test_get_table_fqn(entity_link, expected): + assert get_table_fqn(entity_link) == expected diff --git a/ingestion/tests/unit/profiler/pandas/test_sample.py b/ingestion/tests/unit/profiler/pandas/test_sample.py index 731acfa2f08d..c25c79f980c2 100644 --- a/ingestion/tests/unit/profiler/pandas/test_sample.py +++ b/ingestion/tests/unit/profiler/pandas/test_sample.py @@ -166,7 +166,7 @@ def setUpClass(cls, mock_get_connection, mock_sample_get_connection) -> None: service_connection_config=DatalakeConnection(configSource={}), ometa_client=None, entity=cls.table_entity, - sample_config=SampleConfig(profile_sample=50.0), + sample_config=SampleConfig(profileSample=50.0), ) cls.datalake_profiler_interface = PandasProfilerInterface( service_connection_config=DatalakeConnection(configSource={}), @@ -198,7 +198,7 @@ def test_random_sampler(self, _): service_connection_config=DatalakeConnection(configSource={}), ometa_client=None, entity=self.table_entity, - sample_config=SampleConfig(profile_sample=50.0), + sample_config=SampleConfig(profileSample=50.0), ) random_sample = sampler.get_dataset() res = sum(len(r) for r in random_sample) @@ -232,7 +232,7 @@ def test_sample_property(self, *_): service_connection_config=DatalakeConnection(configSource={}), ometa_client=None, entity=self.table_entity, - sample_config=SampleConfig(profile_sample=50.0), + sample_config=SampleConfig(profileSample=50.0), ) datalake_profiler_interface = PandasProfilerInterface( service_connection_config=DatalakeConnection(configSource={}), @@ -309,7 +309,7 @@ def test_sample_data(self, *_): service_connection_config=DatalakeConnection(configSource={}), ometa_client=None, entity=self.table_entity, - sample_config=SampleConfig(profile_sample=50.0), + sample_config=SampleConfig(profileSample=50.0), ) sample_data = sampler.fetch_sample_data() @@ -337,7 +337,7 @@ def test_sample_from_user_query(self, *_): service_connection_config=DatalakeConnection(configSource={}), ometa_client=None, entity=self.table_entity, - default_sample_config=SampleConfig(profile_sample=50.0), + default_sample_config=SampleConfig(profileSample=50.0), sample_query="`age` > 30", ) sample_data = sampler.fetch_sample_data() diff --git a/ingestion/tests/unit/profiler/sqlalchemy/bigquery/test_bigquery_sampling.py b/ingestion/tests/unit/profiler/sqlalchemy/bigquery/test_bigquery_sampling.py index 03152e3ee958..5c719d7fc753 100644 --- a/ingestion/tests/unit/profiler/sqlalchemy/bigquery/test_bigquery_sampling.py +++ b/ingestion/tests/unit/profiler/sqlalchemy/bigquery/test_bigquery_sampling.py @@ -110,7 +110,7 @@ def test_sampling(self, sampler_mock): ometa_client=None, entity=self.table_entity, sample_config=SampleConfig( - profile_sample_type=ProfileSampleType.PERCENTAGE, profile_sample=50.0 + profileSampleType=ProfileSampleType.PERCENTAGE, profileSample=50.0 ), table_type=TableType.Regular, ) @@ -127,20 +127,31 @@ def test_sampling_for_views(self, sampler_mock): """ Test view sampling """ + view_entity = Table( + id=uuid4(), + name="user", + columns=[ + EntityColumn( + name=ColumnName("id"), + dataType=DataType.INT, + ), + ], + tableType=TableType.View, + ) + sampler = BigQuerySampler( service_connection_config=self.bq_conn, ometa_client=None, - entity=self.table_entity, + entity=view_entity, sample_config=SampleConfig( - profile_sample_type=ProfileSampleType.PERCENTAGE, profile_sample=50.0 + profileSampleType=ProfileSampleType.PERCENTAGE, profileSample=50.0 ), - table_type=TableType.View, ) query: CTE = sampler.get_sample_query() expected_query = ( - "WITH users_rnd AS \n(SELECT users.id AS id, ABS(RANDOM()) * 100 %% 100 AS random \n" - "FROM users)\n SELECT users_rnd.id, users_rnd.random \n" - "FROM users_rnd \nWHERE users_rnd.random <= 50.0" + 'WITH "9bc65c2abec141778ffaa729489f3e87_rnd" AS \n(SELECT users.id AS id, ABS(RANDOM()) * 100 %% 100 AS random \n' + 'FROM users)\n SELECT "9bc65c2abec141778ffaa729489f3e87_rnd".id, "9bc65c2abec141778ffaa729489f3e87_rnd".random \n' + 'FROM "9bc65c2abec141778ffaa729489f3e87_rnd" \nWHERE "9bc65c2abec141778ffaa729489f3e87_rnd".random <= 50.0' ) assert ( expected_query.casefold() @@ -151,12 +162,24 @@ def test_sampling_view_with_partition(self, sampler_mock): """ Test view sampling with partition """ + view_entity = Table( + id=uuid4(), + name="user", + columns=[ + EntityColumn( + name=ColumnName("id"), + dataType=DataType.INT, + ), + ], + tableType=TableType.View, + ) + sampler = BigQuerySampler( service_connection_config=self.bq_conn, ometa_client=None, - entity=self.table_entity, + entity=view_entity, sample_config=SampleConfig( - profile_sample_type=ProfileSampleType.PERCENTAGE, profile_sample=50.0 + profileSampleType=ProfileSampleType.PERCENTAGE, profileSample=50.0 ), partition_details=PartitionProfilerConfig( enablePartitioning=True, @@ -168,9 +191,9 @@ def test_sampling_view_with_partition(self, sampler_mock): ) query: CTE = sampler.get_sample_query() expected_query = ( - "WITH users_rnd AS \n(SELECT users.id AS id, ABS(RANDOM()) * 100 %% 100 AS random \n" - "FROM users \nWHERE id in ('1', '2'))\n SELECT users_rnd.id, users_rnd.random \n" - "FROM users_rnd \nWHERE users_rnd.random <= 50.0" + 'WITH "9bc65c2abec141778ffaa729489f3e87_rnd" AS \n(SELECT users.id AS id, ABS(RANDOM()) * 100 %% 100 AS random \n' + "FROM users \nWHERE id in ('1', '2'))\n SELECT \"9bc65c2abec141778ffaa729489f3e87_rnd\".id, \"9bc65c2abec141778ffaa729489f3e87_rnd\".random \n" + 'FROM "9bc65c2abec141778ffaa729489f3e87_rnd" \nWHERE "9bc65c2abec141778ffaa729489f3e87_rnd".random <= 50.0' ) assert ( expected_query.casefold() @@ -186,7 +209,7 @@ def test_sampling_with_partition(self, sampler_mock): ometa_client=None, entity=self.table_entity, sample_config=SampleConfig( - profile_sample_type=ProfileSampleType.PERCENTAGE, profile_sample=50.0 + profileSampleType=ProfileSampleType.PERCENTAGE, profileSample=50.0 ), partition_details=PartitionProfilerConfig( enablePartitioning=True, diff --git a/ingestion/tests/unit/profiler/sqlalchemy/mssql/test_mssql_sampling.py b/ingestion/tests/unit/profiler/sqlalchemy/mssql/test_mssql_sampling.py index feab9863468b..fb03b6d7c530 100644 --- a/ingestion/tests/unit/profiler/sqlalchemy/mssql/test_mssql_sampling.py +++ b/ingestion/tests/unit/profiler/sqlalchemy/mssql/test_mssql_sampling.py @@ -82,7 +82,7 @@ def test_omit_sampling_method_type(self, sampler_mock): ometa_client=None, entity=self.table_entity, sample_config=SampleConfig( - profile_sample_type=ProfileSampleType.PERCENTAGE, profile_sample=50.0 + profileSampleType=ProfileSampleType.PERCENTAGE, profileSample=50.0 ), ) query: CTE = sampler.get_sample_query() @@ -105,7 +105,7 @@ def test_row_sampling(self, sampler_mock): ometa_client=None, entity=self.table_entity, sample_config=SampleConfig( - profile_sample_type=ProfileSampleType.ROWS, profile_sample=50 + profileSampleType=ProfileSampleType.ROWS, profileSample=50 ), ) query: CTE = sampler.get_sample_query() @@ -128,8 +128,8 @@ def test_sampling_with_partition(self, sampler_mock): ometa_client=None, entity=self.table_entity, sample_config=SampleConfig( - profile_sample_type=ProfileSampleType.PERCENTAGE, - profile_sample=50.0, + profileSampleType=ProfileSampleType.PERCENTAGE, + profileSample=50.0, ), partition_details=PartitionProfilerConfig( enablePartitioning=True, diff --git a/ingestion/tests/unit/profiler/sqlalchemy/postgres/test_postgres_sampling.py b/ingestion/tests/unit/profiler/sqlalchemy/postgres/test_postgres_sampling.py index f6c9a421a0d2..c603d060283a 100644 --- a/ingestion/tests/unit/profiler/sqlalchemy/postgres/test_postgres_sampling.py +++ b/ingestion/tests/unit/profiler/sqlalchemy/postgres/test_postgres_sampling.py @@ -81,8 +81,8 @@ def test_sampling_default(self, sampler_mock): ometa_client=None, entity=self.table_entity, sample_config=SampleConfig( - profile_sample_type=ProfileSampleType.PERCENTAGE, - profile_sample=50.0, + profileSampleType=ProfileSampleType.PERCENTAGE, + profileSample=50.0, ), ) query: CTE = sampler.get_sample_query() @@ -107,9 +107,9 @@ def test_sampling(self, sampler_mock): ometa_client=None, entity=self.table_entity, sample_config=SampleConfig( - profile_sample_type=ProfileSampleType.PERCENTAGE, - profile_sample=50.0, - sampling_method_type=sampling_method_type, + profileSampleType=ProfileSampleType.PERCENTAGE, + profileSample=50.0, + samplingMethodType=sampling_method_type, ), ) query: CTE = sampler.get_sample_query() @@ -128,7 +128,7 @@ def test_sampling_with_partition(self, sampler_mock): ometa_client=None, entity=self.table_entity, sample_config=SampleConfig( - profile_sample_type=ProfileSampleType.PERCENTAGE, profile_sample=50.0 + profileSampleType=ProfileSampleType.PERCENTAGE, profileSample=50.0 ), partition_details=PartitionProfilerConfig( enablePartitioning=True, diff --git a/ingestion/tests/unit/profiler/sqlalchemy/snowflake/test_snowflake_sampling.py b/ingestion/tests/unit/profiler/sqlalchemy/snowflake/test_snowflake_sampling.py index bdee9a860646..dc95877e72bc 100644 --- a/ingestion/tests/unit/profiler/sqlalchemy/snowflake/test_snowflake_sampling.py +++ b/ingestion/tests/unit/profiler/sqlalchemy/snowflake/test_snowflake_sampling.py @@ -80,7 +80,7 @@ def test_omit_sampling_method_type(self, sampler_mock): ometa_client=None, entity=self.table_entity, sample_config=SampleConfig( - profile_sample_type=ProfileSampleType.PERCENTAGE, profile_sample=50.0 + profileSampleType=ProfileSampleType.PERCENTAGE, profileSample=50.0 ), ) query: CTE = sampler.get_sample_query() @@ -107,9 +107,9 @@ def test_specify_sampling_method_type(self, sampler_mock): ometa_client=None, entity=self.table_entity, sample_config=SampleConfig( - profile_sample_type=ProfileSampleType.PERCENTAGE, - profile_sample=50.0, - sampling_method_type=sampling_method_type, + profileSampleType=ProfileSampleType.PERCENTAGE, + profileSample=50.0, + samplingMethodType=sampling_method_type, ), ) query: CTE = sampler.get_sample_query() @@ -132,7 +132,7 @@ def test_row_sampling(self, sampler_mock): ometa_client=None, entity=self.table_entity, sample_config=SampleConfig( - profile_sample_type=ProfileSampleType.ROWS, profile_sample=50 + profileSampleType=ProfileSampleType.ROWS, profileSample=50 ), ) query: CTE = sampler.get_sample_query() @@ -155,8 +155,8 @@ def test_sampling_with_partition(self, sampler_mock): ometa_client=None, entity=self.table_entity, sample_config=SampleConfig( - profile_sample_type=ProfileSampleType.PERCENTAGE, - profile_sample=50.0, + profileSampleType=ProfileSampleType.PERCENTAGE, + profileSample=50.0, ), partition_details=PartitionProfilerConfig( enablePartitioning=True, diff --git a/ingestion/tests/unit/profiler/sqlalchemy/test_runner.py b/ingestion/tests/unit/profiler/sqlalchemy/test_runner.py index 880cc6ed94fe..117916a9a583 100644 --- a/ingestion/tests/unit/profiler/sqlalchemy/test_runner.py +++ b/ingestion/tests/unit/profiler/sqlalchemy/test_runner.py @@ -90,7 +90,7 @@ def setUpClass(cls) -> None: service_connection_config=Mock(), ometa_client=None, entity=None, - sample_config=SampleConfig(profile_sample=50.0), + sample_config=SampleConfig(profileSample=50.0), ) cls.dataset = sampler.get_dataset() diff --git a/ingestion/tests/unit/profiler/sqlalchemy/test_sample.py b/ingestion/tests/unit/profiler/sqlalchemy/test_sample.py index 71f5c0aa453e..3deabc317a67 100644 --- a/ingestion/tests/unit/profiler/sqlalchemy/test_sample.py +++ b/ingestion/tests/unit/profiler/sqlalchemy/test_sample.py @@ -104,7 +104,7 @@ def setUpClass(cls, sampler_mock) -> None: service_connection_config=cls.sqlite_conn, ometa_client=None, entity=None, - sample_config=SampleConfig(profile_sample=50.0), + sample_config=SampleConfig(profileSample=50.0), ) cls.dataset = cls.sampler.get_dataset() cls.sqa_profiler_interface = SQAProfilerInterface( @@ -361,7 +361,7 @@ def test_sample_from_user_query(self, sampler_mock): service_connection_config=self.sqlite_conn, ometa_client=None, entity=None, - sample_config=SampleConfig(profile_sample=50.0), + sample_config=SampleConfig(profileSample=50.0), sample_query=stmt, ) sample_data = sampler.fetch_sample_data() diff --git a/ingestion/tests/unit/profiler/test_entity_fetcher.py b/ingestion/tests/unit/profiler/test_entity_fetcher.py new file mode 100644 index 000000000000..9e1d475cd661 --- /dev/null +++ b/ingestion/tests/unit/profiler/test_entity_fetcher.py @@ -0,0 +1,88 @@ +# Copyright 2021 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Validate entity fetcher filtering strategies +""" +import uuid + +from metadata.generated.schema.entity.data.table import Table, TableType +from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import ( + OpenMetadataConnection, +) +from metadata.generated.schema.metadataIngestion.databaseServiceAutoClassificationPipeline import ( + DatabaseServiceAutoClassificationPipeline, +) +from metadata.generated.schema.metadataIngestion.workflow import ( + OpenMetadataWorkflowConfig, + Source, + SourceConfig, + WorkflowConfig, +) +from metadata.ingestion.api.status import Status +from metadata.profiler.source.fetcher.fetcher_strategy import DatabaseFetcherStrategy + +VIEW = Table( + id=uuid.uuid4(), + name="view", + columns=[], + tableType=TableType.View, +) + +TABLE = Table( + id=uuid.uuid4(), + name="table", + columns=[], + tableType=TableType.Regular, +) + + +def get_db_fetcher(source_config): + """Fetch database""" + workflow_config = OpenMetadataWorkflowConfig( + source=Source( + type="mysql", + serviceName="mysql", + sourceConfig=SourceConfig( + config=source_config, + ), + ), + workflowConfig=WorkflowConfig( + openMetadataServerConfig=OpenMetadataConnection( + hostPort="localhost:8585/api", + ) + ), + ) + return DatabaseFetcherStrategy( + config=workflow_config, + metadata=..., + global_profiler_config=..., + status=Status(), + ) + + +def test_include_views(): + """Validate we can include/exclude views""" + config = DatabaseServiceAutoClassificationPipeline( + includeViews=False, + ) + fetcher = get_db_fetcher(config) + + assert fetcher._filter_views(VIEW) + assert not fetcher._filter_views(TABLE) + + config = DatabaseServiceAutoClassificationPipeline( + includeViews=True, + ) + fetcher = get_db_fetcher(config) + + assert not fetcher._filter_views(VIEW) + assert not fetcher._filter_views(TABLE) diff --git a/ingestion/tests/unit/profiler/test_profiler_interface.py b/ingestion/tests/unit/profiler/test_profiler_interface.py index a0336bcd31ee..ee4d3c566ff2 100644 --- a/ingestion/tests/unit/profiler/test_profiler_interface.py +++ b/ingestion/tests/unit/profiler/test_profiler_interface.py @@ -159,8 +159,8 @@ def test_get_profile_sample_configs(self): source_config = DatabaseServiceProfilerPipeline() expected = SampleConfig( - profile_sample=11, - profile_sample_type=ProfileSampleType.PERCENTAGE, + profileSample=11, + profileSampleType=ProfileSampleType.PERCENTAGE, ) actual = get_profile_sample_config( entity=self.table, @@ -168,9 +168,9 @@ def test_get_profile_sample_configs(self): database_entity=self.database_entity, entity_config=None, default_sample_config=SampleConfig( - profile_sample=source_config.profileSample, - profile_sample_type=source_config.profileSampleType, - sampling_method_type=source_config.samplingMethodType, + profileSample=source_config.profileSample, + profileSampleType=source_config.profileSampleType, + samplingMethodType=source_config.samplingMethodType, ), ) self.assertEqual(expected, actual) @@ -181,8 +181,8 @@ def test_get_profile_sample_configs(self): fullyQualifiedName="demo", ) expected = SampleConfig( - profile_sample=11, - profile_sample_type=ProfileSampleType.PERCENTAGE, + profileSample=11, + profileSampleType=ProfileSampleType.PERCENTAGE, ) actual = get_profile_sample_config( entity=self.table, @@ -190,17 +190,17 @@ def test_get_profile_sample_configs(self): database_entity=self.database_entity, entity_config=profiler, default_sample_config=SampleConfig( - profile_sample=source_config.profileSample, - profile_sample_type=source_config.profileSampleType, - sampling_method_type=source_config.samplingMethodType, + profileSample=source_config.profileSample, + profileSampleType=source_config.profileSampleType, + samplingMethodType=source_config.samplingMethodType, ), ) self.assertEqual(expected, actual) profiler = None expected = SampleConfig( - profile_sample=22, - profile_sample_type=ProfileSampleType.PERCENTAGE, + profileSample=22, + profileSampleType=ProfileSampleType.PERCENTAGE, ) table_copy = deepcopy(self.table) table_copy.tableProfilerConfig = None @@ -210,9 +210,9 @@ def test_get_profile_sample_configs(self): database_entity=self.database_entity, entity_config=profiler, default_sample_config=SampleConfig( - profile_sample=source_config.profileSample, - profile_sample_type=source_config.profileSampleType, - sampling_method_type=source_config.samplingMethodType, + profileSample=source_config.profileSample, + profileSampleType=source_config.profileSampleType, + samplingMethodType=source_config.samplingMethodType, ), ) self.assertEqual(expected, actual) diff --git a/ingestion/tests/unit/test_databricks_lineage.py b/ingestion/tests/unit/test_databricks_lineage.py index 3f710316b669..8647f63f3603 100644 --- a/ingestion/tests/unit/test_databricks_lineage.py +++ b/ingestion/tests/unit/test_databricks_lineage.py @@ -128,23 +128,10 @@ def __init__(self, methodName) -> None: super().__init__(methodName) config = OpenMetadataWorkflowConfig.model_validate(mock_databricks_config) - self.databricks = DatabricksLineageSource.create( - mock_databricks_config["source"], - config.workflowConfig.openMetadataServerConfig, - ) - - @patch( - "metadata.ingestion.source.database.databricks.client.DatabricksClient.list_query_history" - ) - def test_get_table_query(self, list_query_history): - list_query_history.return_value = mock_data - results = self.databricks.get_table_query() - query_list = [] - for result in results: - if isinstance(result, TableQuery): - query_list.append(result) - for _, (expected, original) in enumerate( - zip(EXPECTED_DATABRICKS_DETAILS, query_list) + with patch( + "metadata.ingestion.source.database.databricks.lineage.DatabricksLineageSource.test_connection" ): - expected.analysisDate = original.analysisDate = datetime.now() - self.assertEqual(expected, original) + self.databricks = DatabricksLineageSource.create( + mock_databricks_config["source"], + config.workflowConfig.openMetadataServerConfig, + ) diff --git a/ingestion/tests/unit/test_databricks_schema.py b/ingestion/tests/unit/test_databricks_schema.py deleted file mode 100644 index a5d9ed6b1122..000000000000 --- a/ingestion/tests/unit/test_databricks_schema.py +++ /dev/null @@ -1,21 +0,0 @@ -import pytest - -from metadata.ingestion.source.database.databricks.metadata import format_schema_name - - -@pytest.mark.parametrize( - "input_schema, expected_schema", - [ - ("test_schema-name", "`test_schema-name`"), - ("test_schema_name", "test_schema_name"), - ("schema-with-hyphen", "`schema-with-hyphen`"), - ("schema_with_underscore", "schema_with_underscore"), - ("validSchema", "validSchema"), - ], -) -def test_schema_name_sanitization(input_schema, expected_schema): - """ - Test sanitization of schema names by adding backticks only around hyphenated names. - """ - sanitized_schema = format_schema_name(input_schema) - assert sanitized_schema == expected_schema diff --git a/ingestion/tests/unit/test_dbt.py b/ingestion/tests/unit/test_dbt.py index 1c2db6edc0a2..46022a87bd59 100644 --- a/ingestion/tests/unit/test_dbt.py +++ b/ingestion/tests/unit/test_dbt.py @@ -8,7 +8,11 @@ from unittest import TestCase from unittest.mock import MagicMock, patch -from dbt_artifacts_parser.parser import parse_catalog, parse_manifest, parse_run_results +from collate_dbt_artifacts_parser.parser import ( + parse_catalog, + parse_manifest, + parse_run_results, +) from pydantic import AnyUrl from metadata.generated.schema.entity.data.table import Column, DataModel, Table @@ -53,6 +57,7 @@ "dbtRunResultsFilePath": "sample/dbt_files/run_results.json", "dbtSourcesFilePath": "sample/dbt_files/sources.json", }, + "dbtUpdateOwners": True, } }, }, @@ -675,6 +680,7 @@ def check_yield_datamodel(self, dbt_objects, expected_data_models): data_model_link.right.table_entity.fullyQualifiedName.root, EXPECTED_DATA_MODEL_FQNS, ) + self.check_process_dbt_owners(data_model_link.right) data_model_list.append(data_model_link.right.datamodel) for _, (expected, original) in enumerate( @@ -682,6 +688,12 @@ def check_yield_datamodel(self, dbt_objects, expected_data_models): ): self.assertEqual(expected, original) + def check_process_dbt_owners(self, data_model_link): + process_dbt_owners = self.dbt_source_obj.process_dbt_owners(data_model_link) + for entity in process_dbt_owners: + entity_owner = entity.right.new_entity.owners + self.assertEqual(entity_owner, MOCK_OWNER) + @patch("metadata.ingestion.ometa.mixins.es_mixin.ESMixin.es_search_from_fqn") def test_upstream_nodes_for_lineage(self, es_search_from_fqn): expected_upstream_nodes = [ diff --git a/ingestion/tests/unit/test_sql_lineage.py b/ingestion/tests/unit/test_sql_lineage.py index 4199cf84a5f8..ffe07a0557a1 100644 --- a/ingestion/tests/unit/test_sql_lineage.py +++ b/ingestion/tests/unit/test_sql_lineage.py @@ -229,14 +229,39 @@ def test_table_name_from_query(self): def test_query_masker(self): query_list = [ - """SELECT * FROM user WHERE id=1234 AND name='Alice' AND birthdate=DATE '2023-01-01';""", - """insert into user values ('mayur',123,'my random address 1'), ('mayur',123,'my random address 1');""", - """SELECT * FROM user WHERE address = '5th street' and name = 'john';""", - """INSERT INTO user VALUE ('John', '19', '5TH Street');""", - """SELECT CASE address WHEN '5th Street' THEN 'CEO' ELSE 'Unknown' END AS person FROM user;""", - """with test as (SELECT CASE address WHEN '5th Street' THEN 'CEO' ELSE 'Unknown' END AS person FROM user) select * from test;""", - """select * from (select * from (SELECT CASE address WHEN '5th Street' THEN 'CEO' ELSE 'Unknown' END AS person FROM user));""", - """select * from users where id > 2 and name <> 'pere';""", + ( + """SELECT * FROM user WHERE id=1234 AND name='Alice' AND birthdate=DATE '2023-01-01';""", + Dialect.MYSQL.value, + ), + ( + """insert into user values ('mayur',123,'my random address 1'), ('mayur',123,'my random address 1');""", + Dialect.ANSI.value, + ), + ( + """SELECT * FROM user WHERE address = '5th street' and name = 'john';""", + Dialect.ANSI.value, + ), + ( + """INSERT INTO user VALUE ('John', '19', '5TH Street');""", + Dialect.ANSI.value, + ), + ( + """SELECT CASE address WHEN '5th Street' THEN 'CEO' ELSE 'Unknown' END AS person FROM user;""", + Dialect.ANSI.value, + ), + ( + """with test as (SELECT CASE address WHEN '5th Street' THEN 'CEO' ELSE 'Unknown' END AS person FROM user) select * from test;""", + Dialect.ANSI.value, + ), + ( + """select * from (select * from (SELECT CASE address WHEN '5th Street' THEN 'CEO' ELSE 'Unknown' END AS person FROM user));""", + Dialect.ANSI.value, + ), + ( + """select * from users where id > 2 and name <> 'pere';""", + Dialect.ANSI.value, + ), + ("""select * from users where id > 2 and name <> 'pere';""", "random"), ] expected_query_list = [ @@ -248,7 +273,8 @@ def test_query_masker(self): """with test as (SELECT CASE address WHEN ? THEN ? ELSE ? END AS person FROM user) select * from test;""", """select * from (select * from (SELECT CASE address WHEN ? THEN ? ELSE ? END AS person FROM user));""", """select * from users where id > ? and name <> ?;""", + """select * from users where id > ? and name <> ?;""", ] for i, query in enumerate(query_list): - self.assertEqual(mask_query(query), expected_query_list[i]) + self.assertEqual(mask_query(query[0], query[1]), expected_query_list[i]) diff --git a/ingestion/tests/unit/topology/dashboard/test_looker.py b/ingestion/tests/unit/topology/dashboard/test_looker.py index b5a2b2b0ab46..5f0ca6b485b4 100644 --- a/ingestion/tests/unit/topology/dashboard/test_looker.py +++ b/ingestion/tests/unit/topology/dashboard/test_looker.py @@ -47,6 +47,7 @@ from metadata.generated.schema.type.usageDetails import UsageDetails, UsageStats from metadata.generated.schema.type.usageRequest import UsageRequest from metadata.ingestion.api.steps import InvalidSourceException +from metadata.ingestion.lineage.models import Dialect from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.dashboard.dashboard_service import DashboardUsage from metadata.ingestion.source.dashboard.looker.metadata import LookerSource @@ -124,6 +125,13 @@ serviceType=DashboardServiceType.Looker, ) +EXPECTED_PARSED_VIEWS = { + "v1": "table1", + "v2": "select * from v2", + "v3": "select * from (select * from v2)", + "v4": "select * from (select * from (select * from v2)) inner join (table1)", +} + class LookerUnitTest(TestCase): """ @@ -292,13 +300,33 @@ def test_clean_table_name(self): """ Check table cleaning """ - self.assertEqual(self.looker._clean_table_name("MY_TABLE"), "my_table") + self.assertEqual( + self.looker._clean_table_name("MY_TABLE", Dialect.MYSQL), "my_table" + ) - self.assertEqual(self.looker._clean_table_name(" MY_TABLE "), "my_table") + self.assertEqual( + self.looker._clean_table_name(" MY_TABLE ", Dialect.REDSHIFT), "my_table" + ) - self.assertEqual(self.looker._clean_table_name(" my_table"), "my_table") + self.assertEqual( + self.looker._clean_table_name(" my_table", Dialect.SNOWFLAKE), "my_table" + ) + + self.assertEqual( + self.looker._clean_table_name("TABLE AS ALIAS", Dialect.BIGQUERY), "table" + ) + + self.assertEqual( + self.looker._clean_table_name( + "`project_id.dataset_id.table_id` AS ALIAS", Dialect.BIGQUERY + ), + "project_id.dataset_id.table_id", + ) - self.assertEqual(self.looker._clean_table_name("TABLE AS ALIAS"), "table") + self.assertEqual( + self.looker._clean_table_name("`db.schema.table`", Dialect.POSTGRES), + "`db.schema.table`", + ) def test_render_table_name(self): """ @@ -539,3 +567,33 @@ def test_yield_dashboard_usage(self): self.assertIsNotNone( list(self.looker.yield_dashboard_usage(MOCK_LOOKER_DASHBOARD))[0].left ) + + def test_derived_view_references(self): + """ + Validate if we can find derived references in a SQL query + and replace them with their actual values + """ + # pylint: disable=protected-access + self.looker._parsed_views.update( + { + "v1": "table1", + "v2": "select * from v2", + } + ) + self.looker._unparsed_views.update( + { + "v3": "select * from ${v2.SQL_TABLE_NAME}", + "v4": "select * from ${v3.SQL_TABLE_NAME} inner join ${v1.SQL_TABLE_NAME}", + } + ) + self.looker._derived_dependencies.add_edges_from( + [ + ("v3", "v2"), + ("v4", "v3"), + ("v4", "v1"), + ] + ) + list(self.looker.build_lineage_for_unparsed_views()) + + self.assertEqual(self.looker._parsed_views, EXPECTED_PARSED_VIEWS) + self.assertEqual(self.looker._unparsed_views, {}) diff --git a/ingestion/tests/unit/topology/database/test_postgres.py b/ingestion/tests/unit/topology/database/test_postgres.py index 86da89043473..8848cc55960d 100644 --- a/ingestion/tests/unit/topology/database/test_postgres.py +++ b/ingestion/tests/unit/topology/database/test_postgres.py @@ -320,14 +320,15 @@ def test_datatype(self): @patch("sqlalchemy.engine.base.Engine") def test_get_version_info(self, engine): - engine.execute.return_value = [["15.3 (Debian 15.3-1.pgdg110+1)"]] - self.assertEqual("15.3", get_postgres_version(engine)) + # outdated with a switch to get_server_version_num instead of get_+server_version + # engine.execute.return_value = [["15.3 (Debian 15.3-1.pgdg110+1)"]] + # self.assertEqual("15.3", get_postgres_version(engine)) - engine.execute.return_value = [["11.16"]] - self.assertEqual("11.16", get_postgres_version(engine)) + engine.execute.return_value = [["110016"]] + self.assertEqual("110016", get_postgres_version(engine)) - engine.execute.return_value = [["9.6.24"]] - self.assertEqual("9.6.24", get_postgres_version(engine)) + engine.execute.return_value = [["90624"]] + self.assertEqual("90624", get_postgres_version(engine)) engine.execute.return_value = [[]] self.assertIsNone(get_postgres_version(engine)) diff --git a/ingestion/tests/unit/topology/database/test_salesforce.py b/ingestion/tests/unit/topology/database/test_salesforce.py index 8c2ceb780f39..34dc639a7618 100644 --- a/ingestion/tests/unit/topology/database/test_salesforce.py +++ b/ingestion/tests/unit/topology/database/test_salesforce.py @@ -117,7 +117,7 @@ dataTypeDisplay="textarea", description="Contact Description", fullyQualifiedName=None, - tags=None, + tags=[], constraint=Constraint.NULL, ordinalPosition=1, jsonSchema=None, @@ -136,7 +136,7 @@ dataTypeDisplay="reference", description="Owner ID", fullyQualifiedName=None, - tags=None, + tags=[], constraint=Constraint.NOT_NULL, ordinalPosition=2, jsonSchema=None, @@ -155,7 +155,7 @@ dataTypeDisplay="phone", description="Phone", fullyQualifiedName=None, - tags=None, + tags=[], constraint=Constraint.NOT_NULL, ordinalPosition=3, jsonSchema=None, @@ -174,7 +174,7 @@ dataTypeDisplay="anytype", description="Created By ID", fullyQualifiedName=None, - tags=None, + tags=[], constraint=Constraint.NOT_NULL, ordinalPosition=4, jsonSchema=None, @@ -451,8 +451,12 @@ def __init__(self, methodName, salesforce, test_connection) -> None: "database_schema" ] = MOCK_DATABASE_SCHEMA - def test_table_column(self): - result = self.salesforce_source.get_columns(SALESFORCE_FIELDS) + @patch( + "metadata.ingestion.source.database.salesforce.metadata.SalesforceSource.get_table_column_description" + ) + def test_table_column(self, get_table_column_description): + get_table_column_description.return_value = None + result = self.salesforce_source.get_columns("TEST_TABLE", SALESFORCE_FIELDS) assert EXPECTED_COLUMN_VALUE == result def test_column_type(self): diff --git a/ingestion/tests/utils/sqa.py b/ingestion/tests/utils/sqa.py index 19770c3f2969..73e370b7173c 100644 --- a/ingestion/tests/utils/sqa.py +++ b/ingestion/tests/utils/sqa.py @@ -18,6 +18,15 @@ class User(Base): age = Column(Integer) +class UserWithLongName(Base): + __tablename__ = "u" * 63 # Keep a max length name of 63 chars (max for Postgres) + id = Column(Integer, primary_key=True) + name = Column(String(256)) + fullname = Column(String(256)) + nickname = Column(String(256)) + age = Column(Integer) + + class SQATestUtils: def __init__(self, connection_url: str): self.connection_url = connection_url @@ -34,14 +43,16 @@ def load_data(self, data: Sequence[DeclarativeMeta]): self.session.commit() def load_user_data(self): - data = [ - User(name="John", fullname="John Doe", nickname="johnny b goode", age=30), # type: ignore - User(name="Jane", fullname="Jone Doe", nickname=None, age=31), # type: ignore - ] * 20 - self.load_data(data) + for clz in (User, UserWithLongName): + data = [ + clz(name="John", fullname="John Doe", nickname="johnny b goode", age=30), # type: ignore + clz(name="Jane", fullname="Jone Doe", nickname=None, age=31), # type: ignore + ] * 20 + self.load_data(data) def create_user_table(self): User.__table__.create(bind=self.session.get_bind()) + UserWithLongName.__table__.create(bind=self.session.get_bind()) def close(self): self.session.close() diff --git a/openmetadata-airflow-apis/pyproject.toml b/openmetadata-airflow-apis/pyproject.toml index 349b97021ce2..ae221348681f 100644 --- a/openmetadata-airflow-apis/pyproject.toml +++ b/openmetadata-airflow-apis/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" # since it helps us organize and isolate version management [project] name = "openmetadata_managed_apis" -version = "1.6.0.0.dev0" +version = "1.6.4.0" readme = "README.md" authors = [ {name = "OpenMetadata Committers"} diff --git a/openmetadata-airflow-apis/tests/unit/ingestion_pipeline/test_deploy.py b/openmetadata-airflow-apis/tests/unit/ingestion_pipeline/test_deploy.py index 57d14b33c91f..329e7b3cf936 100644 --- a/openmetadata-airflow-apis/tests/unit/ingestion_pipeline/test_deploy.py +++ b/openmetadata-airflow-apis/tests/unit/ingestion_pipeline/test_deploy.py @@ -15,8 +15,6 @@ import uuid from unittest.mock import patch -from openmetadata_managed_apis.operations.deploy import dump_with_safe_jwt - from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import ( AuthProvider, OpenMetadataConnection, @@ -55,9 +53,11 @@ ) -@patch.dict(os.environ, {"AWS_DEFAULT_REGION": "us-east-2"}) +@patch.dict(os.environ, {"AWS_DEFAULT_REGION": "us-east-2", "AIRFLOW_HOME": "/tmp"}) def test_deploy_ingestion_pipeline(): """We can dump an ingestion pipeline to a file without exposing secrets""" + from openmetadata_managed_apis.operations.deploy import dump_with_safe_jwt + # Instantiate the Secrets Manager SecretsManagerFactory.clear_all() with patch.object(AWSSecretsManager, "get_string_value", return_value=SECRET_VALUE): diff --git a/openmetadata-clients/openmetadata-java-client/pom.xml b/openmetadata-clients/openmetadata-java-client/pom.xml index c60dfde3b786..ec3160813f02 100644 --- a/openmetadata-clients/openmetadata-java-client/pom.xml +++ b/openmetadata-clients/openmetadata-java-client/pom.xml @@ -5,7 +5,7 @@ openmetadata-clients org.open-metadata - 1.6.0-SNAPSHOT + 1.6.4 4.0.0 diff --git a/openmetadata-clients/pom.xml b/openmetadata-clients/pom.xml index 90b7fa975e8c..aaafa098df5a 100644 --- a/openmetadata-clients/pom.xml +++ b/openmetadata-clients/pom.xml @@ -5,7 +5,7 @@ platform org.open-metadata - 1.6.0-SNAPSHOT + 1.6.4 4.0.0 diff --git a/openmetadata-dist/pom.xml b/openmetadata-dist/pom.xml index 0a24ef385cd3..845272aae7e8 100644 --- a/openmetadata-dist/pom.xml +++ b/openmetadata-dist/pom.xml @@ -20,7 +20,7 @@ platform org.open-metadata - 1.6.0-SNAPSHOT + 1.6.4 openmetadata-dist diff --git a/openmetadata-docs/content/partials/v1.4/connectors/storage/manifest.md b/openmetadata-docs/content/partials/v1.4/connectors/storage/manifest.md index be18c834c6d9..a1f243dd6707 100644 --- a/openmetadata-docs/content/partials/v1.4/connectors/storage/manifest.md +++ b/openmetadata-docs/content/partials/v1.4/connectors/storage/manifest.md @@ -57,6 +57,56 @@ Again, this information will be added on top of the inferred schema from the dat {% /codeInfo %} +{% codeInfo srNumber=7 %} + +**Automated Container Ingestion**: Registering all the data paths one by one can be a time consuming job, +to make the automated structure container ingestion you can provide the depth at which all the data is available. + +Let us understand this with the example, suppose following is the file hierarchy within my bucket. + +``` + +# prefix/depth1/depth2/depth3 +athena_service/my_database_a/my_schema_a/table_a/date=01-01-2025/data.parquet +athena_service/my_database_a/my_schema_a/table_a/date=02-01-2025/data.parquet +athena_service/my_database_a/my_schema_a/table_b/date=01-01-2025/data.parquet +athena_service/my_database_a/my_schema_a/table_b/date=02-01-2025/data.parquet + +athena_service/my_database_b/my_schema_a/table_a/date=01-01-2025/data.parquet +athena_service/my_database_b/my_schema_a/table_a/date=02-01-2025/data.parquet +athena_service/my_database_b/my_schema_a/table_b/date=01-01-2025/data.parquet +athena_service/my_database_b/my_schema_a/table_b/date=02-01-2025/data.parquet + +``` + +all my tables folders which contains the actual data are available at depth 3, hence when you specify the `depth: 3` in +manifest entry all following path will get registered as container in OpenMetadata with this single entry + +``` +athena_service/my_database_a/my_schema_a/table_a +athena_service/my_database_a/my_schema_a/table_b +athena_service/my_database_b/my_schema_a/table_a +athena_service/my_database_b/my_schema_a/table_b +``` + +saving efforts to add 4 individual entries compared to 1 + +{% /codeInfo %} + + +{% codeInfo srNumber=6 %} + +**Unstructured Container**: OpenMetadata supports ingesting unstructured files like images, pdf's etc. We support fetching the file names, size and tags associates to such files. + +In case you want to ingest a single unstructured file, then just specifying the full path of the unstructured file in `datapath` would be enough for ingestion. + +In case you want to ingest all unstructured files with a specific extension for example `pdf` & `png` then you can provide the folder name containing such files in `dataPath` and list of extensions in the `unstructuredFormats` field. + +In case you want to ingest all unstructured files with irrespective of their file type or extension then you can provide the folder name containing such files in `dataPath` and `["*"]` in the `unstructuredFormats` field. + +{% /codeInfo %} + + {% /codeInfoContainer %} {% codeBlock fileName="openmetadata.json" %} @@ -110,6 +160,27 @@ Again, this information will be added on top of the inferred schema from the dat } ] } +``` +```json {% srNumber=7 %} + { + "dataPath": "athena_service", + "structureFormat": "parquet", + "isPartitioned": true, + "depth": 2 + } +``` +```json {% srNumber=6 %} + { + "dataPath": "path/to/solution.pdf", + }, + { + "dataPath": "path/to/unstructured_folder_png_pdf", + "unstructuredFormats": ["png","pdf"] + }, + { + "dataPath": "path/to/unstructured_folder_all", + "unstructuredFormats": ["*"] + } ] } ``` diff --git a/openmetadata-docs/content/partials/v1.4/connectors/yaml/dbt/source-config-def.md b/openmetadata-docs/content/partials/v1.4/connectors/yaml/dbt/source-config-def.md index 1938909593e3..eb3c11a332e9 100644 --- a/openmetadata-docs/content/partials/v1.4/connectors/yaml/dbt/source-config-def.md +++ b/openmetadata-docs/content/partials/v1.4/connectors/yaml/dbt/source-config-def.md @@ -4,6 +4,8 @@ **dbtUpdateDescriptions**: Configuration to update the description from dbt or not. If set to true descriptions from dbt will override the already present descriptions on the entity. For more details visit [here](/connectors/ingestion/workflows/dbt/ingest-dbt-descriptions) +**dbtUpdateOwners**: Configuration to update the owner from dbt or not. If set to true owners from dbt will override the already present owners on the entity. For more details visit [here](/connectors/ingestion/workflows/dbt/ingest-dbt-owner) + **includeTags**: true or false, to ingest tags from dbt. Default is true. **dbtClassificationName**: Custom OpenMetadata Classification name for dbt tags. diff --git a/openmetadata-docs/content/partials/v1.4/connectors/yaml/dbt/source-config.md b/openmetadata-docs/content/partials/v1.4/connectors/yaml/dbt/source-config.md index a697981792aa..8f7662283dcf 100644 --- a/openmetadata-docs/content/partials/v1.4/connectors/yaml/dbt/source-config.md +++ b/openmetadata-docs/content/partials/v1.4/connectors/yaml/dbt/source-config.md @@ -1,5 +1,6 @@ ```yaml {% srNumber=120 %} # dbtUpdateDescriptions: true or false + # dbtUpdateOwners: true or false # includeTags: true or false # dbtClassificationName: dbtTags # databaseFilterPattern: diff --git a/openmetadata-docs/content/partials/v1.6/connectors/storage/manifest.md b/openmetadata-docs/content/partials/v1.6/connectors/storage/manifest.md index 0f165a9a57fa..aae373ab44bb 100644 --- a/openmetadata-docs/content/partials/v1.6/connectors/storage/manifest.md +++ b/openmetadata-docs/content/partials/v1.6/connectors/storage/manifest.md @@ -57,6 +57,43 @@ Again, this information will be added on top of the inferred schema from the dat {% /codeInfo %} +{% codeInfo srNumber=7 %} + +**Automated Container Ingestion**: Registering all the data paths one by one can be a time consuming job, +to make the automated structure container ingestion you can provide the depth at which all the data is available. + +Let us understand this with the example, suppose following is the file hierarchy within my bucket. + +``` + +# prefix/depth1/depth2/depth3 +athena_service/my_database_a/my_schema_a/table_a/date=01-01-2025/data.parquet +athena_service/my_database_a/my_schema_a/table_a/date=02-01-2025/data.parquet +athena_service/my_database_a/my_schema_a/table_b/date=01-01-2025/data.parquet +athena_service/my_database_a/my_schema_a/table_b/date=02-01-2025/data.parquet + +athena_service/my_database_b/my_schema_a/table_a/date=01-01-2025/data.parquet +athena_service/my_database_b/my_schema_a/table_a/date=02-01-2025/data.parquet +athena_service/my_database_b/my_schema_a/table_b/date=01-01-2025/data.parquet +athena_service/my_database_b/my_schema_a/table_b/date=02-01-2025/data.parquet + +``` + +all my tables folders which contains the actual data are available at depth 3, hence when you specify the `depth: 3` in +manifest entry all following path will get registered as container in OpenMetadata with this single entry + +``` +athena_service/my_database_a/my_schema_a/table_a +athena_service/my_database_a/my_schema_a/table_b +athena_service/my_database_b/my_schema_a/table_a +athena_service/my_database_b/my_schema_a/table_b +``` + +saving efforts to add 4 individual entries compared to 1 + +{% /codeInfo %} + + {% codeInfo srNumber=6 %} **Unstructured Container**: OpenMetadata supports ingesting unstructured files like images, pdf's etc. We support fetching the file names, size and tags associates to such files. @@ -124,6 +161,14 @@ In case you want to ingest all unstructured files with irrespective of their fil ] } ``` +```json {% srNumber=7 %} + { + "dataPath": "athena_service", + "structureFormat": "parquet", + "isPartitioned": true, + "depth": 2 + } +``` ```json {% srNumber=6 %} { "dataPath": "path/to/solution.pdf", diff --git a/openmetadata-docs/content/partials/v1.6/connectors/yaml/dbt/source-config-def.md b/openmetadata-docs/content/partials/v1.6/connectors/yaml/dbt/source-config-def.md index 1938909593e3..eb3c11a332e9 100644 --- a/openmetadata-docs/content/partials/v1.6/connectors/yaml/dbt/source-config-def.md +++ b/openmetadata-docs/content/partials/v1.6/connectors/yaml/dbt/source-config-def.md @@ -4,6 +4,8 @@ **dbtUpdateDescriptions**: Configuration to update the description from dbt or not. If set to true descriptions from dbt will override the already present descriptions on the entity. For more details visit [here](/connectors/ingestion/workflows/dbt/ingest-dbt-descriptions) +**dbtUpdateOwners**: Configuration to update the owner from dbt or not. If set to true owners from dbt will override the already present owners on the entity. For more details visit [here](/connectors/ingestion/workflows/dbt/ingest-dbt-owner) + **includeTags**: true or false, to ingest tags from dbt. Default is true. **dbtClassificationName**: Custom OpenMetadata Classification name for dbt tags. diff --git a/openmetadata-docs/content/partials/v1.6/connectors/yaml/dbt/source-config.md b/openmetadata-docs/content/partials/v1.6/connectors/yaml/dbt/source-config.md index a697981792aa..8f7662283dcf 100644 --- a/openmetadata-docs/content/partials/v1.6/connectors/yaml/dbt/source-config.md +++ b/openmetadata-docs/content/partials/v1.6/connectors/yaml/dbt/source-config.md @@ -1,5 +1,6 @@ ```yaml {% srNumber=120 %} # dbtUpdateDescriptions: true or false + # dbtUpdateOwners: true or false # includeTags: true or false # dbtClassificationName: dbtTags # databaseFilterPattern: diff --git a/openmetadata-docs/content/partials/v1.6/deployment/upgrade/upgrade-prerequisites.md b/openmetadata-docs/content/partials/v1.6/deployment/upgrade/upgrade-prerequisites.md index 3004c74156d7..3852522895b8 100644 --- a/openmetadata-docs/content/partials/v1.6/deployment/upgrade/upgrade-prerequisites.md +++ b/openmetadata-docs/content/partials/v1.6/deployment/upgrade/upgrade-prerequisites.md @@ -86,6 +86,134 @@ After the migration is finished, you can revert this changes. # Backward Incompatible Changes +## 1.6.4 + +### Airflow 2.9.3 + +We are upgrading the Ingestion Airflow version to 2.9.3. + +The upgrade from the existing 2.9.1 -> 2.9.3 should happen transparently. The only thing to note is that there's +an ongoing issue with Airflow migrations and the `pymysql` driver, which we used before. If you are specifying +on your end the `DB_SCHEME` environment variable in the ingestion image, make sure it now is set to `mysql+mysqldb`. + +We have updated the default values accordingly. + +## 1.6.2 + +### Executable Logical Test Suites + +We are introducing a new feature that allows users to execute logical test suites. This feature will allow users to run +groups of Data Quality tests, even if they belong to different tables (or even services!). Note that before, you could +only schedule and execute the tests for each of the tables. + +From the UI, you can now create a new Test Suite, add any tests you want and create and schedule the run. + +This change, however, requires some adjustments if you are directly interacting with the OpenMetadata API or if you +are running the ingestions externally: + +#### `/executable` endpoints Changes + +CRUD operations around "executable" Test Suites - the ones directly related to a single table - were managed by the +`/executable` endpoints, e.g., `POST /v1/dataQuality/testSuites/executable`. We'll keep this endpoints until the next release, +but users should update their operations to use the new `/base` endpoints, e.g., `POST /v1/dataQuality/testSuites/base`. + +This is to adjust the naming convention since all Test Suites are executable, so we're differentiating between "base" and +"logical" Test Suites. + +In the meantime, you can use the `/executable` endpoints to create and manage the Test Suites, but you'll get deprecation +headers in the response. We recommend migrating to the new endpoints as soon as possible to avoid any issues when the `/executable` +endpoints get completely removed. + +#### YAML Changes + +If you're running the DQ Workflows externally AND YOU ARE NOT STORING THE SERVICE INFORMATION IN OPENMETADATA, this is how they'll change: + +A YAML file for 1.5.x would look like this: + +```yaml +source: + type: testsuite + serviceName: red # Test Suite Name + serviceConnection: + config: + hostPort: + username: + password: + database: + type: Redshift + sourceConfig: + config: + type: TestSuite + entityFullyQualifiedName: red.dev.dbt_jaffle.customers + profileSampleType: PERCENTAGE +processor: + type: "orm-test-runner" + config: {} +sink: + type: metadata-rest + config: {} +workflowConfig: + openMetadataServerConfig: + hostPort: http://localhost:8585/api + authProvider: openmetadata + securityConfig: + jwtToken: "..." +``` + +Basically, if you are not storing the service connection in OpenMetadata, you could leverage the `source.serviceConnection` +entry to pass that information. + +However, with the ability to execute Logical Test Suites, you can now have multiple tests from different services! This means, +that the connection information needs to be placed differently. The new YAML file would look like this: + +```yaml +source: + type: testsuite + serviceName: Logical # Test Suite Name + sourceConfig: + config: + type: TestSuite + serviceConnections: + - serviceName: red + serviceConnection: + config: + hostPort: + username: + password: + database: + type: Redshift + - serviceName: snowflake + serviceConnection: + config: + hostPort: + username: + password: + database: + type: Snowflake +processor: + type: "orm-test-runner" + config: {} +sink: + type: metadata-rest + config: {} +workflowConfig: + openMetadataServerConfig: + hostPort: http://localhost:8585/api + authProvider: openmetadata + securityConfig: + jwtToken: "..." +``` + +As you can see, you can pass multiple `serviceConnections` to the `sourceConfig` entry, each one with the connection information +and the `serviceName` they are linked to. + +{% note noteType="Warning" %} + +If you are already storing the service connection information in OpenMetadata (e.g., because you have created the services via the UI), +there's nothing you need to do. The ingestion will automatically pick up the connection information from the service. + +{% /note %} + ## 1.6.0 ### Ingestion Workflow Status diff --git a/openmetadata-docs/content/v1.4.x/connectors/database/snowflake/index.md b/openmetadata-docs/content/v1.4.x/connectors/database/snowflake/index.md index 92f5e3369403..57ebb385e1e1 100644 --- a/openmetadata-docs/content/v1.4.x/connectors/database/snowflake/index.md +++ b/openmetadata-docs/content/v1.4.x/connectors/database/snowflake/index.md @@ -59,13 +59,18 @@ GRANT SELECT ON ALL TABLES IN SCHEMA TEST_SCHEMA TO ROLE NEW_ROLE; GRANT SELECT ON ALL EXTERNAL TABLES IN SCHEMA TEST_SCHEMA TO ROLE NEW_ROLE; GRANT SELECT ON ALL VIEWS IN SCHEMA TEST_SCHEMA TO ROLE NEW_ROLE; GRANT SELECT ON ALL DYNAMIC TABLES IN SCHEMA TEST_SCHEMA TO ROLE NEW_ROLE; + +-- Grant IMPORTED PRIVILEGES on all Schemas of SNOWFLAKE DB to New Role, +-- optional but required for usage, lineage and stored procedure ingestion +GRANT IMPORTED PRIVILEGES ON ALL SCHEMAS IN DATABASE SNOWFLAKE TO ROLE NEW_ROLE; ``` {% note %} If running any of: - Incremental Extraction - Ingesting Tags - - Usage Workflow + - Ingesting Stored Procedures + - Lineage & Usage Workflow The following Grant is needed {% /note %} @@ -74,24 +79,12 @@ The following Grant is needed - **Ingesting Tags**: Openmetadata fetches the information by querying `snowflake.account_usage.tag_references`. -- **Usage Workflow**: Openmetadata fetches the query logs by querying `snowflake.account_usage.query_history` table. For this the snowflake user should be granted the `ACCOUNTADMIN` role or a role granted IMPORTED PRIVILEGES on the database `SNOWFLAKE`. - -In order to be able to query those tables, the user should be either granted the `ACCOUNTADMIN` role or a role with the `IMPORTED PRIVILEGES` grant on the `SNOWFLAKE` database: - -```sql --- Grant IMPORTED PRIVILEGES on all Schemas of SNOWFLAKE DB to New Role -GRANT IMPORTED PRIVILEGES ON ALL SCHEMAS IN DATABASE SNOWFLAKE TO ROLE NEW_ROLE; -``` +- **Lineage & Usage Workflow**: Openmetadata fetches the query logs by querying `snowflake.account_usage.query_history` table. For this the snowflake user should be granted the `ACCOUNTADMIN` role or a role granted IMPORTED PRIVILEGES on the database `SNOWFLAKE`. You can find more information about the `account_usage` schema [here](https://docs.snowflake.com/en/sql-reference/account-usage). -Regarding Stored Procedures: -1. Snowflake only allows the grant of `USAGE` or `OWNERSHIP` -2. A user can only see the definition of the procedure in 2 situations: - 1. If it has the `OWNERSHIP` grant, - 2. If it has the `USAGE` grant and the procedure is created with `EXECUTE AS CALLER`. +- **Ingesting Stored Procedures**: Openmetadata fetches the information by querying `snowflake.account_usage.procedures` & `snowflake.account_usage.functions`. -Make sure to add the `GRANT ON PROCEDURE () to NEW_ROLE`, e.g., `GRANT USAGE ON PROCEDURE CLEAN_DATA(varchar, varchar) to NEW_ROLE`. ## Metadata Ingestion @@ -122,6 +115,9 @@ Make sure to add the `GRANT ON PROCEDURE () t - **Include Temporary and Transient Tables**: Optional configuration for ingestion of `TRANSIENT` and `TEMPORARY` tables, By default, it will skip the `TRANSIENT` and `TEMPORARY` tables. - **Client Session Keep Alive**: Optional Configuration to keep the session active in case the ingestion job runs for longer duration. +- **Account Usage Schema Name**: Full name of account usage schema, used in case your used do not have direct access to `SNOWFLAKE.ACCOUNT_USAGE` schema. In such case you can replicate tables `QUERY_HISTORY`, `TAG_REFERENCES`, `PROCEDURES`, `FUNCTIONS` to a custom schema let's say `CUSTOM_DB.CUSTOM_SCHEMA` and provide the same name in this field. + +When using this field make sure you have all these tables available within your custom schema `QUERY_HISTORY`, `TAG_REFERENCES`, `PROCEDURES`, `FUNCTIONS`. {% partial file="/v1.4/connectors/database/advanced-configuration.md" /%} diff --git a/openmetadata-docs/content/v1.4.x/connectors/database/snowflake/yaml.md b/openmetadata-docs/content/v1.4.x/connectors/database/snowflake/yaml.md index 750c0f00ff9f..57eea486da9b 100644 --- a/openmetadata-docs/content/v1.4.x/connectors/database/snowflake/yaml.md +++ b/openmetadata-docs/content/v1.4.x/connectors/database/snowflake/yaml.md @@ -150,6 +150,14 @@ This is a sample config for Snowflake: {% /codeInfo %} +{% codeInfo srNumber=40 %} + +**accountUsageSchema**: Full name of account usage schema, used in case your used do not have direct access to `SNOWFLAKE.ACCOUNT_USAGE` schema. In such case you can replicate tables `QUERY_HISTORY`, `TAG_REFERENCES`, `PROCEDURES`, `FUNCTIONS` to a custom schema let's say `CUSTOM_DB.CUSTOM_SCHEMA` and provide the same name in this field. + +When using this field make sure you have all these tables available within your custom schema `QUERY_HISTORY`, `TAG_REFERENCES`, `PROCEDURES`, `FUNCTIONS`. + +{% /codeInfo %} + {% codeInfo srNumber=6 %} **includeTransientTables**: Optional configuration for ingestion of TRANSIENT and TEMPORARY tables, By default, it will skip the TRANSIENT and TEMPORARY tables. @@ -231,6 +239,9 @@ source: ```yaml {% srNumber=5 %} # database: ``` +```yaml {% srNumber=40 %} + # accountUsageSchema: SNOWFLAKE.ACCOUNT_USAGE +``` ```yaml {% srNumber=6 %} includeTransientTables: false ``` diff --git a/openmetadata-docs/content/v1.4.x/connectors/ingestion/workflows/dbt/ingest-dbt-owner.md b/openmetadata-docs/content/v1.4.x/connectors/ingestion/workflows/dbt/ingest-dbt-owner.md index 2a246172fa5f..f32625e71410 100644 --- a/openmetadata-docs/content/v1.4.x/connectors/ingestion/workflows/dbt/ingest-dbt-owner.md +++ b/openmetadata-docs/content/v1.4.x/connectors/ingestion/workflows/dbt/ingest-dbt-owner.md @@ -129,8 +129,16 @@ After running the ingestion workflow with dbt you can see the created user or te -{% note %} +## Overriding the existing table Owners -If a table already has a owner linked to it, owner from the dbt will not update the current owner. +To establish a unified and reliable system for owners, a single source of truth is necessary. It either is directly OpenMetadata, if individuals want to go there and keep updating, or if they prefer to keep it centralized in dbt, then we can always rely on that directly. -{% /note %} +When the `Update Owners` toggle is enabled during the configuration of dbt ingestion, existing owners of tables will be overwritten with the dbt owners. + +If toggle is disabled during the configuration of dbt ingestion, dbt owners will only be updated for tables in OpenMetadata that currently have no owners. Existing owners will remain unchanged and will not be overwritten with dbt owners. + +{% image + src="/images/v1.6/features/ingestion/workflows/dbt/dbt-features/dbt-update-owners.webp" + alt="update-dbt-owners" + caption="Update dbt Owners" + /%} diff --git a/openmetadata-docs/content/v1.4.x/connectors/pipeline/airbyte/index.md b/openmetadata-docs/content/v1.4.x/connectors/pipeline/airbyte/index.md index 50d6fb14ad30..3af86179b6c7 100644 --- a/openmetadata-docs/content/v1.4.x/connectors/pipeline/airbyte/index.md +++ b/openmetadata-docs/content/v1.4.x/connectors/pipeline/airbyte/index.md @@ -38,6 +38,12 @@ Configure and schedule Airbyte metadata and profiler workflows from the OpenMeta - **Host and Port**: Pipeline Service Management UI URL +- **Username**: Username to connect to Airbyte. + +- **Password**: Password to connect to Airbyte. + +- **API Version**: Version of the Airbyte REST API by default `api/v1`. + {% /extraContent %} {% partial file="/v1.4/connectors/test-connection.md" /%} diff --git a/openmetadata-docs/content/v1.4.x/connectors/pipeline/airbyte/yaml.md b/openmetadata-docs/content/v1.4.x/connectors/pipeline/airbyte/yaml.md index 31fb1b194052..3aeed4bbee23 100644 --- a/openmetadata-docs/content/v1.4.x/connectors/pipeline/airbyte/yaml.md +++ b/openmetadata-docs/content/v1.4.x/connectors/pipeline/airbyte/yaml.md @@ -59,6 +59,23 @@ This is a sample config for Airbyte: **hostPort**: Pipeline Service Management UI URL +{% /codeInfo %} + +{% codeInfo srNumber=2 %} + +**username**: Username to connect to Airbyte. + +{% /codeInfo %} + +{% codeInfo srNumber=3 %} + +**password**: Password to connect to Airbyte. + +{% /codeInfo %} + +{% codeInfo srNumber=4 %} + +**apiVersion**: Version of the Airbyte REST API by default `api/v1`. {% /codeInfo %} @@ -86,6 +103,15 @@ source: ```yaml {% srNumber=1 %} hostPort: http://localhost:8000 ``` +```yaml {% srNumber=2 %} + username: +``` +```yaml {% srNumber=3 %} + password: +``` +```yaml {% srNumber=4 %} + apiVersion: api/v1 +``` {% partial file="/v1.4/connectors/yaml/pipeline/source-config.md" /%} diff --git a/openmetadata-docs/content/v1.4.x/how-to-guides/data-discovery/advanced.md b/openmetadata-docs/content/v1.4.x/how-to-guides/data-discovery/advanced.md index 2f55f76168c8..14dd7b0d831d 100644 --- a/openmetadata-docs/content/v1.4.x/how-to-guides/data-discovery/advanced.md +++ b/openmetadata-docs/content/v1.4.x/how-to-guides/data-discovery/advanced.md @@ -33,6 +33,19 @@ caption="Add Complex Queries using Advanced Search" For example, we can set up a complex query as follows: - Group one set of conditions together by defining the `Owner`. You can add multiple conditions to define different owners and use the `OR` condition to ensure that the owner is any one among them. +{% note noteType="Tip" %} +### Note on Custom Properties in Elasticsearch Search + +Elasticsearch does not support searching for custom properties with the following formats: + +- **Time** +- **DateTime** +- Any date formats other than `yyyy-MM-dd` + +Please ensure that custom properties adhere to these constraints for compatibility with Elasticsearch search functionality. + +{% /note %} + {% image src="/images/v1.4/how-to-guides/discovery/adv3.png" alt="Grouped Condition based on the Owner of the Data Assets" diff --git a/openmetadata-docs/content/v1.4.x/main-concepts/metadata-standard/schemas/metadataIngestion/dbtPipeline.md b/openmetadata-docs/content/v1.4.x/main-concepts/metadata-standard/schemas/metadataIngestion/dbtPipeline.md index 842dfc102670..ab655037461f 100644 --- a/openmetadata-docs/content/v1.4.x/main-concepts/metadata-standard/schemas/metadataIngestion/dbtPipeline.md +++ b/openmetadata-docs/content/v1.4.x/main-concepts/metadata-standard/schemas/metadataIngestion/dbtPipeline.md @@ -11,7 +11,9 @@ slug: /main-concepts/metadata-standard/schemas/metadataingestion/dbtpipeline - **`type`**: Pipeline type. Refer to *#/definitions/dbtConfigType*. Default: `DBT`. - **`dbtConfigSource`**: Available sources to fetch DBT catalog and manifest files. +- **`searchAcrossDatabases`** *(boolean)*: Optional configuration to search across databases for tables or not. Default: `False`. - **`dbtUpdateDescriptions`** *(boolean)*: Optional configuration to update the description from DBT or not. Default: `False`. +- **`dbtUpdateOwners`** *(boolean)*: Optional configuration to update the owner from DBT or not. Default: `False`. - **`includeTags`** *(boolean)*: Optional configuration to toggle the tags ingestion. Default: `True`. - **`dbtClassificationName`** *(string)*: Custom OpenMetadata Classification name for dbt tags. Default: `dbtTags`. - **`schemaFilterPattern`**: Regex to only fetch tables or databases that matches the pattern. Refer to *../type/filterPattern.json#/definitions/filterPattern*. diff --git a/openmetadata-docs/content/v1.5.x/how-to-guides/data-discovery/advanced.md b/openmetadata-docs/content/v1.5.x/how-to-guides/data-discovery/advanced.md index 3a8b3d04db5d..6139eeb42d9b 100644 --- a/openmetadata-docs/content/v1.5.x/how-to-guides/data-discovery/advanced.md +++ b/openmetadata-docs/content/v1.5.x/how-to-guides/data-discovery/advanced.md @@ -33,6 +33,19 @@ caption="Add Complex Queries using Advanced Search" For example, we can set up a complex query as follows: - Group one set of conditions together by defining the `Owner`. You can add multiple conditions to define different owners and use the `OR` condition to ensure that the owner is any one among them. +{% note noteType="Tip" %} +### Note on Custom Properties in Elasticsearch Search + +Elasticsearch does not support searching for custom properties with the following formats: + +- **Time** +- **DateTime** +- Any date formats other than `yyyy-MM-dd` + +Please ensure that custom properties adhere to these constraints for compatibility with Elasticsearch search functionality. + +{% /note %} + {% image src="/images/v1.5/how-to-guides/discovery/adv3.png" alt="Grouped Condition based on the Owner of the Data Assets" diff --git a/openmetadata-docs/content/v1.6.x-SNAPSHOT/connectors/database/snowflake/index.md b/openmetadata-docs/content/v1.6.x-SNAPSHOT/connectors/database/snowflake/index.md index a4c5d452b58d..408b6b0b12ef 100644 --- a/openmetadata-docs/content/v1.6.x-SNAPSHOT/connectors/database/snowflake/index.md +++ b/openmetadata-docs/content/v1.6.x-SNAPSHOT/connectors/database/snowflake/index.md @@ -59,13 +59,18 @@ GRANT SELECT ON ALL TABLES IN SCHEMA TEST_SCHEMA TO ROLE NEW_ROLE; GRANT SELECT ON ALL EXTERNAL TABLES IN SCHEMA TEST_SCHEMA TO ROLE NEW_ROLE; GRANT SELECT ON ALL VIEWS IN SCHEMA TEST_SCHEMA TO ROLE NEW_ROLE; GRANT SELECT ON ALL DYNAMIC TABLES IN SCHEMA TEST_SCHEMA TO ROLE NEW_ROLE; + +-- Grant IMPORTED PRIVILEGES on all Schemas of SNOWFLAKE DB to New Role, +-- optional but required for usage, lineage and stored procedure ingestion +GRANT IMPORTED PRIVILEGES ON ALL SCHEMAS IN DATABASE SNOWFLAKE TO ROLE NEW_ROLE; ``` {% note %} If running any of: - Incremental Extraction - Ingesting Tags - - Usage Workflow + - Ingesting Stored Procedures + - Lineage & Usage Workflow The following Grant is needed {% /note %} @@ -74,24 +79,11 @@ The following Grant is needed - **Ingesting Tags**: Openmetadata fetches the information by querying `snowflake.account_usage.tag_references`. -- **Usage Workflow**: Openmetadata fetches the query logs by querying `snowflake.account_usage.query_history` table. For this the snowflake user should be granted the `ACCOUNTADMIN` role or a role granted IMPORTED PRIVILEGES on the database `SNOWFLAKE`. - -In order to be able to query those tables, the user should be either granted the `ACCOUNTADMIN` role or a role with the `IMPORTED PRIVILEGES` grant on the `SNOWFLAKE` database: - -```sql --- Grant IMPORTED PRIVILEGES on all Schemas of SNOWFLAKE DB to New Role -GRANT IMPORTED PRIVILEGES ON ALL SCHEMAS IN DATABASE SNOWFLAKE TO ROLE NEW_ROLE; -``` +- **Lineage & Usage Workflow**: Openmetadata fetches the query logs by querying `snowflake.account_usage.query_history` table. For this the snowflake user should be granted the `ACCOUNTADMIN` role or a role granted IMPORTED PRIVILEGES on the database `SNOWFLAKE`. You can find more information about the `account_usage` schema [here](https://docs.snowflake.com/en/sql-reference/account-usage). -Regarding Stored Procedures: -1. Snowflake only allows the grant of `USAGE` or `OWNERSHIP` -2. A user can only see the definition of the procedure in 2 situations: - 1. If it has the `OWNERSHIP` grant, - 2. If it has the `USAGE` grant and the procedure is created with `EXECUTE AS CALLER`. - -Make sure to add the `GRANT ON PROCEDURE () to NEW_ROLE`, e.g., `GRANT USAGE ON PROCEDURE CLEAN_DATA(varchar, varchar) to NEW_ROLE`. +- **Ingesting Stored Procedures**: Openmetadata fetches the information by querying `snowflake.account_usage.procedures` & `snowflake.account_usage.functions`. ## Metadata Ingestion @@ -122,6 +114,9 @@ Make sure to add the `GRANT ON PROCEDURE () t - **Include Temporary and Transient Tables**: Optional configuration for ingestion of `TRANSIENT` and `TEMPORARY` tables, By default, it will skip the `TRANSIENT` and `TEMPORARY` tables. - **Client Session Keep Alive**: Optional Configuration to keep the session active in case the ingestion job runs for longer duration. +- **Account Usage Schema Name**: Full name of account usage schema, used in case your used do not have direct access to `SNOWFLAKE.ACCOUNT_USAGE` schema. In such case you can replicate tables `QUERY_HISTORY`, `TAG_REFERENCES`, `PROCEDURES`, `FUNCTIONS` to a custom schema let's say `CUSTOM_DB.CUSTOM_SCHEMA` and provide the same name in this field. + +When using this field make sure you have all these tables available within your custom schema `QUERY_HISTORY`, `TAG_REFERENCES`, `PROCEDURES`, `FUNCTIONS`. {% partial file="/v1.6/connectors/database/advanced-configuration.md" /%} diff --git a/openmetadata-docs/content/v1.6.x-SNAPSHOT/connectors/database/snowflake/yaml.md b/openmetadata-docs/content/v1.6.x-SNAPSHOT/connectors/database/snowflake/yaml.md index 8a824e07928b..bd10edc7097b 100644 --- a/openmetadata-docs/content/v1.6.x-SNAPSHOT/connectors/database/snowflake/yaml.md +++ b/openmetadata-docs/content/v1.6.x-SNAPSHOT/connectors/database/snowflake/yaml.md @@ -150,6 +150,14 @@ This is a sample config for Snowflake: {% /codeInfo %} +{% codeInfo srNumber=40 %} + +**accountUsageSchema**: Full name of account usage schema, used in case your used do not have direct access to `SNOWFLAKE.ACCOUNT_USAGE` schema. In such case you can replicate tables `QUERY_HISTORY`, `TAG_REFERENCES`, `PROCEDURES`, `FUNCTIONS` to a custom schema let's say `CUSTOM_DB.CUSTOM_SCHEMA` and provide the same name in this field. + +When using this field make sure you have all these tables available within your custom schema `QUERY_HISTORY`, `TAG_REFERENCES`, `PROCEDURES`, `FUNCTIONS`. + +{% /codeInfo %} + {% codeInfo srNumber=6 %} **includeTransientTables**: Optional configuration for ingestion of TRANSIENT and TEMPORARY tables, By default, it will skip the TRANSIENT and TEMPORARY tables. @@ -231,6 +239,9 @@ source: ```yaml {% srNumber=5 %} # database: ``` +```yaml {% srNumber=40 %} + # accountUsageSchema: SNOWFLAKE.ACCOUNT_USAGE +``` ```yaml {% srNumber=6 %} includeTransientTables: false ``` diff --git a/openmetadata-docs/content/v1.6.x-SNAPSHOT/connectors/ingestion/workflows/dbt/ingest-dbt-owner.md b/openmetadata-docs/content/v1.6.x-SNAPSHOT/connectors/ingestion/workflows/dbt/ingest-dbt-owner.md index b85174d24ad0..0fbdbc7a7dea 100644 --- a/openmetadata-docs/content/v1.6.x-SNAPSHOT/connectors/ingestion/workflows/dbt/ingest-dbt-owner.md +++ b/openmetadata-docs/content/v1.6.x-SNAPSHOT/connectors/ingestion/workflows/dbt/ingest-dbt-owner.md @@ -138,8 +138,16 @@ After running the ingestion workflow with dbt you can see the created user or te -{% note %} +## Overriding the existing table Owners -If a table already has a owner linked to it, owner from the dbt will not update the current owner. +To establish a unified and reliable system for owners, a single source of truth is necessary. It either is directly OpenMetadata, if individuals want to go there and keep updating, or if they prefer to keep it centralized in dbt, then we can always rely on that directly. -{% /note %} +When the `Update Owners` toggle is enabled during the configuration of dbt ingestion, existing owners of tables will be overwritten with the dbt owners. + +If toggle is disabled during the configuration of dbt ingestion, dbt owners will only be updated for tables in OpenMetadata that currently have no owners. Existing owners will remain unchanged and will not be overwritten with dbt owners. + +{% image + src="/images/v1.6/features/ingestion/workflows/dbt/dbt-features/dbt-update-owners.webp" + alt="update-dbt-owners" + caption="Update dbt Owners" + /%} diff --git a/openmetadata-docs/content/v1.6.x-SNAPSHOT/connectors/pipeline/airbyte/index.md b/openmetadata-docs/content/v1.6.x-SNAPSHOT/connectors/pipeline/airbyte/index.md index 9fb026318f51..4c0abb7b60a9 100644 --- a/openmetadata-docs/content/v1.6.x-SNAPSHOT/connectors/pipeline/airbyte/index.md +++ b/openmetadata-docs/content/v1.6.x-SNAPSHOT/connectors/pipeline/airbyte/index.md @@ -38,6 +38,12 @@ Configure and schedule Airbyte metadata and profiler workflows from the OpenMeta - **Host and Port**: Pipeline Service Management UI URL +- **Username**: Username to connect to Airbyte. + +- **Password**: Password to connect to Airbyte. + +- **API Version**: Version of the Airbyte REST API by default `api/v1`. + {% /extraContent %} {% partial file="/v1.6/connectors/test-connection.md" /%} diff --git a/openmetadata-docs/content/v1.6.x-SNAPSHOT/connectors/pipeline/airbyte/yaml.md b/openmetadata-docs/content/v1.6.x-SNAPSHOT/connectors/pipeline/airbyte/yaml.md index 4337cf7db36f..94c2e71439ce 100644 --- a/openmetadata-docs/content/v1.6.x-SNAPSHOT/connectors/pipeline/airbyte/yaml.md +++ b/openmetadata-docs/content/v1.6.x-SNAPSHOT/connectors/pipeline/airbyte/yaml.md @@ -59,6 +59,23 @@ This is a sample config for Airbyte: **hostPort**: Pipeline Service Management UI URL +{% /codeInfo %} + +{% codeInfo srNumber=2 %} + +**username**: Username to connect to Airbyte. + +{% /codeInfo %} + +{% codeInfo srNumber=3 %} + +**password**: Password to connect to Airbyte. + +{% /codeInfo %} + +{% codeInfo srNumber=4 %} + +**apiVersion**: Version of the Airbyte REST API by default `api/v1`. {% /codeInfo %} @@ -86,6 +103,15 @@ source: ```yaml {% srNumber=1 %} hostPort: http://localhost:8000 ``` +```yaml {% srNumber=2 %} + username: +``` +```yaml {% srNumber=3 %} + password: +``` +```yaml {% srNumber=4 %} + apiVersion: api/v1 +``` {% partial file="/v1.6/connectors/yaml/pipeline/source-config.md" /%} diff --git a/openmetadata-docs/content/v1.6.x-SNAPSHOT/how-to-guides/data-discovery/advanced.md b/openmetadata-docs/content/v1.6.x-SNAPSHOT/how-to-guides/data-discovery/advanced.md index 291d69b4556f..400ab4c6f301 100644 --- a/openmetadata-docs/content/v1.6.x-SNAPSHOT/how-to-guides/data-discovery/advanced.md +++ b/openmetadata-docs/content/v1.6.x-SNAPSHOT/how-to-guides/data-discovery/advanced.md @@ -33,6 +33,19 @@ caption="Add Complex Queries using Advanced Search" For example, we can set up a complex query as follows: - Group one set of conditions together by defining the `Owner`. You can add multiple conditions to define different owners and use the `OR` condition to ensure that the owner is any one among them. +{% note noteType="Tip" %} +### Note on Custom Properties in Elasticsearch Search + +Elasticsearch does not support searching for custom properties with the following formats: + +- **Time** +- **DateTime** +- Any date formats other than `yyyy-MM-dd` + +Please ensure that custom properties adhere to these constraints for compatibility with Elasticsearch search functionality. + +{% /note %} + {% image src="/images/v1.6/how-to-guides/discovery/adv3.png" alt="Grouped Condition based on the Owner of the Data Assets" diff --git a/openmetadata-docs/content/v1.6.x-SNAPSHOT/main-concepts/metadata-standard/schemas/metadataIngestion/dbtPipeline.md b/openmetadata-docs/content/v1.6.x-SNAPSHOT/main-concepts/metadata-standard/schemas/metadataIngestion/dbtPipeline.md index 842dfc102670..d98baf90b002 100644 --- a/openmetadata-docs/content/v1.6.x-SNAPSHOT/main-concepts/metadata-standard/schemas/metadataIngestion/dbtPipeline.md +++ b/openmetadata-docs/content/v1.6.x-SNAPSHOT/main-concepts/metadata-standard/schemas/metadataIngestion/dbtPipeline.md @@ -11,7 +11,9 @@ slug: /main-concepts/metadata-standard/schemas/metadataingestion/dbtpipeline - **`type`**: Pipeline type. Refer to *#/definitions/dbtConfigType*. Default: `DBT`. - **`dbtConfigSource`**: Available sources to fetch DBT catalog and manifest files. +- **`searchAcrossDatabases`** *(boolean)*: Optional configuration to search across databases for tables or not. Default: `False`. - **`dbtUpdateDescriptions`** *(boolean)*: Optional configuration to update the description from DBT or not. Default: `False`. +- **`dbtUpdateOwners`** *(boolean)*: Optional configuration to update the owners from DBT or not. Default: `False`. - **`includeTags`** *(boolean)*: Optional configuration to toggle the tags ingestion. Default: `True`. - **`dbtClassificationName`** *(string)*: Custom OpenMetadata Classification name for dbt tags. Default: `dbtTags`. - **`schemaFilterPattern`**: Regex to only fetch tables or databases that matches the pattern. Refer to *../type/filterPattern.json#/definitions/filterPattern*. diff --git a/openmetadata-docs/images/v1.6/features/ingestion/workflows/dbt/dbt-features/dbt-update-owners.webp b/openmetadata-docs/images/v1.6/features/ingestion/workflows/dbt/dbt-features/dbt-update-owners.webp new file mode 100644 index 000000000000..b9e0bd91c42b Binary files /dev/null and b/openmetadata-docs/images/v1.6/features/ingestion/workflows/dbt/dbt-features/dbt-update-owners.webp differ diff --git a/openmetadata-service/pom.xml b/openmetadata-service/pom.xml index 541bb6a41160..e6f9fee14064 100644 --- a/openmetadata-service/pom.xml +++ b/openmetadata-service/pom.xml @@ -5,7 +5,7 @@ platform org.open-metadata - 1.6.0-SNAPSHOT + 1.6.4 4.0.0 openmetadata-service @@ -16,7 +16,7 @@ ${project.basedir}/target/site/jacoco-aggregate/jacoco.xml ${project.basedir}/src/test/java 1.20.3 - 2.29.15 + 2.30.19 1.14.0 4.9.0 1.0.0 @@ -28,6 +28,7 @@ 3.6.0 3.3.1 2.1.1 + 2.5.2 @@ -89,7 +90,7 @@ net.minidev json-smart - 2.5.1 + ${json-smart.version} org.open-metadata @@ -147,6 +148,24 @@ ru.vyarus.guicey guicey-jdbi3 5.9.2 + + + ch.qos.logback + logback-core + + + ch.qos.logback + logback-classic + + + ch.qos.logback + logback-access + + + org.slf4j + log4j-over-slf4j + + com.smoketurner @@ -182,10 +201,46 @@ io.github.maksymdolgykh.dropwizard dropwizard-micrometer-core + + + ch.qos.logback + logback-core + + + ch.qos.logback + logback-classic + + + ch.qos.logback + logback-access + + + org.slf4j + log4j-over-slf4j + + io.github.maksymdolgykh.dropwizard dropwizard-micrometer-jdbi + + + ch.qos.logback + logback-core + + + ch.qos.logback + logback-classic + + + ch.qos.logback + logback-access + + + org.slf4j + log4j-over-slf4j + + io.dropwizard @@ -264,6 +319,10 @@ ch.qos.logback logback-classic + + ch.qos.logback + logback-access + com.jayway.jsonpath json-path diff --git a/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java b/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java index 0121dbdc6d07..f13ae7b53a5c 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java +++ b/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java @@ -50,6 +50,7 @@ import java.util.Optional; import java.util.Set; import java.util.UUID; +import java.util.function.Function; import javax.ws.rs.core.Response; import lombok.extern.slf4j.Slf4j; import org.apache.commons.csv.CSVFormat; @@ -193,7 +194,11 @@ public final void addRecord(CsvFile csvFile, List recordList) { } /** Owner field is in entityType:entityName format */ - public List getOwners(CSVPrinter printer, CSVRecord csvRecord, int fieldNumber) + public List getOwners( + CSVPrinter printer, + CSVRecord csvRecord, + int fieldNumber, + Function invalidMessageCreator) throws IOException { if (!processRecord) { return null; @@ -207,7 +212,7 @@ public List getOwners(CSVPrinter printer, CSVRecord csvRecord, for (String owner : owners) { List ownerTypes = listOrEmpty(CsvUtil.fieldToEntities(owner)); if (ownerTypes.size() != 2) { - importFailure(printer, invalidOwner(fieldNumber), csvRecord); + importFailure(printer, invalidMessageCreator.apply(fieldNumber), csvRecord); return Collections.emptyList(); } EntityReference ownerRef = @@ -219,6 +224,16 @@ public List getOwners(CSVPrinter printer, CSVRecord csvRecord, return refs.isEmpty() ? null : refs; } + public List getOwners(CSVPrinter printer, CSVRecord csvRecord, int fieldNumber) + throws IOException { + return getOwners(printer, csvRecord, fieldNumber, EntityCsv::invalidOwner); + } + + public List getReviewers( + CSVPrinter printer, CSVRecord csvRecord, int fieldNumber) throws IOException { + return getOwners(printer, csvRecord, fieldNumber, EntityCsv::invalidReviewer); + } + /** Owner field is in entityName format */ public EntityReference getOwnerAsUser(CSVPrinter printer, CSVRecord csvRecord, int fieldNumber) throws IOException { @@ -868,6 +883,11 @@ public static String invalidOwner(int field) { return String.format(FIELD_ERROR_MSG, CsvErrorType.INVALID_FIELD, field + 1, error); } + public static String invalidReviewer(int field) { + String error = "Reviewer should be of format user:userName or team:teamName"; + return String.format(FIELD_ERROR_MSG, CsvErrorType.INVALID_FIELD, field + 1, error); + } + public static String invalidExtension(int field, String key, String value) { String error = "Invalid key-value pair in extension string: Key = " diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java b/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java index 9ee396193102..5b3d52970f0b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java @@ -67,6 +67,7 @@ import org.openmetadata.service.jdbi3.SystemRepository; import org.openmetadata.service.jdbi3.TokenRepository; import org.openmetadata.service.jdbi3.UsageRepository; +import org.openmetadata.service.jobs.JobDAO; import org.openmetadata.service.resources.feeds.MessageParser.EntityLink; import org.openmetadata.service.search.SearchRepository; import org.openmetadata.service.search.indexes.SearchIndex; @@ -77,6 +78,7 @@ public final class Entity { private static volatile boolean initializedRepositories = false; @Getter @Setter private static CollectionDAO collectionDAO; + @Getter @Setter private static JobDAO jobDAO; @Getter @Setter private static Jdbi jdbi; public static final String SEPARATOR = "."; // Fully qualified name separator @@ -248,6 +250,8 @@ public final class Entity { public static final String DOCUMENT = "document"; // ServiceType - Service Entity name map static final Map SERVICE_TYPE_ENTITY_MAP = new EnumMap<>(ServiceType.class); + // entity type to service entity name map + static final Map ENTITY_SERVICE_TYPE_MAP = new HashMap<>(); public static final List PARENT_ENTITY_TYPES = new ArrayList<>(); static { @@ -260,6 +264,24 @@ public final class Entity { SERVICE_TYPE_ENTITY_MAP.put(ServiceType.STORAGE, STORAGE_SERVICE); SERVICE_TYPE_ENTITY_MAP.put(ServiceType.SEARCH, SEARCH_SERVICE); SERVICE_TYPE_ENTITY_MAP.put(ServiceType.API, API_SERVICE); + + ENTITY_SERVICE_TYPE_MAP.put(DATABASE, DATABASE_SERVICE); + ENTITY_SERVICE_TYPE_MAP.put(DATABASE_SCHEMA, DATABASE_SERVICE); + ENTITY_SERVICE_TYPE_MAP.put(TABLE, DATABASE_SERVICE); + ENTITY_SERVICE_TYPE_MAP.put(STORED_PROCEDURE, DATABASE_SERVICE); + ENTITY_SERVICE_TYPE_MAP.put(QUERY, DATABASE_SERVICE); + ENTITY_SERVICE_TYPE_MAP.put(DASHBOARD, DASHBOARD_SERVICE); + ENTITY_SERVICE_TYPE_MAP.put(DASHBOARD_DATA_MODEL, DASHBOARD_SERVICE); + ENTITY_SERVICE_TYPE_MAP.put(CHART, DASHBOARD_SERVICE); + ENTITY_SERVICE_TYPE_MAP.put(PIPELINE, PIPELINE_SERVICE); + ENTITY_SERVICE_TYPE_MAP.put(MLMODEL, MLMODEL_SERVICE); + ENTITY_SERVICE_TYPE_MAP.put(TOPIC, MESSAGING_SERVICE); + ENTITY_SERVICE_TYPE_MAP.put(API, API_SERVICE); + ENTITY_SERVICE_TYPE_MAP.put(API_COLLCECTION, API_SERVICE); + ENTITY_SERVICE_TYPE_MAP.put(API_ENDPOINT, API_SERVICE); + ENTITY_SERVICE_TYPE_MAP.put(CONTAINER, STORAGE_SERVICE); + ENTITY_SERVICE_TYPE_MAP.put(SEARCH_INDEX, SEARCH_SERVICE); + PARENT_ENTITY_TYPES.addAll( listOf( DATABASE_SERVICE, @@ -314,6 +336,7 @@ public static void initializeRepositories(OpenMetadataApplicationConfig config, public static void cleanup() { initializedRepositories = false; collectionDAO = null; + jobDAO = null; searchRepository = null; ENTITY_REPOSITORY_MAP.clear(); } @@ -636,4 +659,11 @@ public static T getDao() { public static T getSearchRepo() { return (T) searchRepository; } + + public static String getServiceType(String entityType) { + if (ENTITY_SERVICE_TYPE_MAP.containsKey(entityType)) { + return ENTITY_SERVICE_TYPE_MAP.get(entityType); + } + return entityType; + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java b/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java index 9fedc0c89534..900b34fcf0dd 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java @@ -73,17 +73,22 @@ import org.openmetadata.service.events.EventFilter; import org.openmetadata.service.events.EventPubSub; import org.openmetadata.service.events.scheduled.EventSubscriptionScheduler; -import org.openmetadata.service.events.scheduled.PipelineServiceStatusJobHandler; +import org.openmetadata.service.events.scheduled.ServicesStatusJobHandler; import org.openmetadata.service.exception.CatalogGenericExceptionMapper; import org.openmetadata.service.exception.ConstraintViolationExceptionMapper; import org.openmetadata.service.exception.JsonMappingExceptionMapper; import org.openmetadata.service.exception.OMErrorPageHandler; import org.openmetadata.service.fernet.Fernet; +import org.openmetadata.service.governance.workflows.WorkflowHandler; import org.openmetadata.service.jdbi3.CollectionDAO; import org.openmetadata.service.jdbi3.EntityRepository; import org.openmetadata.service.jdbi3.MigrationDAO; import org.openmetadata.service.jdbi3.locator.ConnectionAwareAnnotationSqlLocator; import org.openmetadata.service.jdbi3.locator.ConnectionType; +import org.openmetadata.service.jobs.EnumCleanupHandler; +import org.openmetadata.service.jobs.GenericBackgroundWorker; +import org.openmetadata.service.jobs.JobDAO; +import org.openmetadata.service.jobs.JobHandlerRegistry; import org.openmetadata.service.limits.DefaultLimits; import org.openmetadata.service.limits.Limits; import org.openmetadata.service.migration.Migration; @@ -162,6 +167,7 @@ public void run(OpenMetadataApplicationConfig catalogConfig, Environment environ jdbi = createAndSetupJDBI(environment, catalogConfig.getDataSourceFactory()); Entity.setCollectionDAO(getDao(jdbi)); + Entity.setJobDAO(jdbi.onDemand(JobDAO.class)); Entity.setJdbi(jdbi); initializeSearchRepository(catalogConfig.getElasticSearchConfiguration()); @@ -173,6 +179,9 @@ public void run(OpenMetadataApplicationConfig catalogConfig, Environment environ // Configure the Fernet instance Fernet.getInstance().setFernetKey(catalogConfig); + // Initialize Workflow Handler + WorkflowHandler.initialize(catalogConfig); + // Init Settings Cache after repositories SettingsCache.initialize(catalogConfig); @@ -186,7 +195,10 @@ public void run(OpenMetadataApplicationConfig catalogConfig, Environment environ EntityMaskerFactory.createEntityMasker(); // Instantiate JWT Token Generator - JWTTokenGenerator.getInstance().init(catalogConfig.getJwtTokenConfiguration()); + JWTTokenGenerator.getInstance() + .init( + catalogConfig.getAuthenticationConfiguration().getTokenValidationAlgorithm(), + catalogConfig.getJwtTokenConfiguration()); // Set the Database type for choosing correct queries from annotations jdbi.getConfig(SqlObjects.class) @@ -227,6 +239,13 @@ public void run(OpenMetadataApplicationConfig catalogConfig, Environment environ // Register Event Handler registerEventFilter(catalogConfig, environment); environment.lifecycle().manage(new ManagedShutdown()); + + JobHandlerRegistry registry = new JobHandlerRegistry(); + registry.register("EnumCleanupHandler", new EnumCleanupHandler(getDao(jdbi))); + environment + .lifecycle() + .manage(new GenericBackgroundWorker(jdbi.onDemand(JobDAO.class), registry)); + // Register Event publishers registerEventPublisher(catalogConfig); @@ -245,16 +264,23 @@ public void run(OpenMetadataApplicationConfig catalogConfig, Environment environ // Asset Servlet Registration registerAssetServlet(catalogConfig.getWebConfiguration(), environment); - // Handle Pipeline Service Client Status job - PipelineServiceStatusJobHandler pipelineServiceStatusJobHandler = - PipelineServiceStatusJobHandler.create( - catalogConfig.getPipelineServiceClientConfiguration(), catalogConfig.getClusterName()); - pipelineServiceStatusJobHandler.addPipelineServiceStatusJob(); + // Handle Services Jobs + registerHealthCheckJobs(catalogConfig); // Register Auth Handlers registerAuthServlets(catalogConfig, environment); } + private void registerHealthCheckJobs(OpenMetadataApplicationConfig catalogConfig) { + ServicesStatusJobHandler healthCheckStatusHandler = + ServicesStatusJobHandler.create( + catalogConfig.getEventMonitorConfiguration(), + catalogConfig.getPipelineServiceClientConfiguration(), + catalogConfig.getClusterName()); + healthCheckStatusHandler.addPipelineServiceStatusJob(); + healthCheckStatusHandler.addDatabaseAndSearchStatusJobs(); + } + private void registerAuthServlets(OpenMetadataApplicationConfig config, Environment environment) { if (config.getAuthenticationConfiguration() != null && config diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/TypeRegistry.java b/openmetadata-service/src/main/java/org/openmetadata/service/TypeRegistry.java index a294a79cc51b..78c26a5668c5 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/TypeRegistry.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/TypeRegistry.java @@ -14,15 +14,22 @@ package org.openmetadata.service; import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; +import static org.openmetadata.service.Entity.ADMIN_USER_NAME; +import static org.openmetadata.service.resources.types.TypeResource.PROPERTIES_FIELD; import com.networknt.schema.JsonSchema; +import java.util.List; import java.util.Map; +import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import lombok.extern.slf4j.Slf4j; import org.openmetadata.schema.entity.Type; +import org.openmetadata.schema.entity.type.Category; import org.openmetadata.schema.entity.type.CustomProperty; import org.openmetadata.service.exception.CatalogExceptionMessage; import org.openmetadata.service.exception.EntityNotFoundException; +import org.openmetadata.service.jdbi3.TypeRepository; +import org.openmetadata.service.util.EntityUtil; import org.openmetadata.service.util.FullyQualifiedName; import org.openmetadata.service.util.JsonUtils; @@ -49,6 +56,35 @@ public static TypeRegistry instance() { return INSTANCE; } + public final void initialize(TypeRepository repository) { + // Load types defined in OpenMetadata schemas + long now = System.currentTimeMillis(); + List types = JsonUtils.getTypes(); + types.forEach( + type -> { + type.withId(UUID.randomUUID()).withUpdatedBy(ADMIN_USER_NAME).withUpdatedAt(now); + LOG.debug("Loading type {}", type.getName()); + try { + EntityUtil.Fields fields = repository.getFields(PROPERTIES_FIELD); + try { + Type storedType = repository.getByName(null, type.getName(), fields); + type.setId(storedType.getId()); + // If entity type already exists, then carry forward custom properties + if (storedType.getCategory().equals(Category.Entity)) { + type.setCustomProperties(storedType.getCustomProperties()); + } + } catch (Exception e) { + LOG.debug( + "Type '{}' not found. Proceeding to add new type entity in database.", + type.getName()); + } + repository.addToRegistry(type); + } catch (Exception e) { + LOG.error("Error loading type {}", type.getName(), e); + } + }); + } + public void addType(Type type) { TYPES.put(type.getName(), type); @@ -111,34 +147,25 @@ public static String getPropertyName(String propertyFQN) { } public static String getCustomPropertyType(String entityType, String propertyName) { - Type type = TypeRegistry.TYPES.get(entityType); - if (type != null && type.getCustomProperties() != null) { - for (CustomProperty property : type.getCustomProperties()) { - if (property.getName().equals(propertyName)) { - return property.getPropertyType().getName(); - } - } + String fqn = getCustomPropertyFQN(entityType, propertyName); + CustomProperty property = CUSTOM_PROPERTIES.get(fqn); + if (property == null) { + throw EntityNotFoundException.byMessage( + CatalogExceptionMessage.entityNotFound(propertyName, entityType)); } - throw EntityNotFoundException.byMessage( - CatalogExceptionMessage.entityNotFound(Entity.TYPE, String.valueOf(type))); + return property.getPropertyType().getName(); } public static String getCustomPropertyConfig(String entityType, String propertyName) { - Type type = TypeRegistry.TYPES.get(entityType); - if (type != null && type.getCustomProperties() != null) { - for (CustomProperty property : type.getCustomProperties()) { - if (property.getName().equals(propertyName) - && property.getCustomPropertyConfig() != null - && property.getCustomPropertyConfig().getConfig() != null) { - Object config = property.getCustomPropertyConfig().getConfig(); - if (config instanceof String || config instanceof Integer) { - return config.toString(); // for simple type config return as string - } else { - return JsonUtils.pojoToJson( - config); // for complex object in config return as JSON string - } - } - } + String fqn = getCustomPropertyFQN(entityType, propertyName); + CustomProperty property = CUSTOM_PROPERTIES.get(fqn); + if (property != null + && property.getCustomPropertyConfig() != null + && property.getCustomPropertyConfig().getConfig() != null) { + Object config = property.getCustomPropertyConfig().getConfig(); + return (config instanceof String || config instanceof Integer) + ? config.toString() // for simple type config return as string + : JsonUtils.pojoToJson(config); // for complex object in config return as JSON string } return null; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/changeEvent/AbstractEventConsumer.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/changeEvent/AbstractEventConsumer.java index a0a11f031205..7db4c4e4f89c 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/changeEvent/AbstractEventConsumer.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/changeEvent/AbstractEventConsumer.java @@ -32,10 +32,12 @@ import org.openmetadata.schema.entity.events.EventSubscriptionOffset; import org.openmetadata.schema.entity.events.FailedEvent; import org.openmetadata.schema.entity.events.SubscriptionDestination; +import org.openmetadata.schema.system.EntityError; import org.openmetadata.schema.type.ChangeEvent; import org.openmetadata.service.Entity; import org.openmetadata.service.events.errors.EventPublisherException; import org.openmetadata.service.util.JsonUtils; +import org.openmetadata.service.util.ResultList; import org.quartz.DisallowConcurrentExecution; import org.quartz.Job; import org.quartz.JobDetail; @@ -70,8 +72,9 @@ private void init(JobExecutionContext context) { (EventSubscription) context.getJobDetail().getJobDataMap().get(ALERT_INFO_KEY); this.jobDetail = context.getJobDetail(); this.eventSubscription = sub; - this.offset = loadInitialOffset(context).getCurrentOffset(); - this.startingOffset = loadInitialOffset(context).getStartingOffset(); + EventSubscriptionOffset eventSubscriptionOffset = loadInitialOffset(context); + this.offset = eventSubscriptionOffset.getCurrentOffset(); + this.startingOffset = eventSubscriptionOffset.getStartingOffset(); this.alertMetrics = loadInitialMetrics(); this.destinationMap = loadDestinationsMap(context); this.doInit(context); @@ -240,34 +243,46 @@ public void commit(JobExecutionContext jobExecutionContext) { } @Override - public List pollEvents(long offset, long batchSize) { - // Read from Change Event Table + public ResultList pollEvents(long offset, long batchSize) { List eventJson = Entity.getCollectionDAO().changeEventDAO().list(batchSize, offset); - List changeEvents = new ArrayList<>(); + List errorEvents = new ArrayList<>(); for (String json : eventJson) { - ChangeEvent event = JsonUtils.readValue(json, ChangeEvent.class); - changeEvents.add(event); + try { + ChangeEvent event = JsonUtils.readValue(json, ChangeEvent.class); + changeEvents.add(event); + } catch (Exception ex) { + errorEvents.add(new EntityError().withMessage(ex.getMessage()).withEntity(json)); + LOG.error("Error in Parsing Change Event : {} , Message: {} ", json, ex.getMessage(), ex); + } } - return changeEvents; + return new ResultList<>(changeEvents, errorEvents, null, null, eventJson.size()); } @Override public void execute(JobExecutionContext jobExecutionContext) { // Must Have , Before Execute the Init, Quartz Requires a Non-Arg Constructor this.init(jobExecutionContext); - // Poll Events from Change Event Table - List batch = pollEvents(offset, eventSubscription.getBatchSize()); - int batchSize = batch.size(); - Map> eventsWithReceivers = createEventsWithReceivers(batch); + long batchSize = 0; + Map> eventsWithReceivers = new HashMap<>(); try { + // Poll Events from Change Event Table + ResultList batch = pollEvents(offset, eventSubscription.getBatchSize()); + batchSize = batch.getPaging().getTotal(); + eventsWithReceivers.putAll(createEventsWithReceivers(batch.getData())); // Publish Events if (!eventsWithReceivers.isEmpty()) { alertMetrics.withTotalEvents(alertMetrics.getTotalEvents() + eventsWithReceivers.size()); publishEvents(eventsWithReceivers); } } catch (Exception e) { - LOG.error("Error in executing the Job : {} ", e.getMessage()); + LOG.error( + "Error in polling events for alert : {} , Offset : {} , Batch Size : {} ", + e.getMessage(), + offset, + batchSize, + e); + } finally { if (!eventsWithReceivers.isEmpty()) { // Commit the Offset diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/changeEvent/Consumer.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/changeEvent/Consumer.java index 9f1c8482062a..0d1be623ce08 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/changeEvent/Consumer.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/changeEvent/Consumer.java @@ -13,16 +13,16 @@ package org.openmetadata.service.apps.bundles.changeEvent; -import java.util.List; import java.util.Map; import java.util.Set; import java.util.UUID; import org.openmetadata.schema.type.ChangeEvent; import org.openmetadata.service.events.errors.EventPublisherException; +import org.openmetadata.service.util.ResultList; import org.quartz.JobExecutionContext; public interface Consumer { - List pollEvents(long offset, long batchSize); + ResultList pollEvents(long offset, long batchSize); void publishEvents(Map> events); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/dataRetention/DataRetention.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/dataRetention/DataRetention.java new file mode 100644 index 000000000000..214807bd10ba --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/dataRetention/DataRetention.java @@ -0,0 +1,100 @@ +package org.openmetadata.service.apps.bundles.dataRetention; + +import java.time.Duration; +import java.time.Instant; +import java.util.function.Supplier; +import lombok.extern.slf4j.Slf4j; +import org.jdbi.v3.sqlobject.transaction.Transaction; +import org.openmetadata.common.utils.CommonUtil; +import org.openmetadata.schema.entity.app.App; +import org.openmetadata.schema.entity.applications.configuration.internal.DataRetentionConfiguration; +import org.openmetadata.service.apps.AbstractNativeApplication; +import org.openmetadata.service.jdbi3.CollectionDAO; +import org.openmetadata.service.search.SearchRepository; +import org.openmetadata.service.util.JsonUtils; +import org.quartz.JobExecutionContext; + +@Slf4j +public class DataRetention extends AbstractNativeApplication { + private static final int BATCH_SIZE = 10_000; + private DataRetentionConfiguration dataRetentionConfiguration; + private final CollectionDAO.EventSubscriptionDAO eventSubscriptionDAO; + + public DataRetention(CollectionDAO collectionDAO, SearchRepository searchRepository) { + super(collectionDAO, searchRepository); + this.eventSubscriptionDAO = collectionDAO.eventSubscriptionDAO(); + } + + @Override + public void init(App app) { + super.init(app); + this.dataRetentionConfiguration = + JsonUtils.convertValue(app.getAppConfiguration(), DataRetentionConfiguration.class); + if (CommonUtil.nullOrEmpty(this.dataRetentionConfiguration)) { + LOG.warn("No retention policy configuration provided. Cleanup tasks will not run."); + } + } + + @Override + public void startApp(JobExecutionContext jobExecutionContext) { + executeCleanup(dataRetentionConfiguration); + } + + public void executeCleanup(DataRetentionConfiguration config) { + if (CommonUtil.nullOrEmpty(config)) { + return; + } + + cleanChangeEvents(config.getChangeEventRetentionPeriod()); + } + + @Transaction + private void cleanChangeEvents(int retentionPeriod) { + LOG.info( + "Initiating change events cleanup: Deleting records with a retention period of {} days.", + retentionPeriod); + long cutoffMillis = getRetentionCutoffMillis(retentionPeriod); + + int totalDeletedSuccessfulEvents = + batchDelete( + () -> + eventSubscriptionDAO.deleteSuccessfulSentChangeEventsInBatches( + cutoffMillis, BATCH_SIZE)); + + int totalDeletedChangeEvents = + batchDelete( + () -> eventSubscriptionDAO.deleteChangeEventsInBatches(cutoffMillis, BATCH_SIZE)); + + int totalDeletedDlq = + batchDelete( + () -> eventSubscriptionDAO.deleteConsumersDlqInBatches(cutoffMillis, BATCH_SIZE)); + + LOG.info( + "Change events cleanup completed: {} successful_sent_change_events, {} change_events, and {} consumers_dlq records deleted (retention period: {} days).", + totalDeletedSuccessfulEvents, + totalDeletedChangeEvents, + totalDeletedDlq, + retentionPeriod); + } + + private long getRetentionCutoffMillis(int retentionPeriodInDays) { + return Instant.now() + .minusMillis(Duration.ofDays(retentionPeriodInDays).toMillis()) + .toEpochMilli(); + } + + /** + * Runs a batch delete operation in a loop until fewer than BATCH_SIZE records are deleted in a single iteration. + */ + private int batchDelete(Supplier deleteFunction) { + var totalDeleted = 0; + while (true) { + var deletedCount = deleteFunction.get(); + totalDeleted += deletedCount; + if (deletedCount < BATCH_SIZE) { + break; + } + } + return totalDeleted; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java index d0b5faf7b6ed..d46f625b6bbd 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsApp.java @@ -146,11 +146,19 @@ private void deleteDataQualityDataIndex() { private void createDataAssetsDataStream() { DataInsightsSearchInterface searchInterface = getSearchInterface(); + ElasticSearchConfiguration config = searchRepository.getElasticSearchConfiguration(); + String language = + config != null && config.getSearchIndexMappingLanguage() != null + ? config.getSearchIndexMappingLanguage().value() + : "en"; + try { for (String dataAssetType : dataAssetTypes) { + IndexMapping dataAssetIndex = searchRepository.getIndexMapping(dataAssetType); String dataStreamName = getDataStreamName(dataAssetType); if (!searchInterface.dataAssetDataStreamExists(dataStreamName)) { - searchInterface.createDataAssetsDataStream(dataStreamName); + searchInterface.createDataAssetsDataStream( + dataStreamName, dataAssetType, dataAssetIndex, language); } } } catch (IOException ex) { @@ -312,7 +320,13 @@ private WorkflowStats processCostAnalysis() { private WorkflowStats processDataAssets() { DataAssetsWorkflow workflow = new DataAssetsWorkflow( - timestamp, batchSize, backfill, dataAssetTypes, collectionDAO, searchRepository); + timestamp, + batchSize, + backfill, + dataAssetTypes, + collectionDAO, + searchRepository, + getSearchInterface()); WorkflowStats workflowStats = workflow.getWorkflowStats(); try { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/search/DataInsightsSearchConfiguration.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/search/DataInsightsSearchConfiguration.java new file mode 100644 index 000000000000..d9fc4dc1672f --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/search/DataInsightsSearchConfiguration.java @@ -0,0 +1,10 @@ +package org.openmetadata.service.apps.bundles.insights.search; + +import java.util.List; +import java.util.Map; +import lombok.Getter; + +@Getter +public class DataInsightsSearchConfiguration { + private Map> mappingFields; +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/search/DataInsightsSearchInterface.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/search/DataInsightsSearchInterface.java index 0d06b3e03be2..3cba896e8ee5 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/search/DataInsightsSearchInterface.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/search/DataInsightsSearchInterface.java @@ -2,9 +2,13 @@ import java.io.IOException; import java.io.InputStream; +import java.util.List; import org.openmetadata.service.exception.UnhandledServerException; +import org.openmetadata.service.search.models.IndexMapping; +import org.openmetadata.service.util.JsonUtils; public interface DataInsightsSearchInterface { + String DATA_INSIGHTS_SEARCH_CONFIG_PATH = "/dataInsights/config.json"; void createLifecyclePolicy(String name, String policy) throws IOException; @@ -23,7 +27,63 @@ default String readResource(String resourceFile) { } } - void createDataAssetsDataStream(String name) throws IOException; + default String buildMapping( + String entityType, + IndexMapping entityIndexMapping, + String language, + String indexMappingTemplateStr) { + IndexMappingTemplate indexMappingTemplate = + JsonUtils.readOrConvertValue(indexMappingTemplateStr, IndexMappingTemplate.class); + EntityIndexMap entityIndexMap = + JsonUtils.readOrConvertValue( + readResource( + String.format(entityIndexMapping.getIndexMappingFile(), language.toLowerCase())), + EntityIndexMap.class); + + DataInsightsSearchConfiguration dataInsightsSearchConfiguration = + readDataInsightsSearchConfiguration(); + List entityAttributeFields = + getEntityAttributeFields(dataInsightsSearchConfiguration, entityType); + + indexMappingTemplate + .getTemplate() + .getSettings() + .put("analysis", entityIndexMap.getSettings().get("analysis")); + + for (String attribute : entityAttributeFields) { + if (!indexMappingTemplate + .getTemplate() + .getMappings() + .getProperties() + .containsKey(attribute)) { + Object value = entityIndexMap.getMappings().getProperties().get(attribute); + if (value != null) { + indexMappingTemplate.getTemplate().getMappings().getProperties().put(attribute, value); + } + } + } + + return JsonUtils.pojoToJson(indexMappingTemplate); + } + + default DataInsightsSearchConfiguration readDataInsightsSearchConfiguration() { + return JsonUtils.readOrConvertValue( + readResource(DATA_INSIGHTS_SEARCH_CONFIG_PATH), DataInsightsSearchConfiguration.class); + } + + default List getEntityAttributeFields( + DataInsightsSearchConfiguration dataInsightsSearchConfiguration, String entityType) { + List entityAttributeFields = + dataInsightsSearchConfiguration.getMappingFields().get("common"); + entityAttributeFields.addAll( + dataInsightsSearchConfiguration.getMappingFields().get(entityType)); + + return entityAttributeFields; + } + + void createDataAssetsDataStream( + String name, String entityType, IndexMapping entityIndexMapping, String language) + throws IOException; void deleteDataAssetDataStream(String name) throws IOException; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/search/EntityIndexMap.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/search/EntityIndexMap.java new file mode 100644 index 000000000000..8a5e9b64dbbf --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/search/EntityIndexMap.java @@ -0,0 +1,18 @@ +package org.openmetadata.service.apps.bundles.insights.search; + +import java.util.Map; +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +public class EntityIndexMap { + private Map settings; + private Mappings mappings; + + @Getter + @Setter + public static class Mappings { + private Map properties; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/search/IndexMappingTemplate.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/search/IndexMappingTemplate.java new file mode 100644 index 000000000000..c418ab63b757 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/search/IndexMappingTemplate.java @@ -0,0 +1,8 @@ +package org.openmetadata.service.apps.bundles.insights.search; + +import lombok.Getter; + +@Getter +public class IndexMappingTemplate { + private EntityIndexMap template; +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/search/elasticsearch/ElasticSearchDataInsightsClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/search/elasticsearch/ElasticSearchDataInsightsClient.java index 150fc48c33cc..d5a1bbab55cd 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/search/elasticsearch/ElasticSearchDataInsightsClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/search/elasticsearch/ElasticSearchDataInsightsClient.java @@ -5,6 +5,7 @@ import es.org.elasticsearch.client.RestClient; import java.io.IOException; import org.openmetadata.service.apps.bundles.insights.search.DataInsightsSearchInterface; +import org.openmetadata.service.search.models.IndexMapping; public class ElasticSearchDataInsightsClient implements DataInsightsSearchInterface { private final RestClient client; @@ -52,7 +53,9 @@ public Boolean dataAssetDataStreamExists(String name) throws IOException { } @Override - public void createDataAssetsDataStream(String name) throws IOException { + public void createDataAssetsDataStream( + String name, String entityType, IndexMapping entityIndexMapping, String language) + throws IOException { String resourcePath = "/dataInsights/elasticsearch"; createLifecyclePolicy( "di-data-assets-lifecycle", @@ -62,7 +65,11 @@ public void createDataAssetsDataStream(String name) throws IOException { readResource(String.format("%s/indexSettingsTemplate.json", resourcePath))); createComponentTemplate( "di-data-assets-mapping", - readResource(String.format("%s/indexMappingsTemplate.json", resourcePath))); + buildMapping( + entityType, + entityIndexMapping, + language, + readResource(String.format("%s/indexMappingsTemplate.json", resourcePath)))); createIndexTemplate( "di-data-assets", readResource(String.format("%s/indexTemplate.json", resourcePath))); createDataStream(name); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/search/opensearch/OpenSearchDataInsightsClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/search/opensearch/OpenSearchDataInsightsClient.java index da94394a3a5f..8d8096a8adfa 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/search/opensearch/OpenSearchDataInsightsClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/search/opensearch/OpenSearchDataInsightsClient.java @@ -2,6 +2,7 @@ import java.io.IOException; import org.openmetadata.service.apps.bundles.insights.search.DataInsightsSearchInterface; +import org.openmetadata.service.search.models.IndexMapping; import os.org.opensearch.client.Request; import os.org.opensearch.client.Response; import os.org.opensearch.client.ResponseException; @@ -63,14 +64,20 @@ public Boolean dataAssetDataStreamExists(String name) throws IOException { } @Override - public void createDataAssetsDataStream(String name) throws IOException { + public void createDataAssetsDataStream( + String name, String entityType, IndexMapping entityIndexMapping, String language) + throws IOException { String resourcePath = "/dataInsights/opensearch"; createLifecyclePolicy( "di-data-assets-lifecycle", readResource(String.format("%s/indexLifecyclePolicy.json", resourcePath))); createComponentTemplate( "di-data-assets-mapping", - readResource(String.format("%s/indexMappingsTemplate.json", resourcePath))); + buildMapping( + entityType, + entityIndexMapping, + language, + readResource(String.format("%s/indexMappingsTemplate.json", resourcePath)))); createIndexTemplate( "di-data-assets", readResource(String.format("%s/indexTemplate.json", resourcePath))); createDataStream(name); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/workflows/dataAssets/DataAssetsWorkflow.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/workflows/dataAssets/DataAssetsWorkflow.java index 2e30832ea5d8..938cba712766 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/workflows/dataAssets/DataAssetsWorkflow.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/workflows/dataAssets/DataAssetsWorkflow.java @@ -23,6 +23,8 @@ import org.openmetadata.schema.system.StepStats; import org.openmetadata.schema.type.Include; import org.openmetadata.service.apps.bundles.insights.DataInsightsApp; +import org.openmetadata.service.apps.bundles.insights.search.DataInsightsSearchConfiguration; +import org.openmetadata.service.apps.bundles.insights.search.DataInsightsSearchInterface; import org.openmetadata.service.apps.bundles.insights.utils.TimestampUtils; import org.openmetadata.service.apps.bundles.insights.workflows.WorkflowStats; import org.openmetadata.service.apps.bundles.insights.workflows.dataAssets.processors.DataInsightsElasticSearchProcessor; @@ -43,6 +45,7 @@ @Slf4j public class DataAssetsWorkflow { public static final String DATA_STREAM_KEY = "DataStreamKey"; + public static final String ENTITY_TYPE_FIELDS_KEY = "EnityTypeFields"; private final int retentionDays = 30; private final Long startTimestamp; private final Long endTimestamp; @@ -51,6 +54,8 @@ public class DataAssetsWorkflow { private final CollectionDAO collectionDAO; private final List sources = new ArrayList<>(); private final Set entityTypes; + private final DataInsightsSearchConfiguration dataInsightsSearchConfiguration; + private final DataInsightsSearchInterface searchInterface; private DataInsightsEntityEnricherProcessor entityEnricher; private Processor entityProcessor; @@ -63,7 +68,8 @@ public DataAssetsWorkflow( Optional backfill, Set entityTypes, CollectionDAO collectionDAO, - SearchRepository searchRepository) { + SearchRepository searchRepository, + DataInsightsSearchInterface searchInterface) { if (backfill.isPresent()) { Long oldestPossibleTimestamp = TimestampUtils.getStartOfDayTimestamp( @@ -95,6 +101,8 @@ public DataAssetsWorkflow( this.searchRepository = searchRepository; this.collectionDAO = collectionDAO; this.entityTypes = entityTypes; + this.searchInterface = searchInterface; + this.dataInsightsSearchConfiguration = searchInterface.readDataInsightsSearchConfiguration(); } private void initialize() { @@ -146,6 +154,10 @@ public void process() throws SearchIndexException { deleteDataBeforeInserting(getDataStreamName(source.getEntityType())); contextData.put(DATA_STREAM_KEY, getDataStreamName(source.getEntityType())); contextData.put(ENTITY_TYPE_KEY, source.getEntityType()); + contextData.put( + ENTITY_TYPE_FIELDS_KEY, + searchInterface.getEntityAttributeFields( + dataInsightsSearchConfiguration, source.getEntityType())); while (!source.isDone().get()) { try { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/workflows/dataAssets/processors/DataInsightsEntityEnricherProcessor.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/workflows/dataAssets/processors/DataInsightsEntityEnricherProcessor.java index a1714dc07eb3..210aaf38bdbe 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/workflows/dataAssets/processors/DataInsightsEntityEnricherProcessor.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/workflows/dataAssets/processors/DataInsightsEntityEnricherProcessor.java @@ -3,6 +3,8 @@ import static org.openmetadata.schema.EntityInterface.ENTITY_TYPE_TO_CLASS_MAP; import static org.openmetadata.service.apps.bundles.insights.utils.TimestampUtils.END_TIMESTAMP_KEY; import static org.openmetadata.service.apps.bundles.insights.utils.TimestampUtils.START_TIMESTAMP_KEY; +import static org.openmetadata.service.apps.bundles.insights.workflows.dataAssets.DataAssetsWorkflow.ENTITY_TYPE_FIELDS_KEY; +import static org.openmetadata.service.search.SearchIndexUtils.parseFollowers; import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.ENTITY_TYPE_KEY; import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.TIMESTAMP_KEY; import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.getUpdatedStats; @@ -141,6 +143,8 @@ private Map enrichEntity( Long endTimestamp = (Long) entityVersionMap.get("endTimestamp"); Map entityMap = JsonUtils.getMap(entity); + entityMap.keySet().retainAll((List) contextData.get(ENTITY_TYPE_FIELDS_KEY)); + String entityType = (String) contextData.get(ENTITY_TYPE_KEY); List> interfaces = List.of(entity.getClass().getInterfaces()); @@ -224,14 +228,12 @@ private Map enrichEntity( } // Modify Custom Property key - Optional oCustomProperties = Optional.ofNullable(entityMap.remove("extension")); + Optional oCustomProperties = Optional.ofNullable(entityMap.get("extension")); oCustomProperties.ifPresent( o -> entityMap.put(String.format("%sCustomProperty", entityType), o)); - // Remove 'changeDescription' field - entityMap.remove("changeDescription"); - // Remove 'sampleData' - entityMap.remove("sampleData"); + // Parse Followers: + entityMap.put("followers", parseFollowers(entity.getFollowers())); return entityMap; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/ElasticSearchIndexSink.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/ElasticSearchIndexSink.java index af2ca8bb53e3..12bac6ad68db 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/ElasticSearchIndexSink.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/ElasticSearchIndexSink.java @@ -86,7 +86,10 @@ public void write(List entities, Map contextData) throws Sear } catch (Exception e) { entityErrorList.add( new EntityError() - .withMessage("Failed to convert entity to request: " + e.getMessage()) + .withMessage( + String.format( + "Failed to convert entity to request: %s , Stack : %s", + e.getMessage(), ExceptionUtils.exceptionStackTraceAsString(e))) .withEntity(entity.toString())); LOG.error("Error converting entity to request", e); } @@ -173,6 +176,14 @@ public void onFailure(Exception e) { handleNonRetriableException(requests.size(), e); } } catch (Exception ex) { + entityErrorList.add( + new EntityError() + .withMessage( + String.format( + "Bulk request failed: %s, StackTrace: %s", + ex.getMessage(), + ExceptionUtils.exceptionStackTraceAsString(ex))) + .withEntity(requests.toString())); LOG.error("Bulk request retry attempt {}/{} failed", 1, maxRetries, ex); } finally { semaphore.release(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/OpenSearchIndexSink.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/OpenSearchIndexSink.java index 4a3b7384619b..65ef6d1737cb 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/OpenSearchIndexSink.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/OpenSearchIndexSink.java @@ -87,7 +87,10 @@ public void write(List entities, Map contextData) throws Sear } catch (Exception e) { entityErrorList.add( new EntityError() - .withMessage("Failed to convert entity to request: " + e.getMessage()) + .withMessage( + String.format( + "Failed to convert entity to request: %s , Stack : %s", + e.getMessage(), ExceptionUtils.exceptionStackTraceAsString(e))) .withEntity(entity.toString())); LOG.error("Error converting entity to request", e); } @@ -174,6 +177,14 @@ public void onFailure(Exception e) { handleNonRetriableException(requests.size(), e); } } catch (Exception ex) { + entityErrorList.add( + new EntityError() + .withMessage( + String.format( + "Bulk request failed: %s, StackTrace: %s", + ex.getMessage(), + ExceptionUtils.exceptionStackTraceAsString(ex))) + .withEntity(requests.toString())); LOG.error("Bulk request retry attempt {}/{} failed", 1, maxRetries, ex); } finally { semaphore.release(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java index fd34c2027f5c..49d23f87b74b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java @@ -1,47 +1,8 @@ package org.openmetadata.service.apps.bundles.searchIndex; import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; -import static org.openmetadata.service.Entity.API_COLLCECTION; -import static org.openmetadata.service.Entity.API_ENDPOINT; -import static org.openmetadata.service.Entity.API_SERVICE; -import static org.openmetadata.service.Entity.CHART; -import static org.openmetadata.service.Entity.CLASSIFICATION; -import static org.openmetadata.service.Entity.CONTAINER; -import static org.openmetadata.service.Entity.DASHBOARD; -import static org.openmetadata.service.Entity.DASHBOARD_DATA_MODEL; -import static org.openmetadata.service.Entity.DASHBOARD_SERVICE; -import static org.openmetadata.service.Entity.DATABASE; -import static org.openmetadata.service.Entity.DATABASE_SCHEMA; -import static org.openmetadata.service.Entity.DATABASE_SERVICE; -import static org.openmetadata.service.Entity.DATA_PRODUCT; -import static org.openmetadata.service.Entity.DOMAIN; -import static org.openmetadata.service.Entity.ENTITY_REPORT_DATA; -import static org.openmetadata.service.Entity.GLOSSARY; -import static org.openmetadata.service.Entity.GLOSSARY_TERM; -import static org.openmetadata.service.Entity.INGESTION_PIPELINE; -import static org.openmetadata.service.Entity.MESSAGING_SERVICE; -import static org.openmetadata.service.Entity.METADATA_SERVICE; -import static org.openmetadata.service.Entity.METRIC; -import static org.openmetadata.service.Entity.MLMODEL; -import static org.openmetadata.service.Entity.MLMODEL_SERVICE; -import static org.openmetadata.service.Entity.PIPELINE; -import static org.openmetadata.service.Entity.PIPELINE_SERVICE; -import static org.openmetadata.service.Entity.QUERY; -import static org.openmetadata.service.Entity.SEARCH_INDEX; -import static org.openmetadata.service.Entity.SEARCH_SERVICE; -import static org.openmetadata.service.Entity.STORAGE_SERVICE; -import static org.openmetadata.service.Entity.STORED_PROCEDURE; -import static org.openmetadata.service.Entity.TABLE; -import static org.openmetadata.service.Entity.TAG; -import static org.openmetadata.service.Entity.TEAM; -import static org.openmetadata.service.Entity.TEST_CASE; import static org.openmetadata.service.Entity.TEST_CASE_RESOLUTION_STATUS; import static org.openmetadata.service.Entity.TEST_CASE_RESULT; -import static org.openmetadata.service.Entity.TEST_SUITE; -import static org.openmetadata.service.Entity.TOPIC; -import static org.openmetadata.service.Entity.USER; -import static org.openmetadata.service.Entity.WEB_ANALYTIC_ENTITY_VIEW_REPORT_DATA; -import static org.openmetadata.service.Entity.WEB_ANALYTIC_USER_ACTIVITY_REPORT_DATA; import static org.openmetadata.service.apps.scheduler.AbstractOmAppJobListener.APP_RUN_STATS; import static org.openmetadata.service.apps.scheduler.AbstractOmAppJobListener.WEBSOCKET_STATUS_CHANNEL; import static org.openmetadata.service.apps.scheduler.AppScheduler.ON_DEMAND_JOB; @@ -99,53 +60,8 @@ @Slf4j public class SearchIndexApp extends AbstractNativeApplication { - private static final String ALL = "all"; - public static final Set ALL_ENTITIES = - Set.of( - TABLE, - DASHBOARD, - TOPIC, - PIPELINE, - INGESTION_PIPELINE, - SEARCH_INDEX, - USER, - TEAM, - GLOSSARY, - GLOSSARY_TERM, - MLMODEL, - TAG, - CLASSIFICATION, - QUERY, - CONTAINER, - DATABASE, - DATABASE_SCHEMA, - TEST_CASE, - TEST_SUITE, - CHART, - DASHBOARD_DATA_MODEL, - DATABASE_SERVICE, - MESSAGING_SERVICE, - DASHBOARD_SERVICE, - PIPELINE_SERVICE, - MLMODEL_SERVICE, - STORAGE_SERVICE, - METADATA_SERVICE, - SEARCH_SERVICE, - ENTITY_REPORT_DATA, - WEB_ANALYTIC_ENTITY_VIEW_REPORT_DATA, - WEB_ANALYTIC_USER_ACTIVITY_REPORT_DATA, - DOMAIN, - STORED_PROCEDURE, - DATA_PRODUCT, - TEST_CASE_RESOLUTION_STATUS, - TEST_CASE_RESULT, - API_SERVICE, - API_ENDPOINT, - API_COLLCECTION, - METRIC); - public static final Set TIME_SERIES_ENTITIES = Set.of( ReportData.ReportDataType.ENTITY_REPORT_DATA.value(), @@ -180,8 +96,9 @@ public void init(App app) { JsonUtils.convertValue(app.getAppConfiguration(), EventPublisherJob.class) .withStats(new Stats()); - if (request.getEntities().contains(ALL)) { - request.setEntities(ALL_ENTITIES); + if (request.getEntities().size() == 1 && request.getEntities().contains(ALL)) { + SearchRepository searchRepo = Entity.getSearchRepo(); + request.setEntities(searchRepo.getSearchEntities()); } jobData = request; @@ -199,6 +116,7 @@ public void startApp(JobExecutionContext jobExecutionContext) { jobData.setRecreateIndex(false); } + reCreateIndexes(jobData.getEntities()); performReindex(jobExecutionContext); } catch (InterruptedException ex) { Thread.currentThread().interrupt(); @@ -326,7 +244,6 @@ private void submitProducerTask( jobExecutor.submit( () -> { try { - reCreateIndexes(entityType); int totalEntityRecords = getTotalEntityRecords(entityType); Source source = createSource(entityType); int loadPerThread = calculateNumberOfThreads(totalEntityRecords); @@ -517,20 +434,22 @@ private void sendUpdates(JobExecutionContext jobExecutionContext) { } } - private void reCreateIndexes(String entityType) throws SearchIndexException { - if (Boolean.FALSE.equals(jobData.getRecreateIndex())) { - LOG.debug("RecreateIndex is false. Skipping index recreation for '{}'.", entityType); - return; - } + private void reCreateIndexes(Set entities) throws SearchIndexException { + for (String entityType : entities) { + if (Boolean.FALSE.equals(jobData.getRecreateIndex())) { + LOG.debug("RecreateIndex is false. Skipping index recreation for '{}'.", entityType); + return; + } - try { - IndexMapping indexType = searchRepository.getIndexMapping(entityType); - searchRepository.deleteIndex(indexType); - searchRepository.createIndex(indexType); - LOG.info("Recreated index for entityType '{}'.", entityType); - } catch (Exception e) { - LOG.error("Failed to recreate index for entityType '{}'.", entityType, e); - throw new SearchIndexException(e); + try { + IndexMapping indexType = searchRepository.getIndexMapping(entityType); + searchRepository.deleteIndex(indexType); + searchRepository.createIndex(indexType); + LOG.info("Recreated index for entityType '{}'.", entityType); + } catch (Exception e) { + LOG.error("Failed to recreate index for entityType '{}'.", entityType, e); + throw new SearchIndexException(e); + } } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/scheduler/AbstractOmAppJobListener.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/scheduler/AbstractOmAppJobListener.java index 8d256f4167fa..d110da423fca 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/scheduler/AbstractOmAppJobListener.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/scheduler/AbstractOmAppJobListener.java @@ -61,7 +61,8 @@ public void jobToBeExecuted(JobExecutionContext jobExecutionContext) { .withTimestamp(jobStartTime) .withRunType(runType) .withStatus(AppRunRecord.Status.RUNNING) - .withScheduleInfo(jobApp.getAppSchedule()); + .withScheduleInfo(jobApp.getAppSchedule()) + .withConfig(JsonUtils.getMap(jobApp.getAppConfiguration())); boolean update = false; if (jobExecutionContext.isRecovering()) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/scheduler/AppScheduler.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/scheduler/AppScheduler.java index 3163fc9a2e66..542edba0ccd6 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/scheduler/AppScheduler.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/scheduler/AppScheduler.java @@ -38,6 +38,7 @@ import org.quartz.TriggerBuilder; import org.quartz.TriggerKey; import org.quartz.impl.StdSchedulerFactory; +import org.quartz.impl.matchers.GroupMatcher; @Slf4j public class AppScheduler { @@ -93,10 +94,32 @@ private AppScheduler( .getListenerManager() .addJobListener(new OmAppJobListener(dao), jobGroupEquals(APPS_JOB_GROUP)); + this.resetErrorTriggers(); + // Start Scheduler this.scheduler.start(); } + private void resetErrorTriggers() { + try { + scheduler + .getTriggerKeys(GroupMatcher.anyGroup()) + .forEach( + triggerKey -> { + try { + if (scheduler.getTriggerState(triggerKey) == Trigger.TriggerState.ERROR) { + LOG.info("Resetting trigger {} from error state", triggerKey); + scheduler.resetTriggerFromErrorState(triggerKey); + } + } catch (SchedulerException e) { + throw new RuntimeException(e); + } + }); + } catch (SchedulerException ex) { + LOG.error("Failed to reset failed triggers", ex); + } + } + public static void initialize( OpenMetadataApplicationConfig config, CollectionDAO dao, SearchRepository searchClient) throws SchedulerException { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/events/scheduled/DatabseAndSearchServiceStatusJob.java b/openmetadata-service/src/main/java/org/openmetadata/service/events/scheduled/DatabseAndSearchServiceStatusJob.java new file mode 100644 index 000000000000..5fac9281e6f1 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/events/scheduled/DatabseAndSearchServiceStatusJob.java @@ -0,0 +1,55 @@ +package org.openmetadata.service.events.scheduled; + +import static org.openmetadata.service.events.scheduled.ServicesStatusJobHandler.JOB_CONTEXT_METER_REGISTRY; +import static org.openmetadata.service.events.scheduled.ServicesStatusJobHandler.UNHEALTHY_STATUS; + +import io.micrometer.core.instrument.Counter; +import io.micrometer.prometheus.PrometheusMeterRegistry; +import lombok.extern.slf4j.Slf4j; +import org.openmetadata.service.Entity; +import org.openmetadata.service.search.SearchHealthStatus; +import org.quartz.Job; +import org.quartz.JobExecutionContext; + +@Slf4j +public class DatabseAndSearchServiceStatusJob implements Job { + private static final String SERVICE_COUNTER = "omd_service_unreachable"; + private static final String SERVICE_NAME = "service_name"; + private static final String SEARCH_SERVICE_NAME = "search"; + private static final String DATABASE_SERVICE_NAME = "database"; + + @Override + public void execute(JobExecutionContext jobExecutionContext) { + PrometheusMeterRegistry meterRegistry = + (PrometheusMeterRegistry) + jobExecutionContext.getJobDetail().getJobDataMap().get(JOB_CONTEXT_METER_REGISTRY); + checkDatabaseStatus(meterRegistry); + checkElasticSearchStatus(meterRegistry); + } + + private void checkElasticSearchStatus(PrometheusMeterRegistry meterRegistry) { + try { + SearchHealthStatus status = + Entity.getSearchRepository().getSearchClient().getSearchHealthStatus(); + if (status.getStatus().equals(UNHEALTHY_STATUS)) { + publishUnhealthyCounter(meterRegistry, SERVICE_NAME, SEARCH_SERVICE_NAME); + } + } catch (Exception ex) { + LOG.error("Elastic Search Health Check encountered issues: {}", ex.getMessage()); + publishUnhealthyCounter(meterRegistry, SERVICE_NAME, SEARCH_SERVICE_NAME); + } + } + + private void checkDatabaseStatus(PrometheusMeterRegistry meterRegistry) { + try { + Entity.getCollectionDAO().systemDAO().testConnection(); + } catch (Exception ex) { + LOG.error("Database Health Check encountered issues: {}", ex.getMessage()); + publishUnhealthyCounter(meterRegistry, SERVICE_NAME, DATABASE_SERVICE_NAME); + } + } + + private void publishUnhealthyCounter(PrometheusMeterRegistry meterRegistry, String... tags) { + Counter.builder(SERVICE_COUNTER).tags(tags).register(meterRegistry).increment(); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/events/scheduled/EventSubscriptionCleanupJob.java b/openmetadata-service/src/main/java/org/openmetadata/service/events/scheduled/EventSubscriptionCleanupJob.java deleted file mode 100644 index f0eb43ab9cc6..000000000000 --- a/openmetadata-service/src/main/java/org/openmetadata/service/events/scheduled/EventSubscriptionCleanupJob.java +++ /dev/null @@ -1,41 +0,0 @@ -package org.openmetadata.service.events.scheduled; - -import java.util.List; -import lombok.extern.slf4j.Slf4j; -import org.jdbi.v3.sqlobject.transaction.Transaction; -import org.openmetadata.service.Entity; -import org.quartz.Job; -import org.quartz.JobExecutionContext; -import org.quartz.JobExecutionException; - -@Slf4j -public class EventSubscriptionCleanupJob implements Job { - private static final int TARGET_COUNT = 50; - private static final int THRESHOLD = 100; - - @Override - public void execute(JobExecutionContext context) throws JobExecutionException { - performCleanup(); - } - - @Transaction - public static void performCleanup() { - List subscriptionsToClean = - Entity.getCollectionDAO().eventSubscriptionDAO().findSubscriptionsAboveThreshold(THRESHOLD); - - for (String subscriptionId : subscriptionsToClean) { - long recordCount = - Entity.getCollectionDAO().eventSubscriptionDAO().getSuccessfulRecordCount(subscriptionId); - - long excessRecords = recordCount - TARGET_COUNT; - if (excessRecords > 0) { - Entity.getCollectionDAO() - .eventSubscriptionDAO() - .deleteOldRecords(subscriptionId, excessRecords); - } - } - LOG.debug( - "Performed cleanup for subscriptions, retaining {} records per subscription.", - TARGET_COUNT); - } -} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/events/scheduled/EventSubscriptionScheduler.java b/openmetadata-service/src/main/java/org/openmetadata/service/events/scheduled/EventSubscriptionScheduler.java index d76bcafcebeb..f49900b1a66d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/events/scheduled/EventSubscriptionScheduler.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/events/scheduled/EventSubscriptionScheduler.java @@ -61,7 +61,6 @@ public class EventSubscriptionScheduler { public static final String ALERT_TRIGGER_GROUP = "OMAlertJobGroup"; private static EventSubscriptionScheduler instance; private static volatile boolean initialized = false; - public static volatile boolean cleanupJobInitialised = false; private final Scheduler alertsScheduler = new StdSchedulerFactory().getScheduler(); @@ -118,7 +117,6 @@ public void addSubscriptionPublisher(EventSubscription eventSubscription) // Schedule the Job alertsScheduler.scheduleJob(jobDetail, trigger); - instance.scheduleCleanupJob(); LOG.info( "Event Subscription started as {} : status {} for all Destinations", @@ -171,33 +169,6 @@ public void deleteEventSubscriptionPublisher(EventSubscription deletedEntity) LOG.info("Alert publisher deleted for {}", deletedEntity.getName()); } - public void scheduleCleanupJob() { - if (!cleanupJobInitialised) { - try { - JobDetail cleanupJob = - JobBuilder.newJob(EventSubscriptionCleanupJob.class) - .withIdentity("CleanupJob", ALERT_JOB_GROUP) - .build(); - - Trigger cleanupTrigger = - TriggerBuilder.newTrigger() - .withIdentity("CleanupTrigger", ALERT_TRIGGER_GROUP) - .withSchedule( - SimpleScheduleBuilder.simpleSchedule() - .withIntervalInSeconds(10) - .repeatForever()) - .startNow() - .build(); - - alertsScheduler.scheduleJob(cleanupJob, cleanupTrigger); - cleanupJobInitialised = true; - LOG.info("Scheduled periodic cleanup job to run every 10 seconds."); - } catch (SchedulerException e) { - LOG.error("Failed to schedule cleanup job", e); - } - } - } - @Transaction public void deleteSuccessfulAndFailedEventsRecordByAlert(UUID id) { Entity.getCollectionDAO() diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/events/scheduled/PipelineServiceStatusJob.java b/openmetadata-service/src/main/java/org/openmetadata/service/events/scheduled/PipelineServiceStatusJob.java index e2c368c965f0..bc45a4bc0570 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/events/scheduled/PipelineServiceStatusJob.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/events/scheduled/PipelineServiceStatusJob.java @@ -2,9 +2,9 @@ import static org.openmetadata.sdk.PipelineServiceClientInterface.HEALTHY_STATUS; import static org.openmetadata.sdk.PipelineServiceClientInterface.STATUS_KEY; -import static org.openmetadata.service.events.scheduled.PipelineServiceStatusJobHandler.JOB_CONTEXT_CLUSTER_NAME; -import static org.openmetadata.service.events.scheduled.PipelineServiceStatusJobHandler.JOB_CONTEXT_METER_REGISTRY; -import static org.openmetadata.service.events.scheduled.PipelineServiceStatusJobHandler.JOB_CONTEXT_PIPELINE_SERVICE_CLIENT; +import static org.openmetadata.service.events.scheduled.ServicesStatusJobHandler.JOB_CONTEXT_CLUSTER_NAME; +import static org.openmetadata.service.events.scheduled.ServicesStatusJobHandler.JOB_CONTEXT_METER_REGISTRY; +import static org.openmetadata.service.events.scheduled.ServicesStatusJobHandler.JOB_CONTEXT_PIPELINE_SERVICE_CLIENT; import io.micrometer.core.instrument.Counter; import io.micrometer.prometheus.PrometheusMeterRegistry; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/events/scheduled/PipelineServiceStatusJobHandler.java b/openmetadata-service/src/main/java/org/openmetadata/service/events/scheduled/ServicesStatusJobHandler.java similarity index 55% rename from openmetadata-service/src/main/java/org/openmetadata/service/events/scheduled/PipelineServiceStatusJobHandler.java rename to openmetadata-service/src/main/java/org/openmetadata/service/events/scheduled/ServicesStatusJobHandler.java index 2e12d51764ff..c48b8df90eeb 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/events/scheduled/PipelineServiceStatusJobHandler.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/events/scheduled/ServicesStatusJobHandler.java @@ -5,7 +5,9 @@ import org.openmetadata.schema.api.configuration.pipelineServiceClient.PipelineServiceClientConfiguration; import org.openmetadata.sdk.PipelineServiceClientInterface; import org.openmetadata.service.clients.pipeline.PipelineServiceClientFactory; +import org.openmetadata.service.monitoring.EventMonitorConfiguration; import org.openmetadata.service.util.MicrometerBundleSingleton; +import org.quartz.Job; import org.quartz.JobBuilder; import org.quartz.JobDataMap; import org.quartz.JobDetail; @@ -17,11 +19,14 @@ import org.quartz.impl.StdSchedulerFactory; @Slf4j -public class PipelineServiceStatusJobHandler { - +public class ServicesStatusJobHandler { + public static final String HEALTHY_STATUS = "healthy"; + public static final String UNHEALTHY_STATUS = "unhealthy"; + public static final String DATABASE_SEARCH_STATUS_JOB = "databaseAndSearchServiceStatusJob"; public static final String PIPELINE_SERVICE_STATUS_JOB = "pipelineServiceStatusJob"; public static final String STATUS_GROUP = "status"; - public static final String STATUS_CRON_TRIGGER = "statusTrigger"; + public static final String PIPELINE_STATUS_CRON_TRIGGER = "pipelineStatusTrigger"; + public static final String DATABSE_SEARCH_STATUS_CRON_TRIGGER = "databaseAndSearchStatusTrigger"; public static final String JOB_CONTEXT_PIPELINE_SERVICE_CLIENT = "pipelineServiceClient"; public static final String JOB_CONTEXT_METER_REGISTRY = "meterRegistry"; public static final String JOB_CONTEXT_CLUSTER_NAME = "clusterName"; @@ -32,51 +37,55 @@ public class PipelineServiceStatusJobHandler { private final String clusterName; private final Integer healthCheckInterval; private final Scheduler scheduler = new StdSchedulerFactory().getScheduler(); + private final int servicesHealthCheckInterval; + private static ServicesStatusJobHandler instance; - private static PipelineServiceStatusJobHandler instance; - - private PipelineServiceStatusJobHandler( - PipelineServiceClientConfiguration config, String clusterName) throws SchedulerException { + private ServicesStatusJobHandler( + EventMonitorConfiguration monitorConfiguration, + PipelineServiceClientConfiguration config, + String clusterName) + throws SchedulerException { this.config = config; this.pipelineServiceClient = PipelineServiceClientFactory.createPipelineServiceClient(config); this.meterRegistry = MicrometerBundleSingleton.prometheusMeterRegistry; this.clusterName = clusterName; this.healthCheckInterval = config.getHealthCheckInterval(); + this.servicesHealthCheckInterval = monitorConfiguration.getServicesHealthCheckInterval(); this.scheduler.start(); } - public static PipelineServiceStatusJobHandler create( - PipelineServiceClientConfiguration config, String clusterName) { + public static ServicesStatusJobHandler create( + EventMonitorConfiguration eventMonitorConfiguration, + PipelineServiceClientConfiguration config, + String clusterName) { if (instance != null) return instance; try { - instance = new PipelineServiceStatusJobHandler(config, clusterName); + instance = new ServicesStatusJobHandler(eventMonitorConfiguration, config, clusterName); } catch (Exception ex) { LOG.error("Failed to initialize the Pipeline Service Status Handler"); } return instance; } - private JobDetail jobBuilder() { + private JobDetail jobBuilder(Class clazz, String jobName, String group) { JobDataMap dataMap = new JobDataMap(); dataMap.put(JOB_CONTEXT_PIPELINE_SERVICE_CLIENT, pipelineServiceClient); dataMap.put(JOB_CONTEXT_METER_REGISTRY, meterRegistry); dataMap.put(JOB_CONTEXT_CLUSTER_NAME, clusterName); JobBuilder jobBuilder = - JobBuilder.newJob(PipelineServiceStatusJob.class) - .withIdentity(PIPELINE_SERVICE_STATUS_JOB, STATUS_GROUP) - .usingJobData(dataMap); + JobBuilder.newJob(clazz).withIdentity(jobName, group).usingJobData(dataMap); return jobBuilder.build(); } - private Trigger getTrigger() { + private Trigger getTrigger(int checkInterval, String identity, String group) { return TriggerBuilder.newTrigger() - .withIdentity(STATUS_CRON_TRIGGER, STATUS_GROUP) + .withIdentity(identity, group) .withSchedule( SimpleScheduleBuilder.simpleSchedule() - .withIntervalInSeconds(healthCheckInterval) + .withIntervalInSeconds(checkInterval) .repeatForever()) .build(); } @@ -86,12 +95,27 @@ public void addPipelineServiceStatusJob() { // enabled if (config.getEnabled().equals(Boolean.TRUE)) { try { - JobDetail jobDetail = jobBuilder(); - Trigger trigger = getTrigger(); + JobDetail jobDetail = + jobBuilder(PipelineServiceStatusJob.class, PIPELINE_SERVICE_STATUS_JOB, STATUS_GROUP); + Trigger trigger = + getTrigger(healthCheckInterval, PIPELINE_STATUS_CRON_TRIGGER, STATUS_GROUP); scheduler.scheduleJob(jobDetail, trigger); } catch (Exception ex) { LOG.error("Failed in setting up job Scheduler for Pipeline Service Status", ex); } } } + + public void addDatabaseAndSearchStatusJobs() { + try { + JobDetail jobDetail = + jobBuilder( + DatabseAndSearchServiceStatusJob.class, DATABASE_SEARCH_STATUS_JOB, STATUS_GROUP); + Trigger trigger = + getTrigger(servicesHealthCheckInterval, DATABSE_SEARCH_STATUS_CRON_TRIGGER, STATUS_GROUP); + scheduler.scheduleJob(jobDetail, trigger); + } catch (Exception ex) { + LOG.error("Failed in setting up job Scheduler for Pipeline Service Status", ex); + } + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/exception/CatalogExceptionMessage.java b/openmetadata-service/src/main/java/org/openmetadata/service/exception/CatalogExceptionMessage.java index 7935488bd8fd..223f2a518961 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/exception/CatalogExceptionMessage.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/exception/CatalogExceptionMessage.java @@ -35,7 +35,7 @@ public final class CatalogExceptionMessage { public static final String PASSWORD_INVALID_FORMAT = "Password must be of minimum 8 characters, with one special, one Upper, one lower case character, and one Digit."; public static final String MAX_FAILED_LOGIN_ATTEMPT = - "Failed Login Attempts Exceeded. Please try after some time."; + "Failed Login Attempts Exceeded. Use Forgot Password or retry after some time."; public static final String INCORRECT_OLD_PASSWORD = "INCORRECT_OLD_PASSWORD"; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/exception/EventSubscriptionJobException.java b/openmetadata-service/src/main/java/org/openmetadata/service/exception/EventSubscriptionJobException.java index 6944de845734..caa0f6fa13d9 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/exception/EventSubscriptionJobException.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/exception/EventSubscriptionJobException.java @@ -5,6 +5,10 @@ public EventSubscriptionJobException(String message) { super(message); } + public EventSubscriptionJobException(String message, Throwable throwable) { + super(message, throwable); + } + public EventSubscriptionJobException(Throwable throwable) { super(throwable); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/formatter/decorators/FeedMessageDecorator.java b/openmetadata-service/src/main/java/org/openmetadata/service/formatter/decorators/FeedMessageDecorator.java index 411391e2772c..cedd3f6fae06 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/formatter/decorators/FeedMessageDecorator.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/formatter/decorators/FeedMessageDecorator.java @@ -37,7 +37,7 @@ public String getLineBreak() { @Override public String getAddMarker() { - return ""; + return ""; } @Override @@ -47,7 +47,7 @@ public String getAddMarkerClose() { @Override public String getRemoveMarker() { - return ""; + return ""; } @Override diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/formatter/decorators/MSTeamsMessageDecorator.java b/openmetadata-service/src/main/java/org/openmetadata/service/formatter/decorators/MSTeamsMessageDecorator.java index f1916c2be585..f16b031f3d4a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/formatter/decorators/MSTeamsMessageDecorator.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/formatter/decorators/MSTeamsMessageDecorator.java @@ -29,6 +29,7 @@ import org.openmetadata.schema.tests.TestCaseParameterValue; import org.openmetadata.schema.type.ChangeEvent; import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.type.FieldChange; import org.openmetadata.schema.type.TagLabel; import org.openmetadata.service.Entity; import org.openmetadata.service.apps.bundles.changeEvent.msteams.TeamsMessage; @@ -41,6 +42,7 @@ import org.openmetadata.service.exception.UnhandledServerException; public class MSTeamsMessageDecorator implements MessageDecorator { + private static final String TEST_CASE_RESULT = "testCaseResult"; @Override public String getBold() { @@ -115,11 +117,25 @@ private TeamsMessage createMessage( String entityType = event.getEntityType(); return switch (entityType) { - case Entity.TEST_CASE -> createDQMessage(publisherName, event, outgoingMessage); + case Entity.TEST_CASE -> createTestCaseMessage(publisherName, event, outgoingMessage); default -> createGeneralChangeEventMessage(publisherName, event, outgoingMessage); }; } + private TeamsMessage createTestCaseMessage( + String publisherName, ChangeEvent event, OutgoingMessage outgoingMessage) { + List fieldsAdded = event.getChangeDescription().getFieldsAdded(); + List fieldsUpdated = event.getChangeDescription().getFieldsUpdated(); + + boolean hasRelevantChange = + fieldsAdded.stream().anyMatch(field -> TEST_CASE_RESULT.equals(field.getName())) + || fieldsUpdated.stream().anyMatch(field -> TEST_CASE_RESULT.equals(field.getName())); + + return hasRelevantChange + ? createDQMessage(event, outgoingMessage) + : createGeneralChangeEventMessage(publisherName, event, outgoingMessage); + } + private TeamsMessage createGeneralChangeEventMessage( String publisherName, ChangeEvent event, OutgoingMessage outgoingMessage) { @@ -173,11 +189,9 @@ private TeamsMessage createGeneralChangeEventMessage( return TeamsMessage.builder().type("message").attachments(List.of(attachment)).build(); } - private TeamsMessage createDQMessage( - String publisherName, ChangeEvent event, OutgoingMessage outgoingMessage) { - + private TeamsMessage createDQMessage(ChangeEvent event, OutgoingMessage outgoingMessage) { Map, Object>> dqTemplateData = - buildDQTemplateData(publisherName, event, outgoingMessage); + MessageDecorator.buildDQTemplateData(event, outgoingMessage); TextBlock changeEventDetailsTextBlock = createHeader(); @@ -569,33 +583,6 @@ private Map, Object>> buildGeneralTemplate return builder.build(); } - // todo - complete buildDQTemplateData fn - private Map, Object>> buildDQTemplateData( - String publisherName, ChangeEvent event, OutgoingMessage outgoingMessage) { - - TemplateDataBuilder builder = new TemplateDataBuilder<>(); - - // Use DQ_Template_Section directly - builder - .add( - DQ_Template_Section.EVENT_DETAILS, - EventDetailsKeys.EVENT_TYPE, - event.getEventType().value()) - .add(DQ_Template_Section.EVENT_DETAILS, EventDetailsKeys.UPDATED_BY, event.getUserName()) - .add(DQ_Template_Section.EVENT_DETAILS, EventDetailsKeys.ENTITY_TYPE, event.getEntityType()) - .add( - DQ_Template_Section.EVENT_DETAILS, - EventDetailsKeys.ENTITY_FQN, - MessageDecorator.getFQNForChangeEventEntity(event)) - .add( - DQ_Template_Section.EVENT_DETAILS, - EventDetailsKeys.TIME, - new Date(event.getTimestamp()).toString()) - .add(DQ_Template_Section.EVENT_DETAILS, EventDetailsKeys.OUTGOING_MESSAGE, outgoingMessage); - - return builder.build(); - } - private TextBlock createHeader() { return TextBlock.builder() .type("TextBlock") diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/WorkflowEventConsumer.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/WorkflowEventConsumer.java index 7cbc974a4d25..0167230c8330 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/WorkflowEventConsumer.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/WorkflowEventConsumer.java @@ -27,11 +27,7 @@ public class WorkflowEventConsumer implements Destination { // TODO: Understand if we need to consider ENTITY_NO_CHANGE, ENTITY_FIELDS_CHANGED or // ENTITY_RESTORED. private static List validEventTypes = - List.of( - EventType.ENTITY_CREATED, - EventType.ENTITY_UPDATED, - EventType.ENTITY_SOFT_DELETED, - EventType.ENTITY_DELETED); + List.of(EventType.ENTITY_CREATED, EventType.ENTITY_UPDATED); public WorkflowEventConsumer( EventSubscription eventSubscription, SubscriptionDestination subscriptionDestination) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/WorkflowHandler.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/WorkflowHandler.java index ae3ecab0ac34..3e28d2d39885 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/WorkflowHandler.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/WorkflowHandler.java @@ -2,6 +2,7 @@ import static org.openmetadata.service.governance.workflows.elements.TriggerFactory.getTriggerWorkflowId; +import java.time.Duration; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -14,6 +15,7 @@ import org.flowable.engine.HistoryService; import org.flowable.engine.ProcessEngine; import org.flowable.engine.ProcessEngineConfiguration; +import org.flowable.engine.ProcessEngines; import org.flowable.engine.RepositoryService; import org.flowable.engine.RuntimeService; import org.flowable.engine.TaskService; @@ -23,44 +25,82 @@ import org.flowable.engine.runtime.Execution; import org.flowable.engine.runtime.ProcessInstance; import org.flowable.task.api.Task; +import org.openmetadata.schema.configuration.WorkflowSettings; +import org.openmetadata.service.Entity; import org.openmetadata.service.OpenMetadataApplicationConfig; import org.openmetadata.service.exception.UnhandledServerException; +import org.openmetadata.service.jdbi3.SystemRepository; import org.openmetadata.service.jdbi3.locator.ConnectionType; @Slf4j public class WorkflowHandler { - private final RepositoryService repositoryService; - private final RuntimeService runtimeService; - private final TaskService taskService; - private final HistoryService historyService; + private ProcessEngine processEngine; + private RepositoryService repositoryService; + private RuntimeService runtimeService; + private TaskService taskService; + private HistoryService historyService; private static WorkflowHandler instance; private static volatile boolean initialized = false; private WorkflowHandler(OpenMetadataApplicationConfig config) { ProcessEngineConfiguration processEngineConfiguration = new StandaloneProcessEngineConfiguration() - .setAsyncExecutorActivate(true) - .setAsyncExecutorCorePoolSize(50) - .setAsyncExecutorMaxPoolSize(100) - .setAsyncExecutorThreadPoolQueueSize(1000) - .setAsyncExecutorMaxAsyncJobsDuePerAcquisition(20) .setJdbcUrl(config.getDataSourceFactory().getUrl()) .setJdbcUsername(config.getDataSourceFactory().getUser()) .setJdbcPassword(config.getDataSourceFactory().getPassword()) .setJdbcDriver(config.getDataSourceFactory().getDriverClass()) .setDatabaseSchemaUpdate(ProcessEngineConfiguration.DB_SCHEMA_UPDATE_FALSE); - // Add Global Failure Listener - processEngineConfiguration.setEventListeners(List.of(new WorkflowFailureListener())); - if (ConnectionType.MYSQL.label.equals(config.getDataSourceFactory().getDriverClass())) { processEngineConfiguration.setDatabaseType(ProcessEngineConfiguration.DATABASE_TYPE_MYSQL); } else { processEngineConfiguration.setDatabaseType(ProcessEngineConfiguration.DATABASE_TYPE_POSTGRES); } + initializeNewProcessEngine(processEngineConfiguration); + } + + public void initializeNewProcessEngine( + ProcessEngineConfiguration currentProcessEngineConfiguration) { + ProcessEngines.destroy(); + SystemRepository systemRepository = Entity.getSystemRepository(); + WorkflowSettings workflowSettings = systemRepository.getWorkflowSettingsOrDefault(); + + StandaloneProcessEngineConfiguration processEngineConfiguration = + new StandaloneProcessEngineConfiguration(); + + // Setting Database Configuration + processEngineConfiguration + .setJdbcUrl(currentProcessEngineConfiguration.getJdbcUrl()) + .setJdbcUsername(currentProcessEngineConfiguration.getJdbcUsername()) + .setJdbcPassword(currentProcessEngineConfiguration.getJdbcPassword()) + .setJdbcDriver(currentProcessEngineConfiguration.getJdbcDriver()) + .setDatabaseType(currentProcessEngineConfiguration.getDatabaseType()) + .setDatabaseSchemaUpdate(ProcessEngineConfiguration.DB_SCHEMA_UPDATE_FALSE); + + // Setting Async Executor Configuration + processEngineConfiguration + .setAsyncExecutorActivate(true) + .setAsyncExecutorCorePoolSize(workflowSettings.getExecutorConfiguration().getCorePoolSize()) + .setAsyncExecutorMaxPoolSize(workflowSettings.getExecutorConfiguration().getMaxPoolSize()) + .setAsyncExecutorThreadPoolQueueSize( + workflowSettings.getExecutorConfiguration().getQueueSize()) + .setAsyncExecutorMaxAsyncJobsDuePerAcquisition( + workflowSettings.getExecutorConfiguration().getTasksDuePerAcquisition()); + + // Setting History CleanUp + processEngineConfiguration + .setEnableHistoryCleaning(true) + .setCleanInstancesEndedAfter( + Duration.ofDays( + workflowSettings.getHistoryCleanUpConfiguration().getCleanAfterNumberOfDays())); + + // Add Global Failure Listener + processEngineConfiguration.setEventListeners(List.of(new WorkflowFailureListener())); + ProcessEngine processEngine = processEngineConfiguration.buildProcessEngine(); + this.processEngine = processEngine; this.repositoryService = processEngine.getRepositoryService(); this.runtimeService = processEngine.getRuntimeService(); this.taskService = processEngine.getTaskService(); @@ -81,6 +121,14 @@ public static WorkflowHandler getInstance() { throw new UnhandledServerException("WorkflowHandler is not initialized."); } + public ProcessEngineConfiguration getProcessEngineConfiguration() { + if (processEngine != null) { + return processEngine.getProcessEngineConfiguration(); + } else { + return null; + } + } + public void deploy(Workflow workflow) { BpmnXMLConverter bpmnXMLConverter = new BpmnXMLConverter(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/impl/SetEntityCertificationImpl.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/impl/SetEntityCertificationImpl.java index 1de5a955412c..122714e4d99a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/impl/SetEntityCertificationImpl.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/impl/SetEntityCertificationImpl.java @@ -40,7 +40,7 @@ public void execute(DelegateExecution execution) { .orElse(null); String user = Optional.ofNullable((String) execution.getVariable(RESOLVED_BY_VARIABLE)) - .orElse(entity.getUpdatedBy()); + .orElse("governance-bot"); setStatus(entity, entityType, user, certification); } catch (Exception exc) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/impl/SetGlossaryTermStatusImpl.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/impl/SetGlossaryTermStatusImpl.java index 6d2d4060cfd4..cfe38e652771 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/impl/SetGlossaryTermStatusImpl.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/impl/SetGlossaryTermStatusImpl.java @@ -35,7 +35,7 @@ public void execute(DelegateExecution execution) { String status = (String) statusExpr.getValue(execution); String user = Optional.ofNullable((String) execution.getVariable(RESOLVED_BY_VARIABLE)) - .orElse(glossaryTerm.getUpdatedBy()); + .orElse("governance-bot"); setStatus(glossaryTerm, user, status); } catch (Exception exc) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/userTask/impl/CreateApprovalTaskImpl.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/userTask/impl/CreateApprovalTaskImpl.java index af14ad5c3841..c20411d94689 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/userTask/impl/CreateApprovalTaskImpl.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/userTask/impl/CreateApprovalTaskImpl.java @@ -26,7 +26,7 @@ import org.openmetadata.service.exception.EntityNotFoundException; import org.openmetadata.service.governance.workflows.WorkflowHandler; import org.openmetadata.service.jdbi3.FeedRepository; -import org.openmetadata.service.resources.feeds.FeedResource; +import org.openmetadata.service.resources.feeds.FeedMapper; import org.openmetadata.service.resources.feeds.MessageParser; import org.openmetadata.service.util.WebsocketNotificationHandler; @@ -91,7 +91,7 @@ private Thread createApprovalTask(GlossaryTerm entity, List ass } catch (EntityNotFoundException ex) { TaskDetails taskDetails = new TaskDetails() - .withAssignees(FeedResource.formatAssignees(assignees)) + .withAssignees(FeedMapper.formatAssignees(assignees)) .withType(TaskType.RequestApproval) .withStatus(TaskStatus.Open); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java index a5938ea9f4a2..cf3f32d8254b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java @@ -46,6 +46,7 @@ import org.jdbi.v3.sqlobject.CreateSqlObject; import org.jdbi.v3.sqlobject.config.RegisterRowMapper; import org.jdbi.v3.sqlobject.customizer.Bind; +import org.jdbi.v3.sqlobject.customizer.BindBean; import org.jdbi.v3.sqlobject.customizer.BindBeanList; import org.jdbi.v3.sqlobject.customizer.BindList; import org.jdbi.v3.sqlobject.customizer.BindMap; @@ -53,6 +54,7 @@ import org.jdbi.v3.sqlobject.statement.SqlQuery; import org.jdbi.v3.sqlobject.statement.SqlUpdate; import org.jdbi.v3.sqlobject.statement.UseRowMapper; +import org.jdbi.v3.sqlobject.transaction.Transaction; import org.openmetadata.api.configuration.UiThemePreference; import org.openmetadata.schema.TokenInterface; import org.openmetadata.schema.analytics.ReportData; @@ -67,6 +69,7 @@ import org.openmetadata.schema.auth.RefreshToken; import org.openmetadata.schema.auth.TokenType; import org.openmetadata.schema.configuration.AssetCertificationSettings; +import org.openmetadata.schema.configuration.WorkflowSettings; import org.openmetadata.schema.dataInsight.DataInsightChart; import org.openmetadata.schema.dataInsight.custom.DataInsightCustomChart; import org.openmetadata.schema.dataInsight.kpi.Kpi; @@ -140,6 +143,7 @@ import org.openmetadata.service.jdbi3.CollectionDAO.TagUsageDAO.TagLabelMapper; import org.openmetadata.service.jdbi3.CollectionDAO.UsageDAO.UsageDetailsMapper; import org.openmetadata.service.jdbi3.FeedRepository.FilterType; +import org.openmetadata.service.jdbi3.locator.ConnectionAwareSqlBatch; import org.openmetadata.service.jdbi3.locator.ConnectionAwareSqlQuery; import org.openmetadata.service.jdbi3.locator.ConnectionAwareSqlUpdate; import org.openmetadata.service.resources.events.subscription.TypedEvent; @@ -148,6 +152,7 @@ import org.openmetadata.service.util.EntityUtil; import org.openmetadata.service.util.FullyQualifiedName; import org.openmetadata.service.util.JsonUtils; +import org.openmetadata.service.util.jdbi.BindConcat; import org.openmetadata.service.util.jdbi.BindFQN; import org.openmetadata.service.util.jdbi.BindListFQN; import org.openmetadata.service.util.jdbi.BindUUID; @@ -724,11 +729,42 @@ void insert( @SqlQuery( "SELECT id, extension, json " + "FROM entity_extension " - + "WHERE id IN () AND extension LIKE CONCAT(:extensionPrefix, '.%') " + + "WHERE id IN () AND extension LIKE :extension " + "ORDER BY id, extension") @RegisterRowMapper(ExtensionRecordWithIdMapper.class) List getExtensionsBatch( - @BindList("ids") List ids, @Bind("extensionPrefix") String extensionPrefix); + @BindList("ids") List ids, + @BindConcat( + value = "extension", + parts = {":extensionPrefix", ".%"}) + String extensionPrefix); + + @SqlQuery( + "SELECT id, extension, json, jsonschema " + + "FROM entity_extension " + + "WHERE extension LIKE :extension " + + "ORDER BY id, extension") + @RegisterRowMapper(ExtensionWithIdAndSchemaRowMapper.class) + List getExtensionsByPrefixBatch( + @BindConcat( + value = "extension", + parts = {":extensionPrefix", "%"}) + String extensionPrefix); + + @Transaction + @ConnectionAwareSqlBatch( + value = + "INSERT INTO entity_extension (id, extension, json, jsonschema) " + + "VALUES (:id, :extension, :json, :jsonschema) " + + "ON DUPLICATE KEY UPDATE json = VALUES(json), jsonschema = VALUES(jsonschema)", + connectionType = MYSQL) + @ConnectionAwareSqlBatch( + value = + "INSERT INTO entity_extension (id, extension, json,jsonschema) VALUES (:id, :extension, :json::jsonb,:jsonschema) " + + "ON CONFLICT (id, extension) DO UPDATE SET json = EXCLUDED.json , jsonschema = EXCLUDED.jsonschema", + connectionType = POSTGRES) + void bulkUpsertExtensions( + @BindBean List extensionWithIdObjects); @RegisterRowMapper(ExtensionMapper.class) @SqlQuery( @@ -791,6 +827,28 @@ public ExtensionRecordWithId map(ResultSet rs, StatementContext ctx) throws SQLE } } + @Getter + @Setter + @Builder + class ExtensionWithIdAndSchemaObject { + private String id; + private String extension; + private String json; + private String jsonschema; + } + + class ExtensionWithIdAndSchemaRowMapper implements RowMapper { + @Override + public ExtensionWithIdAndSchemaObject map(ResultSet rs, StatementContext ctx) + throws SQLException { + String id = rs.getString("id"); + String extensionName = rs.getString("extension"); + String extensionJson = rs.getString("json"); + String jsonSchema = rs.getString("jsonschema"); + return new ExtensionWithIdAndSchemaObject(id, extensionName, extensionJson, jsonSchema); + } + } + @Getter @Builder class EntityRelationshipRecord { @@ -1179,10 +1237,7 @@ void deleteLineageBySource( + "AND json->>'source' = :source", connectionType = POSTGRES) void deleteLineageBySourcePipeline( - @BindUUID("toId") UUID toId, - @Bind("toEntity") String toEntity, - @Bind("source") String source, - @Bind("relation") int relation); + @BindUUID("toId") UUID toId, @Bind("source") String source, @Bind("relation") int relation); class FromRelationshipMapper implements RowMapper { @Override @@ -1327,6 +1382,11 @@ List listTasksOfUser( @Bind("limit") int limit, @Define("condition") String condition); + @SqlQuery( + "SELECT id FROM thread_entity WHERE type = 'Conversation' AND createdAt < :cutoffMillis LIMIT :batchSize") + List fetchConversationThreadIdsOlderThan( + @Bind("cutoffMillis") long cutoffMillis, @Bind("batchSize") int batchSize); + @ConnectionAwareSqlQuery( value = "SELECT count(id) FROM thread_entity " @@ -1413,16 +1473,25 @@ default List listThreadsByEntityLink( @SqlQuery( "SELECT json FROM thread_entity " + "AND hash_id in (SELECT fromFQNHash FROM field_relationship WHERE " - + "(:fqnPrefixHash IS NULL OR toFQNHash LIKE CONCAT(:fqnPrefixHash, '.%') OR toFQNHash=:fqnPrefixHash) AND fromType='THREAD' AND " - + "(:toType IS NULL OR toType LIKE CONCAT(:toType, '.%') OR toType=:toType) AND relation= :relation) " + + "(:fqnPrefixHash IS NULL OR toFQNHash LIKE :concatFqnPrefixHash OR toFQNHash=:fqnPrefixHash) AND fromType='THREAD' AND " + + "(:toType IS NULL OR toType LIKE :concatToType OR toType=:toType) AND relation= :relation) " + "AND (:userName IS NULL OR MD5(id) in (SELECT toFQNHash FROM field_relationship WHERE " + " ((fromType='user' AND fromFQNHash= :userName) OR" + " (fromType='team' AND fromFQNHash IN ())) AND toType='THREAD' AND relation= :filterRelation) )" + "ORDER BY createdAt DESC " + "LIMIT :limit") List listThreadsByEntityLink( - @BindFQN("fqnPrefixHash") String fqnPrefixHash, - @Bind("toType") String toType, + @BindConcat( + value = "concatFqnPrefixHash", + original = "fqnPrefixHash", + parts = {":fqnPrefixHash", ".%"}, + hash = true) + String fqnPrefixHash, + @BindConcat( + value = "concatToType", + original = "toType", + parts = {":toType", ".%"}) + String toType, @Bind("limit") int limit, @Bind("relation") int relation, @BindFQN("userName") String userName, @@ -1453,14 +1522,23 @@ default int listCountThreadsByEntityLink( @SqlQuery( "SELECT count(id) FROM thread_entity " + "AND hash_id in (SELECT fromFQNHash FROM field_relationship WHERE " - + "(:fqnPrefixHash IS NULL OR toFQNHash LIKE CONCAT(:fqnPrefixHash, '.%') OR toFQNHash=:fqnPrefixHash) AND fromType='THREAD' AND " - + "(:toType IS NULL OR toType LIKE CONCAT(:toType, '.%') OR toType=:toType) AND relation= :relation) " + + "(:fqnPrefixHash IS NULL OR toFQNHash LIKE :concatFqnPrefixHash OR toFQNHash=:fqnPrefixHash) AND fromType='THREAD' AND " + + "(:toType IS NULL OR toType LIKE :concatToType OR toType=:toType) AND relation= :relation) " + "AND (:userName IS NULL OR id in (SELECT toFQNHash FROM field_relationship WHERE " + " ((fromType='user' AND fromFQNHash= :userName) OR" + " (fromType='team' AND fromFQNHash IN ())) AND toType='THREAD' AND relation= :filterRelation) )") int listCountThreadsByEntityLink( - @BindFQN("fqnPrefixHash") String fqnPrefixHash, - @Bind("toType") String toType, + @BindConcat( + value = "concatFqnPrefixHash", + original = "fqnPrefixHash", + parts = {":fqnPrefixHash", ".%"}, + hash = true) + String fqnPrefixHash, + @BindConcat( + value = "concatToType", + original = "toType", + parts = {":toType", ".%"}) + String toType, @Bind("relation") int relation, @Bind("userName") String userName, @BindList("teamNames") List teamNames, @@ -1482,9 +1560,9 @@ int listCountThreadsByEntityLink( + " WHERE hash_id IN ( " + " SELECT fromFQNHash FROM field_relationship " + " WHERE " - + " (:fqnPrefixHash IS NULL OR toFQNHash LIKE CONCAT(:fqnPrefixHash, '.%') OR toFQNHash = :fqnPrefixHash) " + + " (:fqnPrefixHash IS NULL OR toFQNHash LIKE :concatFqnPrefixHash OR toFQNHash = :fqnPrefixHash) " + " AND fromType = 'THREAD' " - + " AND (:toType IS NULL OR toType LIKE CONCAT(:toType, '.%') OR toType = :toType) " + + " AND (:toType IS NULL OR toType LIKE :concatToType OR toType = :toType) " + " AND relation = 3 " + " ) " + " UNION " @@ -1496,8 +1574,17 @@ int listCountThreadsByEntityLink( @RegisterRowMapper(ThreadCountFieldMapper.class) List> listCountByEntityLink( @BindUUID("entityId") UUID entityId, - @BindFQN("fqnPrefixHash") String fqnPrefixHash, - @Bind("toType") String toType); + @BindConcat( + value = "concatFqnPrefixHash", + original = "fqnPrefixHash", + parts = {":fqnPrefixHash", ".%"}, + hash = true) + String fqnPrefixHash, + @BindConcat( + value = "concatToType", + original = "toType", + parts = {":toType", ".%"}) + String toType); @ConnectionAwareSqlQuery( value = @@ -1707,18 +1794,31 @@ int listCountThreadsByMentions( @SqlQuery( "SELECT json FROM thread_entity " + "AND MD5(id) in (SELECT fromFQNHash FROM field_relationship WHERE " - + "(:fqnPrefixHash IS NULL OR toFQNHash LIKE CONCAT(:fqnPrefixHash, '.%') OR toFQNHash=:fqnPrefixHash) AND fromType='THREAD' AND " - + "((:toType1 IS NULL OR toType LIKE CONCAT(:toType1, '.%') OR toType=:toType1) OR " - + "(:toType2 IS NULL OR toType LIKE CONCAT(:toType2, '.%') OR toType=:toType2)) AND relation= :relation)" + + "(:fqnPrefixHash IS NULL OR toFQNHash LIKE :concatFqnPrefixHash OR toFQNHash=:fqnPrefixHash) AND fromType='THREAD' AND " + + "((:toType1 IS NULL OR toType LIKE :concatToType1 OR toType=:toType1) OR " + + "(:toType2 IS NULL OR toType LIKE :concatToType2 OR toType=:toType2)) AND relation= :relation)" + "AND (:userName IS NULL OR MD5(id) in (SELECT toFQNHash FROM field_relationship WHERE " + " ((fromType='user' AND fromFQNHash= :userName) OR" + " (fromType='team' AND fromFQNHash IN ())) AND toType='THREAD' AND relation= :filterRelation) )" + "ORDER BY createdAt DESC " + "LIMIT :limit") List listThreadsByGlossaryAndTerms( - @BindFQN("fqnPrefixHash") String fqnPrefixHash, - @Bind("toType1") String toType1, - @Bind("toType2") String toType2, + @BindConcat( + value = "concatFqnPrefixHash", + original = "fqnPrefixHash", + parts = {":fqnPrefixHash", ".%"}, + hash = true) + String fqnPrefixHash, + @BindConcat( + value = "concatToType1", + original = "toType1", + parts = {":toType1", ".%"}) + String toType1, + @BindConcat( + value = "concatToType2", + original = "toType2", + parts = {":toType2", ".%"}) + String toType2, @Bind("limit") int limit, @Bind("relation") int relation, @BindFQN("userName") String userName, @@ -1749,9 +1849,9 @@ default List> listCountThreadsByGlossaryAndTerms( + " WHERE te.hash_id IN ( " + " SELECT fr.fromFQNHash " + " FROM field_relationship fr " - + " WHERE (:fqnPrefixHash IS NULL OR fr.toFQNHash LIKE CONCAT(:fqnPrefixHash, '.%') OR fr.toFQNHash = :fqnPrefixHash) " + + " WHERE (:fqnPrefixHash IS NULL OR fr.toFQNHash LIKE :concatFqnPrefixHash OR fr.toFQNHash = :fqnPrefixHash) " + " AND fr.fromType = 'THREAD' " - + " AND (:toType1 IS NULL OR fr.toType LIKE CONCAT(:toType1, '.%') OR fr.toType = :toType1) " + + " AND (:toType1 IS NULL OR fr.toType LIKE :concatToType1 OR fr.toType = :toType1) " + " AND fr.relation = 3 " + " ) " + " UNION " @@ -1762,9 +1862,9 @@ default List> listCountThreadsByGlossaryAndTerms( + " SELECT fr.fromFQNHash " + " FROM field_relationship fr " + " JOIN thread_entity te2 ON te2.hash_id = fr.fromFQNHash WHERE fr.fromFQNHash = te.hash_id AND te2.type = 'Task' " - + " AND (:fqnPrefixHash IS NULL OR fr.toFQNHash LIKE CONCAT(:fqnPrefixHash, '.%') OR fr.toFQNHash = :fqnPrefixHash) " + + " AND (:fqnPrefixHash IS NULL OR fr.toFQNHash LIKE :concatFqnPrefixHash OR fr.toFQNHash = :fqnPrefixHash) " + " AND fr.fromType = 'THREAD' " - + " AND (:toType2 IS NULL OR fr.toType LIKE CONCAT(:toType2, '.%') OR fr.toType = :toType2) " + + " AND (:toType2 IS NULL OR fr.toType LIKE :concatToType2 OR fr.toType = :toType2) " + " AND fr.relation = 3 " + " ) " + ") AS combined_results WHERE combined_results.type is not NULL " @@ -1772,9 +1872,22 @@ default List> listCountThreadsByGlossaryAndTerms( @RegisterRowMapper(ThreadCountFieldMapper.class) List> listCountThreadsByGlossaryAndTerms( @BindUUID("entityId") UUID entityId, - @BindFQN("fqnPrefixHash") String fqnPrefixHash, - @Bind("toType1") String toType1, - @Bind("toType2") String toType2); + @BindConcat( + value = "concatFqnPrefixHash", + original = "fqnPrefixHash", + parts = {":fqnPrefixHash", ".%"}, + hash = true) + String fqnPrefixHash, + @BindConcat( + value = "concatToType1", + original = "toType1", + parts = {":toType1", ".%"}) + String toType1, + @BindConcat( + value = "concatToType2", + original = "toType2", + parts = {":toType2", ".%"}) + String toType2); @SqlQuery("select id from thread_entity where entityId = :entityId") List findByEntityId(@Bind("entityId") String entityId); @@ -1879,11 +1992,15 @@ List> findFrom( @SqlQuery( "SELECT fromFQN, toFQN, json FROM field_relationship WHERE " - + "fromFQNHash LIKE CONCAT(:fqnPrefixHash, '%') AND fromType = :fromType AND toType = :toType " + + "fromFQNHash LIKE :concatFqnPrefixHash AND fromType = :fromType AND toType = :toType " + "AND relation = :relation") @RegisterRowMapper(ToFieldMapper.class) List> listToByPrefix( - @BindFQN("fqnPrefixHash") String fqnPrefixHash, + @BindConcat( + value = "concatFqnPrefixHash", + parts = {":fqnPrefixHash", "%"}, + hash = true) + String fqnPrefixHash, @Bind("fromType") String fromType, @Bind("toType") String toType, @Bind("relation") int relation); @@ -1909,13 +2026,17 @@ List> listBidirectional( @SqlQuery( "SELECT fromFQN, toFQN, json FROM field_relationship WHERE " - + "fromFQNHash LIKE CONCAT(:fqnPrefixHash, '%') AND fromType = :type AND toType = :otherType AND relation = :relation " + + "fromFQNHash LIKE :concatFqnPrefixHash AND fromType = :type AND toType = :otherType AND relation = :relation " + "UNION " + "SELECT toFQN, fromFQN, json FROM field_relationship WHERE " - + "toFQNHash LIKE CONCAT(:fqnPrefixHash, '%') AND toType = :type AND fromType = :otherType AND relation = :relation") + + "toFQNHash LIKE :concatFqnPrefixHash AND toType = :type AND fromType = :otherType AND relation = :relation") @RegisterRowMapper(ToFieldMapper.class) List> listBidirectionalByPrefix( - @BindFQN("fqnPrefixHash") String fqnPrefixHash, + @BindConcat( + value = "concatFqnPrefixHash", + parts = {":fqnPrefixHash", "%"}, + hash = true) + String fqnPrefixHash, @Bind("type") String type, @Bind("otherType") String otherType, @Bind("relation") int relation); @@ -2180,6 +2301,52 @@ void upsertSuccessfulChangeEvent( void deleteOldRecords( @Bind("eventSubscriptionId") String eventSubscriptionId, @Bind("limit") long limit); + @ConnectionAwareSqlUpdate( + value = + "DELETE FROM successful_sent_change_events " + + "WHERE timestamp < :cutoff ORDER BY timestamp LIMIT :limit", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "DELETE FROM successful_sent_change_events " + + "WHERE ctid IN ( " + + " SELECT ctid FROM successful_sent_change_events " + + " WHERE timestamp < :cutoff ORDER BY timestamp LIMIT :limit " + + ")", + connectionType = POSTGRES) + int deleteSuccessfulSentChangeEventsInBatches( + @Bind("cutoff") long cutoff, @Bind("limit") int limit); + + @ConnectionAwareSqlUpdate( + value = + "DELETE FROM change_event " + + "WHERE eventTime < :cutoff ORDER BY eventTime LIMIT :limit", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "DELETE FROM change_event " + + "WHERE ctid IN ( " + + " SELECT ctid FROM change_event " + + " WHERE eventTime < :cutoff ORDER BY eventTime LIMIT :limit " + + ")", + connectionType = POSTGRES) + int deleteChangeEventsInBatches(@Bind("cutoff") long cutoff, @Bind("limit") int limit); + + @ConnectionAwareSqlUpdate( + value = + "DELETE FROM consumers_dlq " + + "WHERE timestamp < :cutoff ORDER BY timestamp LIMIT :limit", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "DELETE FROM consumers_dlq " + + "WHERE ctid IN ( " + + " SELECT ctid FROM consumers_dlq " + + " WHERE timestamp < :cutoff ORDER BY timestamp LIMIT :limit " + + ")", + connectionType = POSTGRES) + int deleteConsumersDlqInBatches(@Bind("cutoff") long cutoff, @Bind("limit") int limit); + @SqlQuery( "SELECT json FROM successful_sent_change_events WHERE event_subscription_id = :eventSubscriptionId ORDER BY timestamp DESC LIMIT :limit OFFSET :paginationOffset") List getSuccessfulChangeEventBySubscriptionId( @@ -2361,12 +2528,12 @@ default int listCount(ListFilter filter) { String directChildrenOf = filter.getQueryParam("directChildrenOf"); if (!nullOrEmpty(directChildrenOf)) { - filter.queryParams.put( - "directChildrenOfHash", FullyQualifiedName.buildHash(directChildrenOf)); - condition = - String.format( - " %s AND fqnHash = CONCAT(:directChildrenOfHash, '.', MD5(CASE WHEN name LIKE '%%.%%' THEN CONCAT('\"', name, '\"') ELSE name END)) ", - condition); + String parentFqnHash = FullyQualifiedName.buildHash(directChildrenOf); + filter.queryParams.put("fqnHashSingleLevel", parentFqnHash + ".%"); + filter.queryParams.put("fqnHashNestedLevel", parentFqnHash + ".%.%"); + + condition += + " AND fqnHash LIKE :fqnHashSingleLevel AND fqnHash NOT LIKE :fqnHashNestedLevel"; } return listCount(getTableName(), getNameHashColumn(), filter.getQueryParams(), condition); @@ -2379,12 +2546,12 @@ default List listBefore( String directChildrenOf = filter.getQueryParam("directChildrenOf"); if (!nullOrEmpty(directChildrenOf)) { - filter.queryParams.put( - "directChildrenOfHash", FullyQualifiedName.buildHash(directChildrenOf)); - condition = - String.format( - " %s AND fqnHash = CONCAT(:directChildrenOfHash, '.', MD5(CASE WHEN name LIKE '%%.%%' THEN CONCAT('\"', name, '\"') ELSE name END)) ", - condition); + String parentFqnHash = FullyQualifiedName.buildHash(directChildrenOf); + filter.queryParams.put("fqnHashSingleLevel", parentFqnHash + ".%"); + filter.queryParams.put("fqnHashNestedLevel", parentFqnHash + ".%.%"); + + condition += + " AND fqnHash LIKE :fqnHashSingleLevel AND fqnHash NOT LIKE :fqnHashNestedLevel"; } return listBefore( @@ -2397,19 +2564,24 @@ default List listAfter(ListFilter filter, int limit, String afterName, S String directChildrenOf = filter.getQueryParam("directChildrenOf"); if (!nullOrEmpty(directChildrenOf)) { - filter.queryParams.put( - "directChildrenOfHash", FullyQualifiedName.buildHash(directChildrenOf)); - condition = - String.format( - " %s AND fqnHash = CONCAT(:directChildrenOfHash, '.', MD5(CASE WHEN name LIKE '%%.%%' THEN CONCAT('\"', name, '\"') ELSE name END)) ", - condition); + String parentFqnHash = FullyQualifiedName.buildHash(directChildrenOf); + filter.queryParams.put("fqnHashSingleLevel", parentFqnHash + ".%"); + filter.queryParams.put("fqnHashNestedLevel", parentFqnHash + ".%.%"); + + condition += + " AND fqnHash LIKE :fqnHashSingleLevel AND fqnHash NOT LIKE :fqnHashNestedLevel"; } return listAfter( getTableName(), filter.getQueryParams(), condition, limit, afterName, afterId); } - @SqlQuery("select json FROM glossary_term_entity where fqnhash LIKE CONCAT(:fqnhash, '.%')") - List getNestedTerms(@BindFQN("fqnhash") String fqnhash); + @SqlQuery("select json FROM glossary_term_entity where fqnhash LIKE :concatFqnhash ") + List getNestedTerms( + @BindConcat( + value = "concatFqnhash", + parts = {":fqnhash", ".%"}, + hash = true) + String fqnhash); } interface IngestionPipelineDAO extends EntityDAO { @@ -3040,8 +3212,13 @@ default List listAfter(ListFilter filter, int limit, String afterName, S afterId); } - @SqlQuery("select json FROM tag where fqnhash LIKE CONCAT(:fqnhash, '.%')") - List getTagsStartingWithPrefix(@BindFQN("fqnhash") String fqnhash); + @SqlQuery("select json FROM tag where fqnhash LIKE :concatFqnhash") + List getTagsStartingWithPrefix( + @BindConcat( + value = "concatFqnhash", + parts = {":fqnhash", ".%"}, + hash = true) + String fqnhash); } @RegisterRowMapper(TagLabelMapper.class) @@ -3070,10 +3247,11 @@ default List getTags(String targetFQN) { default Map> getTagsByPrefix( String targetFQNPrefix, String postfix, boolean requiresFqnHash) { - String fqnHash = + String targetFQNPrefixHash = requiresFqnHash ? FullyQualifiedName.buildHash(targetFQNPrefix) : targetFQNPrefix; Map> resultSet = new LinkedHashMap<>(); - List> tags = getTagsInternalByPrefix(fqnHash, postfix); + List> tags = + getTagsInternalByPrefix(new String[] {targetFQNPrefixHash, postfix}); tags.forEach( pair -> { String targetHash = pair.getLeft(); @@ -3119,7 +3297,7 @@ List getTagsInternalBatch( + " ON ta.fqnHash = tu.tagFQNHash " + " WHERE tu.source = 0 " + ") AS combined_data " - + "WHERE combined_data.targetFQNHash LIKE CONCAT(:targetFQNHashPrefix, :postfix)", + + "WHERE combined_data.targetFQNHash LIKE :targetFQNHash", connectionType = MYSQL) @ConnectionAwareSqlQuery( value = @@ -3135,11 +3313,14 @@ List getTagsInternalBatch( + " JOIN tag_usage AS tu ON ta.fqnHash = tu.tagFQNHash " + " WHERE tu.source = 0 " + ") AS combined_data " - + "WHERE combined_data.targetFQNHash LIKE CONCAT(:targetFQNHashPrefix, :postfix)", + + "WHERE combined_data.targetFQNHash LIKE :targetFQNHash", connectionType = POSTGRES) @RegisterRowMapper(TagLabelRowMapperWithTargetFqnHash.class) List> getTagsInternalByPrefix( - @Bind("targetFQNHashPrefix") String targetFQNHashPrefix, @Bind("postfix") String postfix); + @BindConcat( + value = "targetFQNHash", + parts = {":targetFQNHashPrefix", ":postfix"}) + String... targetFQNHash); @SqlQuery("SELECT * FROM tag_usage") @Deprecated(since = "Release 1.1") @@ -3148,17 +3329,29 @@ List> getTagsInternalByPrefix( @SqlQuery( "SELECT COUNT(*) FROM tag_usage " - + "WHERE (tagFQNHash LIKE CONCAT(:tagFqnHash, '.%') OR tagFQNHash = :tagFqnHash) " + + "WHERE (tagFQNHash LIKE :concatTagFQNHash OR tagFQNHash = :tagFqnHash) " + "AND source = :source") - int getTagCount(@Bind("source") int source, @BindFQN("tagFqnHash") String tagFqnHash); + int getTagCount( + @Bind("source") int source, + @BindConcat( + value = "concatTagFQNHash", + original = "tagFqnHash", + parts = {":tagFqnHash", ".%"}, + hash = true) + String tagFqnHash); @SqlUpdate("DELETE FROM tag_usage where targetFQNHash = :targetFQNHash") void deleteTagsByTarget(@BindFQN("targetFQNHash") String targetFQNHash); @SqlUpdate( - "DELETE FROM tag_usage where tagFQNHash = :tagFqnHash AND targetFQNHash LIKE CONCAT(:targetFQNHash, '%')") + "DELETE FROM tag_usage where tagFQNHash = :tagFqnHash AND targetFQNHash LIKE :targetFQNHash") void deleteTagsByTagAndTargetEntity( - @BindFQN("tagFqnHash") String tagFqnHash, @BindFQN("targetFQNHash") String targetFQNHash); + @BindFQN("tagFqnHash") String tagFqnHash, + @BindConcat( + value = "targetFQNHash", + parts = {":targetFQNHashPrefix", "%"}, + hash = true) + String targetFQNHashPrefix); @SqlUpdate("DELETE FROM tag_usage where tagFQNHash = :tagFQNHash AND source = :source") void deleteTagLabels(@Bind("source") int source, @BindFQN("tagFQNHash") String tagFQNHash); @@ -3167,8 +3360,14 @@ void deleteTagsByTagAndTargetEntity( void deleteTagLabelsByFqn(@BindFQN("tagFQNHash") String tagFQNHash); @SqlUpdate( - "DELETE FROM tag_usage where targetFQNHash = :targetFQNHash OR targetFQNHash LIKE CONCAT(:targetFQNHash, '.%')") - void deleteTagLabelsByTargetPrefix(@BindFQN("targetFQNHash") String targetFQNHash); + "DELETE FROM tag_usage where targetFQNHash = :targetFQNHash OR targetFQNHash LIKE :concatTargetFQNHash") + void deleteTagLabelsByTargetPrefix( + @BindConcat( + value = "concatTargetFQNHash", + original = "targetFQNHash", + parts = {":targetFQNHashPrefix", ".%"}, + hash = true) + String targetFQNHashPrefix); @Deprecated(since = "Release 1.1") @ConnectionAwareSqlUpdate( @@ -3250,9 +3449,14 @@ void renameInternal( @RegisterRowMapper(TagLabelMapper.class) List getTargetFQNHashForTag(@BindFQN("tagFQNHash") String tagFQNHash); - @SqlQuery("select targetFQNHash FROM tag_usage where tagFQNHash LIKE CONCAT(:tagFQNHash, '.%')") + @SqlQuery("select targetFQNHash FROM tag_usage where tagFQNHash LIKE :tagFQNHash") @RegisterRowMapper(TagLabelMapper.class) - List getTargetFQNHashForTagPrefix(@BindFQN("tagFQNHash") String tagFQNHash); + List getTargetFQNHashForTagPrefix( + @BindConcat( + value = "tagFQNHash", + parts = {":tagFQNHashPrefix", ".%"}, + hash = true) + String tagFQNHashPrefix); class TagLabelMapper implements RowMapper { @Override @@ -4151,7 +4355,7 @@ List listWithoutEntityFilter( @Bind("eventType") String eventType, @Bind("timestamp") long timestamp); @SqlQuery( - "SELECT json FROM change_event ce where ce.offset > :offset ORDER BY ce.eventTime ASC LIMIT :limit") + "SELECT json FROM change_event ce WHERE ce.offset > :offset ORDER BY ce.offset ASC LIMIT :limit") List list(@Bind("limit") long limit, @Bind("offset") long offset); @ConnectionAwareSqlQuery(value = "SELECT MAX(offset) FROM change_event", connectionType = MYSQL) @@ -4159,6 +4363,9 @@ List listWithoutEntityFilter( value = "SELECT MAX(\"offset\") FROM change_event", connectionType = POSTGRES) long getLatestOffset(); + + @SqlQuery("SELECT count(*) FROM change_event") + long listCount(); } class FailedEventResponseMapper implements RowMapper { @@ -4626,11 +4833,11 @@ default String getTimeSeriesTableName() { interface AppsDataStore { @ConnectionAwareSqlUpdate( value = - "INSERT INTO apps_data_store(identifier, type, json) VALUES (:identifier, :type, :json)", + "INSERT INTO apps_data_store(identifier, type, json) VALUES (:identifier, :type, :json) ON DUPLICATE KEY UPDATE json = VALUES(json)", connectionType = MYSQL) @ConnectionAwareSqlUpdate( value = - "INSERT INTO apps_data_store(identifier, type, json) VALUES (:identifier, :type, :json :: jsonb)", + "INSERT INTO apps_data_store(identifier, type, json) VALUES (:identifier, :type, :json :: jsonb) ON CONFLICT (identifier, type) DO UPDATE SET json = EXCLUDED.json", connectionType = POSTGRES) void insert( @Bind("identifier") String identifier, @@ -5156,6 +5363,7 @@ public static Settings getSettings(SettingsType configType, String json) { case SEARCH_SETTINGS -> JsonUtils.readValue(json, SearchSettings.class); case ASSET_CERTIFICATION_SETTINGS -> JsonUtils.readValue( json, AssetCertificationSettings.class); + case WORKFLOW_SETTINGS -> JsonUtils.readValue(json, WorkflowSettings.class); case LINEAGE_SETTINGS -> JsonUtils.readValue(json, LineageSettings.class); default -> throw new IllegalArgumentException("Invalid Settings Type " + configType); }; @@ -5422,8 +5630,8 @@ default List listBefore( if (fqnPrefix != null) { String fqnPrefixHash = FullyQualifiedName.buildHash(fqnPrefix); filter.queryParams.put("fqnPrefixHash", fqnPrefixHash); - String fqnCond = - " AND (fqnHash LIKE CONCAT(:fqnPrefixHash, '.%') OR fqnHash=:fqnPrefixHash)"; + filter.queryParams.put("concatFqnPrefixHash", fqnPrefixHash + ".%"); + String fqnCond = " AND (fqnHash LIKE :concatFqnPrefixHash OR fqnHash=:fqnPrefixHash)"; mysqlCondition.append(fqnCond); psqlCondition.append(fqnCond); } @@ -5461,8 +5669,8 @@ default List listAfter(ListFilter filter, int limit, String afterName, S if (fqnPrefix != null) { String fqnPrefixHash = FullyQualifiedName.buildHash(fqnPrefix); filter.queryParams.put("fqnPrefixHash", fqnPrefixHash); - String fqnCond = - " AND (fqnHash LIKE CONCAT(:fqnPrefixHash, '.%') OR fqnHash=:fqnPrefixHash)"; + filter.queryParams.put("concatFqnPrefixHash", fqnPrefixHash + ".%"); + String fqnCond = " AND (fqnHash LIKE :concatFqnPrefixHash OR fqnHash=:fqnPrefixHash)"; mysqlCondition.append(fqnCond); psqlCondition.append(fqnCond); } @@ -5499,8 +5707,8 @@ default int listCount(ListFilter filter) { if (fqnPrefix != null) { String fqnPrefixHash = FullyQualifiedName.buildHash(fqnPrefix); filter.queryParams.put("fqnPrefixHash", fqnPrefixHash); - String fqnCond = - " AND (fqnHash LIKE CONCAT(:fqnPrefixHash, '.%') OR fqnHash=:fqnPrefixHash)"; + filter.queryParams.put("concatFqnPrefixHash", fqnPrefixHash + ".%"); + String fqnCond = " AND (fqnHash LIKE :concatFqnPrefixHash OR fqnHash=:fqnPrefixHash)"; mysqlCondition.append(fqnCond); psqlCondition.append(fqnCond); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataInsightSystemChartRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataInsightSystemChartRepository.java index 5d02304282bc..046ffa346920 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataInsightSystemChartRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataInsightSystemChartRepository.java @@ -5,6 +5,7 @@ import java.io.IOException; import java.util.HashMap; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.Set; import org.openmetadata.schema.dataInsight.custom.DataInsightCustomChart; @@ -87,7 +88,13 @@ public DataInsightCustomChartResultList getPreviewData( DataInsightCustomChart chart, long startTimestamp, long endTimestamp, String filter) throws IOException { if (chart.getChartDetails() != null && filter != null) { - ((LinkedHashMap) chart.getChartDetails()).put("filter", filter); + HashMap chartDetails = (LinkedHashMap) chart.getChartDetails(); + if (chartDetails.get("metrics") != null) { + for (LinkedHashMap metrics : + (List>) chartDetails.get("metrics")) { + metrics.put("filter", filter); + } + } } return getPreviewData(chart, startTimestamp, endTimestamp); } @@ -110,7 +117,13 @@ public Map listChartData( if (chart != null) { if (chart.getChartDetails() != null && filter != null) { - ((LinkedHashMap) chart.getChartDetails()).put("filter", filter); + HashMap chartDetails = (LinkedHashMap) chart.getChartDetails(); + if (chartDetails.get("metrics") != null) { + for (LinkedHashMap metrics : + (List>) chartDetails.get("metrics")) { + metrics.put("filter", filter); + } + } } DataInsightCustomChartResultList data = searchClient.buildDIChart(chart, startTimestamp, endTimestamp); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataProductRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataProductRepository.java index ff9f463d98a1..5af70aa6e4e5 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataProductRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataProductRepository.java @@ -14,7 +14,9 @@ package org.openmetadata.service.jdbi3; import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; +import static org.openmetadata.schema.type.Include.ALL; import static org.openmetadata.service.Entity.DATA_PRODUCT; +import static org.openmetadata.service.Entity.DOMAIN; import static org.openmetadata.service.Entity.FIELD_ASSETS; import static org.openmetadata.service.util.EntityUtil.entityReferenceMatch; @@ -26,6 +28,7 @@ import org.jdbi.v3.sqlobject.transaction.Transaction; import org.openmetadata.schema.EntityInterface; import org.openmetadata.schema.entity.domains.DataProduct; +import org.openmetadata.schema.entity.domains.Domain; import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.Include; import org.openmetadata.schema.type.Relationship; @@ -93,6 +96,22 @@ public void storeRelationships(DataProduct entity) { } } + public final EntityReference getDomain(Domain domain) { + return getFromEntityRef(domain.getId(), Relationship.CONTAINS, DOMAIN, false); + } + + @Override + public void setInheritedFields(DataProduct dataProduct, Fields fields) { + // If dataProduct does not have owners and experts, inherit them from its domain + EntityReference parentRef = + dataProduct.getDomain() != null ? dataProduct.getDomain() : getDomain(dataProduct); + if (parentRef != null) { + Domain parent = Entity.getEntity(DOMAIN, parentRef.getId(), "owners,experts", ALL); + inheritOwners(dataProduct, fields, parent); + inheritExperts(dataProduct, fields, parent); + } + } + @Override public EntityUpdater getUpdater(DataProduct original, DataProduct updated, Operation operation) { return new DataProductUpdater(original, updated, operation); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DomainRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DomainRepository.java index 40943f8bc185..5a6f6dcca48e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DomainRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DomainRepository.java @@ -18,10 +18,15 @@ import static org.openmetadata.service.Entity.DOMAIN; import static org.openmetadata.service.Entity.FIELD_ASSETS; +import java.util.ArrayList; import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.jdbi.v3.sqlobject.transaction.Transaction; import org.openmetadata.schema.EntityInterface; +import org.openmetadata.schema.entity.data.EntityHierarchy; import org.openmetadata.schema.entity.domains.Domain; import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.Include; @@ -30,8 +35,11 @@ import org.openmetadata.schema.type.api.BulkOperationResult; import org.openmetadata.service.Entity; import org.openmetadata.service.resources.domains.DomainResource; +import org.openmetadata.service.util.EntityUtil; import org.openmetadata.service.util.EntityUtil.Fields; import org.openmetadata.service.util.FullyQualifiedName; +import org.openmetadata.service.util.JsonUtils; +import org.openmetadata.service.util.ResultList; @Slf4j public class DomainRepository extends EntityRepository { @@ -140,6 +148,47 @@ public EntityInterface getParentEntity(Domain entity, String fields) { : null; } + public List buildHierarchy(String fieldsParam, int limit) { + fieldsParam = EntityUtil.addField(fieldsParam, Entity.FIELD_PARENT); + Fields fields = getFields(fieldsParam); + ResultList resultList = listAfter(null, fields, new ListFilter(null), limit, null); + List domains = resultList.getData(); + + /* + Maintaining hierarchy in terms of EntityHierarchy to get all other fields of Domain like style, + which would have been restricted if built using hierarchy of Domain, as Domain.getChildren() returns List + and EntityReference does not support additional properties + */ + List rootDomains = new ArrayList<>(); + + Map entityHierarchyMap = + domains.stream() + .collect( + Collectors.toMap( + Domain::getId, + domain -> { + EntityHierarchy entityHierarchy = + JsonUtils.readValue(JsonUtils.pojoToJson(domain), EntityHierarchy.class); + entityHierarchy.setChildren(new ArrayList<>()); + return entityHierarchy; + })); + + for (Domain domain : domains) { + EntityHierarchy entityHierarchy = entityHierarchyMap.get(domain.getId()); + + if (domain.getParent() != null) { + EntityHierarchy parentHierarchy = entityHierarchyMap.get(domain.getParent().getId()); + if (parentHierarchy != null) { + parentHierarchy.getChildren().add(entityHierarchy); + } + } else { + rootDomains.add(entityHierarchy); + } + } + + return rootDomains; + } + public class DomainUpdater extends EntityUpdater { public DomainUpdater(Domain original, Domain updated, Operation operation) { super(original, updated, operation); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityDAO.java index 67d5a62078af..ba0e95a91cb2 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityDAO.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityDAO.java @@ -322,6 +322,14 @@ List listAll( @Bind("startHash") String startHash, @Bind("endHash") String endHash); + @SqlQuery("SELECT json FROM AND BETWEEN :startHash AND :endHash ") + List listAll( + @Define("table") String table, + @Define("cond") String cond, + @Define("nameHashColumn") String nameHashColumn, + @Bind("startHash") String startHash, + @Bind("endHash") String endHash); + @SqlQuery("SELECT json FROM
LIMIT :limit OFFSET :offset") List listAfterWithOffset( @Define("table") String table, @Bind("limit") int limit, @Bind("offset") int offset); @@ -470,6 +478,11 @@ default List listAll(String startHash, String endHash) { return listAll(getTableName(), getNameHashColumn(), startHash, endHash); } + default List listAll(String startHash, String endHash, ListFilter filter) { + // Quoted name is stored in fullyQualifiedName column and not in the name column + return listAll(getTableName(), filter.getCondition(), getNameHashColumn(), startHash, endHash); + } + default List listAfterWithOffset(int limit, int offset) { // No ordering return listAfterWithOffset(getTableName(), limit, offset); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java index 221da3625c89..5ff760745c29 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java @@ -107,6 +107,7 @@ import java.util.function.BiPredicate; import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.StreamSupport; import javax.json.JsonPatch; import javax.validation.ConstraintViolationException; import javax.validation.constraints.NotNull; @@ -131,6 +132,7 @@ import org.openmetadata.schema.entity.feed.Suggestion; import org.openmetadata.schema.entity.teams.Team; import org.openmetadata.schema.entity.teams.User; +import org.openmetadata.schema.entity.type.Style; import org.openmetadata.schema.system.EntityError; import org.openmetadata.schema.type.ApiStatus; import org.openmetadata.schema.type.AssetCertification; @@ -168,6 +170,7 @@ import org.openmetadata.service.jdbi3.CollectionDAO.ExtensionRecord; import org.openmetadata.service.jdbi3.FeedRepository.TaskWorkflow; import org.openmetadata.service.jdbi3.FeedRepository.ThreadContext; +import org.openmetadata.service.jobs.JobDAO; import org.openmetadata.service.resources.tags.TagLabelUtil; import org.openmetadata.service.search.SearchClient; import org.openmetadata.service.search.SearchListFilter; @@ -236,6 +239,7 @@ public record EntityHistoryWithOffset(EntityHistory entityHistory, int nextOffse @Getter protected final String entityType; @Getter protected final EntityDAO dao; @Getter protected final CollectionDAO daoCollection; + @Getter protected final JobDAO jobDao; @Getter protected final SearchRepository searchRepository; @Getter protected final Set allowedFields; public final boolean supportsSoftDelete; @@ -277,6 +281,7 @@ protected EntityRepository( allowedFields = getEntityFields(entityClass); this.dao = entityDAO; this.daoCollection = Entity.getCollectionDAO(); + this.jobDao = Entity.getJobDAO(); this.searchRepository = Entity.getSearchRepository(); this.entityType = entityType; this.patchFields = getFields(patchFields); @@ -684,7 +689,8 @@ public final List listAll(Fields fields, ListFilter filter) { } public final List listAllForCSV(Fields fields, String parentFqn) { - List jsons = listAllByParentFqn(parentFqn); + ListFilter filter = new ListFilter(Include.NON_DELETED); + List jsons = listAllByParentFqn(parentFqn, filter); List entities = new ArrayList<>(); setFieldsInBulk(jsons, fields, entities); return entities; @@ -709,6 +715,13 @@ public List listAllByParentFqn(String parentFqn) { return dao.listAll(startHash, endHash); } + public List listAllByParentFqn(String parentFqn, ListFilter filter) { + String fqnPrefixHash = FullyQualifiedName.buildHash(parentFqn); + String startHash = fqnPrefixHash + ".00000000000000000000000000000000"; + String endHash = fqnPrefixHash + ".ffffffffffffffffffffffffffffffff"; + return dao.listAll(startHash, endHash, filter); + } + public ResultList listAfter( UriInfo uriInfo, Fields fields, ListFilter filter, int limitParam, String after) { int total = dao.listCount(filter); @@ -1148,15 +1161,32 @@ public final DeleteResponse delete( String updatedBy, UUID id, boolean recursive, boolean hardDelete) { DeleteResponse response = deleteInternal(updatedBy, id, recursive, hardDelete); postDelete(response.entity()); + deleteFromSearch(response.entity(), hardDelete); return response; } + @SuppressWarnings("unused") + @Transaction + public final DeleteResponse deleteByNameIfExists( + String updatedBy, String name, boolean recursive, boolean hardDelete) { + name = quoteFqn ? quoteName(name) : name; + T entity = findByNameOrNull(name, ALL); + if (entity != null) { + DeleteResponse response = deleteInternalByName(updatedBy, name, recursive, hardDelete); + postDelete(response.entity()); + return response; + } else { + return new DeleteResponse<>(null, ENTITY_DELETED); + } + } + @Transaction public final DeleteResponse deleteByName( String updatedBy, String name, boolean recursive, boolean hardDelete) { name = quoteFqn ? quoteName(name) : name; DeleteResponse response = deleteInternalByName(updatedBy, name, recursive, hardDelete); postDelete(response.entity()); + deleteFromSearch(response.entity(), hardDelete); return response; } @@ -1167,12 +1197,12 @@ protected void preDelete(T entity, String deletedBy) { protected void postDelete(T entity) {} - public final void deleteFromSearch(T entity, EventType changeType) { + public final void deleteFromSearch(T entity, boolean hardDelete) { if (supportsSearch) { - if (changeType.equals(ENTITY_SOFT_DELETED)) { - searchRepository.softDeleteOrRestoreEntity(entity, true); - } else { + if (hardDelete) { searchRepository.deleteEntity(entity); + } else { + searchRepository.softDeleteOrRestoreEntity(entity, true); } } } @@ -1279,7 +1309,7 @@ private void deleteChildren(UUID id, boolean recursive, boolean hardDelete, Stri List.of(Relationship.CONTAINS.ordinal(), Relationship.PARENT_OF.ordinal())); if (childrenRecords.isEmpty()) { - LOG.info("No children to delete"); + LOG.debug("No children to delete"); return; } // Entity being deleted contains children entities @@ -1528,7 +1558,16 @@ private void validateAndUpdateExtensionBasedOnPropertyType( jsonNode.put(fieldName, formattedValue); } case "table-cp" -> validateTableType(fieldValue, propertyConfig, fieldName); - case "enum" -> validateEnumKeys(fieldName, fieldValue, propertyConfig); + case "enum" -> { + validateEnumKeys(fieldName, fieldValue, propertyConfig); + List enumValues = + StreamSupport.stream(fieldValue.spliterator(), false) + .map(JsonNode::asText) + .sorted() + .collect(Collectors.toList()); + jsonNode.set(fieldName, JsonUtils.valueToTree(enumValues)); + entity.setExtension(jsonNode); + } default -> {} } } @@ -1675,8 +1714,19 @@ public final Object getExtension(T entity) { } ObjectNode objectNode = JsonUtils.getObjectNode(); for (ExtensionRecord extensionRecord : records) { - String fieldName = TypeRegistry.getPropertyName(extensionRecord.extensionName()); - objectNode.set(fieldName, JsonUtils.readTree(extensionRecord.extensionJson())); + String fieldName = extensionRecord.extensionName().substring(fieldFQNPrefix.length() + 1); + JsonNode fieldValue = JsonUtils.readTree(extensionRecord.extensionJson()); + String customPropertyType = TypeRegistry.getCustomPropertyType(entityType, fieldName); + if ("enum".equals(customPropertyType) && fieldValue.isArray() && fieldValue.size() > 1) { + List sortedEnumValues = + StreamSupport.stream(fieldValue.spliterator(), false) + .map(JsonNode::asText) + .sorted() + .collect(Collectors.toList()); + fieldValue = JsonUtils.valueToTree(sortedEnumValues); + } + + objectNode.set(fieldName, fieldValue); } return objectNode; } @@ -2052,11 +2102,12 @@ public final void validateUsers(List entityReferences) { } } - private boolean validateIfAllRefsAreEntityType(List list, String entityType) { + private static boolean validateIfAllRefsAreEntityType( + List list, String entityType) { return list.stream().allMatch(obj -> obj.getType().equals(entityType)); } - public final void validateReviewers(List entityReferences) { + public static void validateReviewers(List entityReferences) { if (!nullOrEmpty(entityReferences)) { boolean areAllTeam = validateIfAllRefsAreEntityType(entityReferences, TEAM); boolean areAllUsers = validateIfAllRefsAreEntityType(entityReferences, USER); @@ -2414,7 +2465,7 @@ protected void checkSystemEntityDeletion(T entity) { } } - public final List validateOwners(List owners) { + public static List validateOwners(List owners) { if (nullOrEmpty(owners)) { return null; } @@ -2821,6 +2872,13 @@ private void updateExtension() { return; } + if (operation == Operation.PUT && updatedExtension == null) { + // Revert change to non-empty extension if it is being updated by a PUT request + // For PUT operations, existing extension can't be removed. + updated.setExtension(origExtension); + return; + } + List added = new ArrayList<>(); List deleted = new ArrayList<>(); JsonNode origFields = JsonUtils.valueToTree(origExtension); @@ -2969,6 +3027,14 @@ private static List getEntityReferences(List r private void updateStyle() { if (supportsStyle) { + Style originalStyle = original.getStyle(); + Style updatedStyle = updated.getStyle(); + + if (originalStyle == updatedStyle) return; + if (operation == Operation.PUT && updatedStyle == null) { + updatedStyle = originalStyle; + updated.setStyle(updatedStyle); + } recordChange(FIELD_STYLE, original.getStyle(), updated.getStyle(), true); } } @@ -3023,7 +3089,7 @@ private void updateCertification() { SystemRepository systemRepository = Entity.getSystemRepository(); AssetCertificationSettings assetCertificationSettings = - systemRepository.getAssetCertificationSettings(); + systemRepository.getAssetCertificationSettingOrDefault(); String certificationLabel = updatedCertification.getTagLabel().getTagFQN(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EventSubscriptionRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EventSubscriptionRepository.java index 2eefbef79458..52ac95ba0e42 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EventSubscriptionRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EventSubscriptionRepository.java @@ -15,6 +15,7 @@ import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; +import static org.openmetadata.service.apps.bundles.changeEvent.AbstractEventConsumer.OFFSET_EXTENSION; import static org.openmetadata.service.events.subscription.AlertUtil.validateAndBuildFilteringConditions; import static org.openmetadata.service.fernet.Fernet.encryptWebhookSecretKey; import static org.openmetadata.service.util.EntityUtil.objectMatch; @@ -28,12 +29,14 @@ import org.openmetadata.schema.entity.events.ArgumentsInput; import org.openmetadata.schema.entity.events.EventFilterRule; import org.openmetadata.schema.entity.events.EventSubscription; +import org.openmetadata.schema.entity.events.EventSubscriptionOffset; import org.openmetadata.schema.entity.events.SubscriptionDestination; import org.openmetadata.service.Entity; import org.openmetadata.service.events.scheduled.EventSubscriptionScheduler; import org.openmetadata.service.events.subscription.AlertUtil; import org.openmetadata.service.resources.events.subscription.EventSubscriptionResource; import org.openmetadata.service.util.EntityUtil.Fields; +import org.openmetadata.service.util.JsonUtils; @Slf4j public class EventSubscriptionRepository extends EntityRepository { @@ -111,6 +114,29 @@ private void validateFilterRules(EventSubscription entity) { } } + public EventSubscriptionOffset syncEventSubscriptionOffset(String eventSubscriptionName) { + EventSubscription eventSubscription = getByName(null, eventSubscriptionName, getFields("*")); + long latestOffset = daoCollection.changeEventDAO().getLatestOffset(); + long currentTime = System.currentTimeMillis(); + // Upsert Offset + EventSubscriptionOffset eventSubscriptionOffset = + new EventSubscriptionOffset() + .withCurrentOffset(latestOffset) + .withStartingOffset(latestOffset) + .withTimestamp(currentTime); + + Entity.getCollectionDAO() + .eventSubscriptionDAO() + .upsertSubscriberExtension( + eventSubscription.getId().toString(), + OFFSET_EXTENSION, + "eventSubscriptionOffset", + JsonUtils.pojoToJson(eventSubscriptionOffset)); + + EventSubscriptionScheduler.getInstance().updateEventSubscription(eventSubscription); + return eventSubscriptionOffset; + } + @Override public void storeEntity(EventSubscription entity, boolean update) { store(entity, update); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/FeedRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/FeedRepository.java index 17103ba77a0c..bd95cdfd7c9b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/FeedRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/FeedRepository.java @@ -961,7 +961,8 @@ private void validateAssignee(Thread thread) { String createdByUserName = thread.getCreatedBy(); User createdByUser = Entity.getEntityByName(USER, createdByUserName, TEAMS_FIELD, NON_DELETED); - if (Boolean.TRUE.equals(createdByUser.getIsBot())) { + if (Boolean.TRUE.equals(createdByUser.getIsBot()) + && !createdByUser.getName().equals("governance-bot")) { throw new IllegalArgumentException("Task cannot be created by bot only by user or teams"); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryRepository.java index 5192f1afe27b..d6519748d334 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryRepository.java @@ -200,7 +200,7 @@ protected void createEntity(CSVPrinter printer, List csvRecords) thro .withTags( getTagLabels( printer, csvRecord, List.of(Pair.of(7, TagLabel.TagSource.CLASSIFICATION)))) - .withReviewers(getOwners(printer, csvRecord, 8)) + .withReviewers(getReviewers(printer, csvRecord, 8)) .withOwners(getOwners(printer, csvRecord, 9)) .withStatus(getTermStatus(printer, csvRecord)) .withExtension(getExtension(printer, csvRecord, 11)); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java index 2631fefd7159..3ed9baf66d90 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java @@ -803,26 +803,31 @@ private void fetchAndSetRelatedTerms(List entities, Fields fields) allRecords.addAll(fromRecords); allRecords.addAll(toRecords); - Map> relatedTermsMap = new HashMap<>(); + Map> relatedTermIdsMap = new HashMap<>(); for (CollectionDAO.EntityRelationshipObject rec : allRecords) { - UUID termId; - UUID relatedTermId; - - if (termIdsSet.contains(rec.getFromId())) { - termId = UUID.fromString(rec.getFromId()); - relatedTermId = UUID.fromString(rec.getToId()); - } else { - termId = UUID.fromString(rec.getToId()); - relatedTermId = UUID.fromString(rec.getFromId()); - } - - EntityReference relatedTermRef = - new EntityReference().withId(relatedTermId).withType(Entity.GLOSSARY_TERM); + UUID termId = UUID.fromString(rec.getFromId()); + UUID relatedTermId = UUID.fromString(rec.getToId()); - relatedTermsMap.computeIfAbsent(termId, k -> new ArrayList<>()).add(relatedTermRef); + // store the bidirectional relationship in related-term map + relatedTermIdsMap.computeIfAbsent(termId, k -> new HashSet<>()).add(relatedTermId); + relatedTermIdsMap.computeIfAbsent(relatedTermId, k -> new HashSet<>()).add(termId); } + Map> relatedTermsMap = + relatedTermIdsMap.entrySet().stream() + .collect( + Collectors.toMap( + Map.Entry::getKey, + entry -> + entry.getValue().stream() + .map( + id -> + Entity.getEntityReferenceById( + Entity.GLOSSARY_TERM, id, Include.ALL)) + .sorted(EntityUtil.compareEntityReference) + .toList())); + if (!allRecords.isEmpty()) { for (GlossaryTerm term : entities) { List relatedTerms = diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java index 134fe998ec15..9026c4f36daf 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java @@ -633,7 +633,7 @@ public void deleteLineageBySource(UUID toId, String toEntity, String source) { .findLineageBySourcePipeline(toId, toEntity, source, Relationship.UPSTREAM.ordinal()); // Finally, delete lineage relationship dao.relationshipDAO() - .deleteLineageBySourcePipeline(toId, toEntity, source, Relationship.UPSTREAM.ordinal()); + .deleteLineageBySourcePipeline(toId, toEntity, Relationship.UPSTREAM.ordinal()); } else { relations = dao.relationshipDAO() diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ListFilter.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ListFilter.java index 08f299b5cf4c..425c49ed2548 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ListFilter.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ListFilter.java @@ -11,6 +11,7 @@ import org.openmetadata.schema.utils.EntityInterfaceUtil; import org.openmetadata.service.Entity; import org.openmetadata.service.resources.databases.DatasourceConfig; +import org.openmetadata.service.security.policyevaluator.ResourceContext; import org.openmetadata.service.util.FullyQualifiedName; public class ListFilter extends Filter { @@ -50,6 +51,25 @@ public String getCondition(String tableName) { return condition.isEmpty() ? "WHERE TRUE" : "WHERE " + condition; } + public ResourceContext getResourceContext(String entityType) { + if (queryParams.containsKey("service") && queryParams.get("service") != null) { + return new ResourceContext<>( + Entity.getServiceType(entityType), null, queryParams.get("service")); + } else if (queryParams.containsKey(Entity.DATABASE) + && queryParams.get(Entity.DATABASE) != null) { + return new ResourceContext<>(Entity.DATABASE, null, queryParams.get(Entity.DATABASE)); + } else if (queryParams.containsKey(Entity.DATABASE_SCHEMA) + && queryParams.get(Entity.DATABASE_SCHEMA) != null) { + return new ResourceContext<>( + Entity.DATABASE_SCHEMA, null, queryParams.get(Entity.DATABASE_SCHEMA)); + } else if (queryParams.containsKey(Entity.API_COLLCECTION) + && queryParams.get(Entity.API_COLLCECTION) != null) { + return new ResourceContext<>( + Entity.API_COLLCECTION, null, queryParams.get(Entity.API_COLLCECTION)); + } + return new ResourceContext<>(entityType); + } + private String getAssignee() { String assignee = queryParams.get("assignee"); return assignee == null ? "" : String.format("assignee = '%s'", assignee); @@ -258,22 +278,22 @@ private String getTestSuiteTypeCondition(String tableName) { } return switch (testSuiteType) { - case ("executable") -> { + // We'll clean up the executable when we deprecate the /executable endpoints + case "basic", "executable" -> { if (Boolean.TRUE.equals(DatasourceConfig.getInstance().isMySQL())) { yield String.format( - "(JSON_UNQUOTE(JSON_EXTRACT(%s.json, '$.executable')) = 'true')", tableName); + "(JSON_UNQUOTE(JSON_EXTRACT(%s.json, '$.basic')) = 'true')", tableName); } - yield String.format("(%s.json->>'executable' = 'true')", tableName); + yield String.format("(%s.json->>'basic' = 'true')", tableName); } - case ("logical") -> { + case "logical" -> { if (Boolean.TRUE.equals(DatasourceConfig.getInstance().isMySQL())) { yield String.format( - "(JSON_UNQUOTE(JSON_EXTRACT(%s.json, '$.executable')) = 'false' OR JSON_UNQUOTE(JSON_EXTRACT(%s.json, '$.executable')) IS NULL)", + "(JSON_UNQUOTE(JSON_EXTRACT(%s.json, '$.basic')) = 'false' OR JSON_UNQUOTE(JSON_EXTRACT(%s.json, '$.basic')) IS NULL)", tableName, tableName); } yield String.format( - "(%s.json->>'executable' = 'false' or %s.json -> 'executable' is null)", - tableName, tableName); + "(%s.json->>'basic' = 'false' or %s.json -> 'basic' is null)", tableName, tableName); } default -> ""; }; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/MlModelRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/MlModelRepository.java index e2b582ecb7c8..e9ada22bd098 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/MlModelRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/MlModelRepository.java @@ -17,6 +17,7 @@ import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; import static org.openmetadata.service.Entity.DASHBOARD; import static org.openmetadata.service.Entity.MLMODEL; +import static org.openmetadata.service.Entity.getEntityReference; import static org.openmetadata.service.resources.tags.TagLabelUtil.checkMutuallyExclusive; import static org.openmetadata.service.util.EntityUtil.entityReferenceMatch; import static org.openmetadata.service.util.EntityUtil.mlFeatureMatch; @@ -149,8 +150,7 @@ private void validateReferences(List mlFeatures) { private void validateMlDataSource(MlFeatureSource source) { if (source.getDataSource() != null) { - Entity.getEntityReferenceById( - source.getDataSource().getType(), source.getDataSource().getId(), Include.NON_DELETED); + Entity.getEntityReference(source.getDataSource(), Include.NON_DELETED); } } @@ -211,7 +211,8 @@ private void setMlFeatureSourcesLineage(MlModel mlModel) { .getFeatureSources() .forEach( mlFeatureSource -> { - EntityReference targetEntity = mlFeatureSource.getDataSource(); + EntityReference targetEntity = + getEntityReference(mlFeatureSource.getDataSource(), Include.ALL); if (targetEntity != null) { addRelationship( targetEntity.getId(), diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/PipelineRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/PipelineRepository.java index 8aaf4db08d8f..2b81433ac352 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/PipelineRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/PipelineRepository.java @@ -29,7 +29,6 @@ import java.util.ArrayList; import java.util.List; import javax.ws.rs.core.Response; -import javax.ws.rs.core.UriInfo; import org.apache.commons.lang3.tuple.Triple; import org.jdbi.v3.sqlobject.transaction.Transaction; import org.openmetadata.schema.EntityInterface; @@ -42,6 +41,8 @@ import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.FieldChange; import org.openmetadata.schema.type.Include; +import org.openmetadata.schema.type.LineageDetails; +import org.openmetadata.schema.type.Relationship; import org.openmetadata.schema.type.Status; import org.openmetadata.schema.type.TagLabel; import org.openmetadata.schema.type.Task; @@ -171,8 +172,7 @@ private PipelineStatus getPipelineStatus(Pipeline pipeline) { PipelineStatus.class); } - public RestUtil.PutResponse addPipelineStatus( - UriInfo uriInfo, String fqn, PipelineStatus pipelineStatus) { + public RestUtil.PutResponse addPipelineStatus(String fqn, PipelineStatus pipelineStatus) { // Validate the request content Pipeline pipeline = daoCollection.pipelineDAO().findEntityByName(fqn); pipeline.setService(getContainer(pipeline.getId())); @@ -296,6 +296,18 @@ public void storeEntity(Pipeline pipeline, boolean update) { pipeline.withService(service).withTasks(taskWithTagsAndOwners); } + @Override + protected void cleanup(Pipeline pipeline) { + // When a pipeline is removed , the linege needs to be removed + daoCollection + .relationshipDAO() + .deleteLineageBySourcePipeline( + pipeline.getId(), + LineageDetails.Source.PIPELINE_LINEAGE.value(), + Relationship.UPSTREAM.ordinal()); + super.cleanup(pipeline); + } + @Override public void storeRelationships(Pipeline pipeline) { addServiceRelationship(pipeline, pipeline.getService()); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/StoredProcedureRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/StoredProcedureRepository.java index 60ad7db78d23..11624a475808 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/StoredProcedureRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/StoredProcedureRepository.java @@ -11,6 +11,7 @@ import org.openmetadata.schema.entity.data.StoredProcedure; import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.Include; +import org.openmetadata.schema.type.LineageDetails; import org.openmetadata.schema.type.Relationship; import org.openmetadata.service.Entity; import org.openmetadata.service.resources.databases.StoredProcedureResource; @@ -69,6 +70,18 @@ public void storeRelationships(StoredProcedure storedProcedure) { Relationship.CONTAINS); } + @Override + protected void cleanup(StoredProcedure storedProcedure) { + // When a pipeline is removed , the linege needs to be removed + daoCollection + .relationshipDAO() + .deleteLineageBySourcePipeline( + storedProcedure.getId(), + LineageDetails.Source.QUERY_LINEAGE.value(), + Relationship.UPSTREAM.ordinal()); + super.cleanup(storedProcedure); + } + @Override public void setInheritedFields(StoredProcedure storedProcedure, EntityUtil.Fields fields) { DatabaseSchema schema = diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SystemRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SystemRepository.java index d08b2ca6c336..fbcd2118163d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SystemRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SystemRepository.java @@ -16,6 +16,9 @@ import org.jdbi.v3.sqlobject.transaction.Transaction; import org.openmetadata.api.configuration.UiThemePreference; import org.openmetadata.schema.configuration.AssetCertificationSettings; +import org.openmetadata.schema.configuration.ExecutorConfiguration; +import org.openmetadata.schema.configuration.HistoryCleanUpConfiguration; +import org.openmetadata.schema.configuration.WorkflowSettings; import org.openmetadata.schema.email.SmtpSettings; import org.openmetadata.schema.entity.services.ingestionPipelines.PipelineServiceClientResponse; import org.openmetadata.schema.security.client.OpenMetadataJWTClientConfig; @@ -32,13 +35,16 @@ import org.openmetadata.service.OpenMetadataApplicationConfig; import org.openmetadata.service.exception.CustomExceptionMessage; import org.openmetadata.service.fernet.Fernet; +import org.openmetadata.service.governance.workflows.WorkflowHandler; import org.openmetadata.service.jdbi3.CollectionDAO.SystemDAO; import org.openmetadata.service.migration.MigrationValidationClient; import org.openmetadata.service.resources.settings.SettingsCache; import org.openmetadata.service.search.SearchRepository; import org.openmetadata.service.secrets.SecretsManager; import org.openmetadata.service.secrets.SecretsManagerFactory; +import org.openmetadata.service.secrets.masker.PasswordEntityMasker; import org.openmetadata.service.security.JwtFilter; +import org.openmetadata.service.security.auth.LoginAttemptCache; import org.openmetadata.service.util.JsonUtils; import org.openmetadata.service.util.OpenMetadataConnectionBuilder; import org.openmetadata.service.util.RestUtil; @@ -103,6 +109,12 @@ public Settings getConfigWithKey(String key) { return null; } + if (fetchedSettings.getConfigType() == SettingsType.EMAIL_CONFIGURATION) { + SmtpSettings emailConfig = (SmtpSettings) fetchedSettings.getConfigValue(); + emailConfig.setPassword(PasswordEntityMasker.PASSWORD_MASK); + fetchedSettings.setConfigValue(emailConfig); + } + return fetchedSettings; } catch (Exception ex) { LOG.error("Error while trying fetch Settings ", ex); @@ -119,6 +131,37 @@ public AssetCertificationSettings getAssetCertificationSettings() { .orElse(null); } + public AssetCertificationSettings getAssetCertificationSettingOrDefault() { + AssetCertificationSettings assetCertificationSettings = getAssetCertificationSettings(); + if (assetCertificationSettings == null) { + assetCertificationSettings = + new AssetCertificationSettings() + .withAllowedClassification("Certification") + .withValidityPeriod("P30D"); + } + return assetCertificationSettings; + } + + public WorkflowSettings getWorkflowSettings() { + Optional oWorkflowSettings = + Optional.ofNullable(getConfigWithKey(SettingsType.WORKFLOW_SETTINGS.value())); + + return oWorkflowSettings + .map(settings -> (WorkflowSettings) settings.getConfigValue()) + .orElse(null); + } + + public WorkflowSettings getWorkflowSettingsOrDefault() { + WorkflowSettings workflowSettings = getWorkflowSettings(); + if (workflowSettings == null) { + workflowSettings = + new WorkflowSettings() + .withExecutorConfiguration(new ExecutorConfiguration()) + .withHistoryCleanUpConfiguration(new HistoryCleanUpConfiguration()); + } + return workflowSettings; + } + public Settings getEmailConfigInternal() { try { Settings setting = dao.getConfigWithKey(SettingsType.EMAIL_CONFIGURATION.value()); @@ -209,6 +252,17 @@ public Response patchSetting(String settingName, JsonPatch patch) { return (new RestUtil.PutResponse<>(Response.Status.OK, original, ENTITY_UPDATED)).toResponse(); } + private void postUpdate(SettingsType settingsType) { + if (settingsType == SettingsType.WORKFLOW_SETTINGS) { + WorkflowHandler workflowHandler = WorkflowHandler.getInstance(); + workflowHandler.initializeNewProcessEngine(workflowHandler.getProcessEngineConfiguration()); + } + + if (settingsType == SettingsType.LOGIN_CONFIGURATION) { + LoginAttemptCache.updateLoginConfiguration(); + } + } + public void updateSetting(Settings setting) { try { if (setting.getConfigType() == SettingsType.EMAIL_CONFIGURATION) { @@ -235,6 +289,7 @@ public void updateSetting(Settings setting) { setting.getConfigType().toString(), JsonUtils.pojoToJson(setting.getConfigValue())); // Invalidate Cache SettingsCache.invalidateSettings(setting.getConfigType().value()); + postUpdate(setting.getConfigType()); } catch (Exception ex) { LOG.error("Failing in Updating Setting.", ex); throw new CustomExceptionMessage( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java index 0247bf0b7cdb..308b7adbd8e4 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java @@ -1138,7 +1138,7 @@ private void validateTableConstraints(Table table) { String toParent = FullyQualifiedName.getParentFQN(column); String columnName = FullyQualifiedName.getColumnName(column); try { - Table toTable = findByName(toParent, NON_DELETED); + Table toTable = findByName(toParent, ALL); validateColumn(toTable, columnName); } catch (EntityNotFoundException e) { throw new EntitySpecViolationException("Table not found: " + toParent); @@ -1232,8 +1232,7 @@ private void addConstraintRelationship(Table table, List constr for (String column : constraint.getReferredColumns()) { String toParent = FullyQualifiedName.getParentFQN(column); try { - EntityReference toTable = - Entity.getEntityReferenceByName(TABLE, toParent, NON_DELETED); + EntityReference toTable = Entity.getEntityReferenceByName(TABLE, toParent, ALL); addRelationship( table.getId(), toTable.getId(), TABLE, TABLE, Relationship.RELATED_TO); } catch (EntityNotFoundException e) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java index 42978ca47b72..ec76915cf0a3 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java @@ -23,6 +23,8 @@ import java.util.Locale; import java.util.Map; import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import javax.json.JsonPatch; import javax.ws.rs.core.Response; import javax.ws.rs.core.UriInfo; @@ -33,7 +35,6 @@ import org.openmetadata.schema.EntityTimeSeriesInterface; import org.openmetadata.schema.api.feed.CloseTask; import org.openmetadata.schema.api.feed.ResolveTask; -import org.openmetadata.schema.api.tests.CreateTestCaseResult; import org.openmetadata.schema.entity.data.Table; import org.openmetadata.schema.entity.teams.User; import org.openmetadata.schema.tests.TestCase; @@ -76,8 +77,9 @@ public class TestCaseRepository extends EntityRepository { private static final String UPDATE_FIELDS = "owners,entityLink,testSuite,testSuites,testDefinition"; private static final String PATCH_FIELDS = - "owners,entityLink,testSuite,testDefinition,computePassedFailedRowCount,useDynamicAssertion"; + "owners,entityLink,testSuite,testSuites,testDefinition,computePassedFailedRowCount,useDynamicAssertion"; public static final String FAILED_ROWS_SAMPLE_EXTENSION = "testCase.failedRowsSample"; + private final ExecutorService asyncExecutor = Executors.newFixedThreadPool(1); public TestCaseRepository() { super( @@ -185,7 +187,7 @@ private EntityReference getTestSuite(TestCase test) throws EntityNotFoundExcepti findFromRecords(test.getId(), entityType, Relationship.CONTAINS, TEST_SUITE); for (CollectionDAO.EntityRelationshipRecord testSuiteId : records) { TestSuite testSuite = Entity.getEntity(TEST_SUITE, testSuiteId.getId(), "", Include.ALL); - if (Boolean.TRUE.equals(testSuite.getExecutable())) { + if (Boolean.TRUE.equals(testSuite.getBasic())) { return testSuite.getEntityReference(); } } @@ -245,14 +247,16 @@ public void storeEntity(TestCase test, boolean update) { EntityReference testSuite = test.getTestSuite(); EntityReference testDefinition = test.getTestDefinition(); TestCaseResult testCaseResult = test.getTestCaseResult(); + List testSuites = test.getTestSuites(); // Don't store testCaseResult, owner, database, href and tags as JSON. // Build it on the fly based on relationships - test.withTestSuite(null).withTestDefinition(null).withTestCaseResult(null); + test.withTestSuite(null).withTestSuites(null).withTestDefinition(null).withTestCaseResult(null); store(test, update); // Restore the relationships test.withTestSuite(testSuite) + .withTestSuites(testSuites) .withTestDefinition(testDefinition) .withTestCaseResult(testCaseResult); } @@ -304,26 +308,10 @@ private void updateLogicalTestSuite(UUID testSuiteId) { public RestUtil.PutResponse addTestCaseResult( String updatedBy, UriInfo uriInfo, String fqn, TestCaseResult testCaseResult) { - // TODO: REMOVED ONCE DEPRECATED IN TEST CASE RESOURCE - CreateTestCaseResult createTestCaseResult = - new CreateTestCaseResult() - .withTimestamp(testCaseResult.getTimestamp()) - .withTestCaseStatus(testCaseResult.getTestCaseStatus()) - .withResult(testCaseResult.getResult()) - .withSampleData(testCaseResult.getSampleData()) - .withTestResultValue(testCaseResult.getTestResultValue()) - .withPassedRows(testCaseResult.getPassedRows()) - .withFailedRows(testCaseResult.getFailedRows()) - .withPassedRowsPercentage(testCaseResult.getPassedRowsPercentage()) - .withFailedRowsPercentage(testCaseResult.getFailedRowsPercentage()) - .withIncidentId(testCaseResult.getIncidentId()) - .withMaxBound(testCaseResult.getMaxBound()) - .withMinBound(testCaseResult.getMinBound()); - TestCaseResultRepository testCaseResultRepository = (TestCaseResultRepository) Entity.getEntityTimeSeriesRepository(TEST_CASE_RESULT); Response response = - testCaseResultRepository.addTestCaseResult(updatedBy, uriInfo, fqn, createTestCaseResult); + testCaseResultRepository.addTestCaseResult(updatedBy, uriInfo, fqn, testCaseResult); return new RestUtil.PutResponse<>( Response.Status.CREATED, (TestCaseResult) response.getEntity(), ENTITY_UPDATED); } @@ -367,10 +355,17 @@ public RestUtil.PutResponse deleteTestCaseResult( } private void deleteAllTestCaseResults(String fqn) { - // Delete all the test case results TestCaseResultRepository testCaseResultRepository = (TestCaseResultRepository) Entity.getEntityTimeSeriesRepository(TEST_CASE_RESULT); testCaseResultRepository.deleteAllTestCaseResults(fqn); + asyncExecutor.submit( + () -> { + try { + testCaseResultRepository.deleteAllTestCaseResults(fqn); + } catch (Exception e) { + LOG.error("Error deleting test case results for test case {}", fqn, e); + } + }); } @SneakyThrows @@ -444,13 +439,13 @@ public int getTestCaseCount(List testCaseIds) { return daoCollection.testCaseDAO().countOfTestCases(testCaseIds); } - public void isTestSuiteExecutable(String testSuiteFqn) { + public void isTestSuiteBasic(String testSuiteFqn) { TestSuite testSuite = Entity.getEntityByName(Entity.TEST_SUITE, testSuiteFqn, null, null); - if (Boolean.FALSE.equals(testSuite.getExecutable())) { + if (Boolean.FALSE.equals(testSuite.getBasic())) { throw new IllegalArgumentException( "Test suite " + testSuite.getName() - + " is not executable. Cannot create test cases for non-executable test suites."); + + " is not basic. Cannot create test cases for non-basic test suites."); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseResultRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseResultRepository.java index 4e247fbab2bc..0e42f27d26cb 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseResultRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseResultRepository.java @@ -17,7 +17,6 @@ import javax.ws.rs.core.UriInfo; import lombok.SneakyThrows; import org.openmetadata.common.utils.CommonUtil; -import org.openmetadata.schema.api.tests.CreateTestCaseResult; import org.openmetadata.schema.tests.ResultSummary; import org.openmetadata.schema.tests.TestCase; import org.openmetadata.schema.tests.TestSuite; @@ -80,9 +79,8 @@ public ResultList getTestCaseResults(String fqn, Long startTs, L } public Response addTestCaseResult( - String updatedBy, UriInfo uriInfo, String fqn, CreateTestCaseResult createTestCaseResult) { + String updatedBy, UriInfo uriInfo, String fqn, TestCaseResult testCaseResult) { TestCase testCase = Entity.getEntityByName(TEST_CASE, fqn, "", Include.ALL); - TestCaseResult testCaseResult = getTestCaseResult(createTestCaseResult, testCase); if (testCaseResult.getTestCaseStatus() == TestCaseStatus.Success) { testCaseRepository.deleteTestCaseFailedRowsSample(testCase.getId()); } @@ -299,24 +297,4 @@ public boolean hasTestCaseFailure(String fqn) throws IOException { .anyMatch( testCaseResult -> testCaseResult.getTestCaseStatus().equals(TestCaseStatus.Failed)); } - - private TestCaseResult getTestCaseResult( - CreateTestCaseResult createTestCaseResults, TestCase testCase) { - RestUtil.validateTimestampMilliseconds(createTestCaseResults.getTimestamp()); - return new TestCaseResult() - .withId(UUID.randomUUID()) - .withTestCaseFQN(testCase.getFullyQualifiedName()) - .withTimestamp(createTestCaseResults.getTimestamp()) - .withTestCaseStatus(createTestCaseResults.getTestCaseStatus()) - .withResult(createTestCaseResults.getResult()) - .withSampleData(createTestCaseResults.getSampleData()) - .withTestResultValue(createTestCaseResults.getTestResultValue()) - .withPassedRows(createTestCaseResults.getPassedRows()) - .withFailedRows(createTestCaseResults.getFailedRows()) - .withPassedRowsPercentage(createTestCaseResults.getPassedRowsPercentage()) - .withFailedRowsPercentage(createTestCaseResults.getFailedRowsPercentage()) - .withIncidentId(createTestCaseResults.getIncidentId()) - .withMaxBound(createTestCaseResults.getMaxBound()) - .withMinBound(createTestCaseResults.getMinBound()); - } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java index 7d4b486e8f08..55a5d3b67976 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java @@ -127,10 +127,10 @@ public void setFields(TestSuite entity, EntityUtil.Fields fields) { @Override public void setInheritedFields(TestSuite testSuite, EntityUtil.Fields fields) { - if (Boolean.TRUE.equals(testSuite.getExecutable())) { + if (Boolean.TRUE.equals(testSuite.getBasic()) && testSuite.getBasicEntityReference() != null) { Table table = Entity.getEntity( - TABLE, testSuite.getExecutableEntityReference().getId(), "owners,domain", ALL); + TABLE, testSuite.getBasicEntityReference().getId(), "owners,domain", ALL); inheritOwners(testSuite, fields, table); inheritDomain(testSuite, fields, table); } @@ -145,10 +145,10 @@ public void clearFields(TestSuite entity, EntityUtil.Fields fields) { @Override public void setFullyQualifiedName(TestSuite testSuite) { - if (testSuite.getExecutableEntityReference() != null) { + if (testSuite.getBasicEntityReference() != null) { testSuite.setFullyQualifiedName( FullyQualifiedName.add( - testSuite.getExecutableEntityReference().getFullyQualifiedName(), "testSuite")); + testSuite.getBasicEntityReference().getFullyQualifiedName(), "testSuite")); } else { testSuite.setFullyQualifiedName(quoteName(testSuite.getName())); } @@ -328,22 +328,21 @@ public TestSummary getTestSummary(UUID testSuiteId) { @Override protected void postCreate(TestSuite entity) { super.postCreate(entity); - if (Boolean.TRUE.equals(entity.getExecutable()) - && entity.getExecutableEntityReference() != null) { + if (Boolean.TRUE.equals(entity.getBasic()) && entity.getBasicEntityReference() != null) { // Update table index with test suite field EntityInterface entityInterface = - getEntity(entity.getExecutableEntityReference(), "testSuite", ALL); + getEntity(entity.getBasicEntityReference(), "testSuite", ALL); IndexMapping indexMapping = - searchRepository.getIndexMapping(entity.getExecutableEntityReference().getType()); + searchRepository.getIndexMapping(entity.getBasicEntityReference().getType()); SearchClient searchClient = searchRepository.getSearchClient(); SearchIndex index = searchRepository .getSearchIndexFactory() - .buildIndex(entity.getExecutableEntityReference().getType(), entityInterface); + .buildIndex(entity.getBasicEntityReference().getType(), entityInterface); Map doc = index.buildSearchIndexDoc(); searchClient.updateEntity( indexMapping.getIndexName(searchRepository.getClusterAlias()), - entity.getExecutableEntityReference().getId().toString(), + entity.getBasicEntityReference().getId().toString(), doc, "ctx._source.testSuite = params.testSuite;"); } @@ -413,7 +412,7 @@ public void storeEntity(TestSuite entity, boolean update) { @Override public void storeRelationships(TestSuite entity) { - if (Boolean.TRUE.equals(entity.getExecutable())) { + if (Boolean.TRUE.equals(entity.getBasic())) { storeExecutableRelationship(entity); } } @@ -421,10 +420,7 @@ public void storeRelationships(TestSuite entity) { public void storeExecutableRelationship(TestSuite testSuite) { Table table = Entity.getEntityByName( - Entity.TABLE, - testSuite.getExecutableEntityReference().getFullyQualifiedName(), - null, - null); + Entity.TABLE, testSuite.getBasicEntityReference().getFullyQualifiedName(), null, null); addRelationship( table.getId(), testSuite.getId(), Entity.TABLE, TEST_SUITE, Relationship.CONTAINS); } @@ -436,6 +432,7 @@ public RestUtil.DeleteResponse deleteLogicalTestSuite( String updatedBy = securityContext.getUserPrincipal().getName(); preDelete(original, updatedBy); setFieldsInternal(original, putFields); + deleteChildIngestionPipelines(original.getId(), hardDelete, updatedBy); EventType changeType; TestSuite updated = JsonUtils.readValue(JsonUtils.pojoToJson(original), TestSuite.class); @@ -456,6 +453,24 @@ public RestUtil.DeleteResponse deleteLogicalTestSuite( return new RestUtil.DeleteResponse<>(updated, changeType); } + /** + * Always delete as if it was marked recursive. Deleting a Logical Suite should + * just go ahead and clean the Ingestion Pipelines + */ + private void deleteChildIngestionPipelines(UUID id, boolean hardDelete, String updatedBy) { + List childrenRecords = + daoCollection + .relationshipDAO() + .findTo(id, entityType, Relationship.CONTAINS.ordinal(), Entity.INGESTION_PIPELINE); + + if (childrenRecords.isEmpty()) { + LOG.debug("No children to delete"); + return; + } + // Delete all the contained entities + deleteChildren(childrenRecords, hardDelete, updatedBy); + } + private void updateTestSummaryFromBucket(JsonObject bucket, TestSummary testSummary) { String key = bucket.getString("key"); Integer count = bucket.getJsonNumber("doc_count").intValue(); @@ -492,8 +507,8 @@ public static TestSuite copyTestSuite(TestSuite testSuite) { .withHref(testSuite.getHref()) .withId(testSuite.getId()) .withName(testSuite.getName()) - .withExecutable(testSuite.getExecutable()) - .withExecutableEntityReference(testSuite.getExecutableEntityReference()) + .withBasic(testSuite.getBasic()) + .withBasicEntityReference(testSuite.getBasicEntityReference()) .withServiceType(testSuite.getServiceType()) .withOwners(testSuite.getOwners()) .withUpdatedBy(testSuite.getUpdatedBy()) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TypeRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TypeRepository.java index 1d9078a264f6..9c73d1a5502c 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TypeRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TypeRepository.java @@ -27,6 +27,7 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; @@ -38,6 +39,8 @@ import org.openmetadata.schema.entity.Type; import org.openmetadata.schema.entity.type.Category; import org.openmetadata.schema.entity.type.CustomProperty; +import org.openmetadata.schema.jobs.BackgroundJob; +import org.openmetadata.schema.jobs.EnumCleanupArgs; import org.openmetadata.schema.type.CustomPropertyConfig; import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.Include; @@ -47,6 +50,7 @@ import org.openmetadata.service.Entity; import org.openmetadata.service.TypeRegistry; import org.openmetadata.service.exception.CatalogExceptionMessage; +import org.openmetadata.service.jobs.EnumCleanupHandler; import org.openmetadata.service.resources.types.TypeResource; import org.openmetadata.service.util.EntityUtil; import org.openmetadata.service.util.EntityUtil.Fields; @@ -121,6 +125,11 @@ public EntityUpdater getUpdater(Type original, Type updated, Operation operation return new TypeUpdater(original, updated, operation); } + @Override + public void postUpdate(Type original, Type updated) { + super.postUpdate(original, updated); + } + public PutResponse addCustomProperty( UriInfo uriInfo, String updatedBy, UUID id, CustomProperty property) { Type type = find(id, Include.NON_DELETED); @@ -166,6 +175,11 @@ private List getCustomProperties(Type type) { for (Triple result : results) { CustomProperty property = JsonUtils.readValue(result.getRight(), CustomProperty.class); property.setPropertyType(this.getReferenceByName(result.getMiddle(), NON_DELETED)); + + if ("enum".equals(property.getPropertyType().getName())) { + sortEnumKeys(property); + } + customProperties.add(property); } customProperties.sort(EntityUtil.compareCustomProperty); @@ -262,6 +276,19 @@ private void validateTableTypeConfig(CustomPropertyConfig config) { } } + @SuppressWarnings("unchecked") + private void sortEnumKeys(CustomProperty property) { + Object enumConfig = property.getCustomPropertyConfig().getConfig(); + if (enumConfig instanceof Map) { + Map configMap = (Map) enumConfig; + if (configMap.get("values") instanceof List) { + List values = (List) configMap.get("values"); + List sortedValues = values.stream().sorted().collect(Collectors.toList()); + configMap.put("values", sortedValues); + } + } + } + /** Handles entity updated from PUT and POST operation. */ public class TypeUpdater extends EntityUpdater { public TypeUpdater(Type original, Type updated, Operation operation) { @@ -428,6 +455,7 @@ private void updateCustomPropertyConfig( Relationship.HAS.ordinal(), "customProperty", customPropertyJson); + postUpdateCustomPropertyConfig(entity, origProperty, updatedProperty); } } @@ -443,9 +471,51 @@ private void validatePropertyConfigUpdate( HashSet updatedValues = new HashSet<>(updatedConfig.getValues()); if (updatedValues.size() != updatedConfig.getValues().size()) { throw new IllegalArgumentException("Enum Custom Property values cannot have duplicates."); - } else if (!updatedValues.containsAll(origConfig.getValues())) { - throw new IllegalArgumentException( - "Existing Enum Custom Property values cannot be removed."); + } + } + } + + private void postUpdateCustomPropertyConfig( + Type entity, CustomProperty origProperty, CustomProperty updatedProperty) { + String updatedBy = entity.getUpdatedBy(); + if (origProperty.getPropertyType().getName().equals("enum")) { + sortEnumKeys(updatedProperty); + EnumConfig origConfig = + JsonUtils.convertValue( + origProperty.getCustomPropertyConfig().getConfig(), EnumConfig.class); + EnumConfig updatedConfig = + JsonUtils.convertValue( + updatedProperty.getCustomPropertyConfig().getConfig(), EnumConfig.class); + HashSet origKeys = new HashSet<>(origConfig.getValues()); + HashSet updatedKeys = new HashSet<>(updatedConfig.getValues()); + + HashSet removedKeys = new HashSet<>(origKeys); + removedKeys.removeAll(updatedKeys); + HashSet addedKeys = new HashSet<>(updatedKeys); + addedKeys.removeAll(origKeys); + + if (!removedKeys.isEmpty() && addedKeys.isEmpty()) { + List removedEnumKeys = new ArrayList<>(removedKeys); + + try { + EnumCleanupArgs enumCleanupArgs = + new EnumCleanupArgs() + .withPropertyName(updatedProperty.getName()) + .withRemovedEnumKeys(removedEnumKeys) + .withEntityType(entity.getName()); + + String jobArgs = JsonUtils.pojoToJson(enumCleanupArgs); + long jobId = + jobDao.insertJob( + BackgroundJob.JobType.CUSTOM_PROPERTY_ENUM_CLEANUP, + new EnumCleanupHandler(daoCollection), + jobArgs, + updatedBy); + + } catch (Exception e) { + LOG.error("Failed to trigger background job for enum cleanup", e); + throw new RuntimeException("Failed to trigger background job", e); + } } } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/locator/ConnectionAwareAnnotationSqlLocator.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/locator/ConnectionAwareAnnotationSqlLocator.java index 9e0a70e4e782..a4d7a34ee011 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/locator/ConnectionAwareAnnotationSqlLocator.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/locator/ConnectionAwareAnnotationSqlLocator.java @@ -19,6 +19,7 @@ import java.lang.reflect.Method; import java.util.Arrays; +import java.util.List; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -74,6 +75,22 @@ private Optional getAnnotationValue(Method method) { .findFirst()) .flatMap(identity()) // Unwrap Option> to Optional .map(ConnectionAwareSqlQuery::value), + () -> { + Optional containerAnnotation = + Optional.ofNullable(method.getAnnotation(ConnectionAwareSqlBatchContainer.class)); + + Optional> annotationsList = + containerAnnotation.map(ConnectionAwareSqlBatchContainer::value).map(Arrays::asList); + + Optional matchingAnnotation = + annotationsList.flatMap( + annotations -> + annotations.stream() + .filter(annotation -> annotation.connectionType().equals(connectionType)) + .findFirst()); + + return matchingAnnotation.map(ConnectionAwareSqlBatch::value); + }, () -> SqlAnnotations.getAnnotationValue(method)); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/locator/ConnectionAwareSqlBatch.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/locator/ConnectionAwareSqlBatch.java new file mode 100644 index 000000000000..6b74526cca72 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/locator/ConnectionAwareSqlBatch.java @@ -0,0 +1,16 @@ +package org.openmetadata.service.jdbi3.locator; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Repeatable; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.METHOD}) +@Repeatable(ConnectionAwareSqlBatchContainer.class) +public @interface ConnectionAwareSqlBatch { + String value() default ""; + + ConnectionType connectionType(); +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/locator/ConnectionAwareSqlBatchContainer.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/locator/ConnectionAwareSqlBatchContainer.java new file mode 100644 index 000000000000..663a16769b7b --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/locator/ConnectionAwareSqlBatchContainer.java @@ -0,0 +1,15 @@ +package org.openmetadata.service.jdbi3.locator; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import org.jdbi.v3.sqlobject.SqlOperation; +import org.jdbi.v3.sqlobject.statement.internal.SqlBatchHandler; + +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.METHOD}) +@SqlOperation(SqlBatchHandler.class) +public @interface ConnectionAwareSqlBatchContainer { + ConnectionAwareSqlBatch[] value(); +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jobs/BackgroundJobException.java b/openmetadata-service/src/main/java/org/openmetadata/service/jobs/BackgroundJobException.java new file mode 100644 index 000000000000..429d35550a47 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jobs/BackgroundJobException.java @@ -0,0 +1,19 @@ +package org.openmetadata.service.jobs; + +public class BackgroundJobException extends RuntimeException { + private final long jobId; + + public BackgroundJobException(long jobId, String message) { + super(message); + this.jobId = jobId; + } + + public BackgroundJobException(long jobId, String message, Throwable cause) { + super(message, cause); + this.jobId = jobId; + } + + public long getJobId() { + return jobId; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jobs/EnumCleanupHandler.java b/openmetadata-service/src/main/java/org/openmetadata/service/jobs/EnumCleanupHandler.java new file mode 100644 index 000000000000..28dceacde87d --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jobs/EnumCleanupHandler.java @@ -0,0 +1,81 @@ +package org.openmetadata.service.jobs; + +import static org.openmetadata.service.TypeRegistry.getCustomPropertyFQN; + +import com.fasterxml.jackson.core.type.TypeReference; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.openmetadata.schema.jobs.BackgroundJob; +import org.openmetadata.schema.jobs.EnumCleanupArgs; +import org.openmetadata.service.jdbi3.CollectionDAO; +import org.openmetadata.service.util.JsonUtils; + +@Slf4j +public class EnumCleanupHandler implements JobHandler { + private final CollectionDAO daoCollection; + + public EnumCleanupHandler(CollectionDAO daoCollection) { + this.daoCollection = daoCollection; + } + + @Override + public void runJob(BackgroundJob job) throws BackgroundJobException { + try { + Object jobArgs = job.getJobArgs(); + LOG.debug("Starting EnumCleanupHandler job with args: {}", jobArgs); + EnumCleanupArgs enumCleanupArgs; + try { + enumCleanupArgs = JsonUtils.convertValue(jobArgs, EnumCleanupArgs.class); + } catch (IllegalArgumentException e) { + throw new BackgroundJobException(job.getId(), "Invalid arguments " + jobArgs.toString()); + } + + String propertyName = enumCleanupArgs.getPropertyName(); + List removedEnumKeys = enumCleanupArgs.getRemovedEnumKeys(); + String entityType = enumCleanupArgs.getEntityType(); + String customPropertyFQN = getCustomPropertyFQN(entityType, propertyName); + + List extensions = + daoCollection.entityExtensionDAO().getExtensionsByPrefixBatch(customPropertyFQN); + + List updatedExtensions = + extensions.stream() + .map( + extension -> { + List rowValues = + Optional.ofNullable(extension.getJson()) + .map( + json -> + JsonUtils.readValue(json, new TypeReference>() {})) + .orElseGet(ArrayList::new); + rowValues.removeAll(removedEnumKeys); + + return CollectionDAO.ExtensionWithIdAndSchemaObject.builder() + .id(extension.getId()) + .extension(extension.getExtension()) + .json(JsonUtils.pojoToJson(rowValues)) + .jsonschema(extension.getJsonschema()) + .build(); + }) + .collect(Collectors.toList()); + + LOG.debug("Updated extensions: {}", updatedExtensions); + daoCollection.entityExtensionDAO().bulkUpsertExtensions(updatedExtensions); + LOG.debug( + "Completed EnumCleanupHandler job for entityType: {}, propertyName: {}", + entityType, + propertyName); + } catch (Exception e) { + throw new BackgroundJobException( + job.getId(), "Failed to run EnumCleanupHandler job. Error:" + e.getMessage(), e); + } + } + + @Override + public boolean sendStatusToWebSocket() { + return true; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jobs/GenericBackgroundWorker.java b/openmetadata-service/src/main/java/org/openmetadata/service/jobs/GenericBackgroundWorker.java new file mode 100644 index 000000000000..684aa9a09b65 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jobs/GenericBackgroundWorker.java @@ -0,0 +1,124 @@ +package org.openmetadata.service.jobs; + +import io.dropwizard.lifecycle.Managed; +import java.util.Optional; +import lombok.extern.slf4j.Slf4j; +import org.openmetadata.schema.entity.teams.User; +import org.openmetadata.schema.jobs.BackgroundJob; +import org.openmetadata.service.Entity; +import org.openmetadata.service.socket.WebSocketManager; +import org.openmetadata.service.util.FullyQualifiedName; +import org.openmetadata.service.util.JsonUtils; + +@Slf4j +public class GenericBackgroundWorker implements Managed { + + private static final int INITIAL_BACKOFF_SECONDS = 1; + private static final int MAX_BACKOFF_SECONDS = 600; // 10 minutes + private static final int NO_JOB_SLEEP_SECONDS = 10; // Sleep if no jobs are available + + private final JobDAO jobDao; + private final JobHandlerRegistry handlerRegistry; + private volatile boolean running = true; + + public GenericBackgroundWorker(JobDAO jobDao, JobHandlerRegistry handlerRegistry) { + this.jobDao = jobDao; + this.handlerRegistry = handlerRegistry; + } + + @Override + public void start() throws Exception { + LOG.info("Starting background job worker"); + Thread workerThread = new Thread(this::runWorker, "background-job-worker"); + workerThread.setDaemon(true); + workerThread.start(); + } + + @Override + public void stop() throws Exception { + running = false; + } + + private void runWorker() { + int backoff = INITIAL_BACKOFF_SECONDS; + + while (running) { + try { + Optional jobOpt = jobDao.fetchPendingJob(); + if (jobOpt.isPresent()) { + processJob(jobOpt.get()); + backoff = INITIAL_BACKOFF_SECONDS; // Reset backoff after successful processing + } else { + sleep(NO_JOB_SLEEP_SECONDS); + } + } catch (BackgroundJobException e) { + long jobId = e.getJobId(); + jobDao.updateJobStatus(jobId, BackgroundJob.Status.FAILED); + LOG.error("Background Job {} failed. Error: {}", jobId, e.getMessage(), e); + } catch (Exception e) { + LOG.error("Unexpected error in background job worker: {}", e.getMessage(), e); + backoff = Math.min(backoff * 5, MAX_BACKOFF_SECONDS); // Exponential backoff with max limit + sleep(backoff); + } + } + + LOG.info("Background job worker terminated successfully."); + } + + private void processJob(BackgroundJob job) { + try { + jobDao.updateJobStatus(job.getId(), BackgroundJob.Status.RUNNING); + JobHandler handler = handlerRegistry.getHandler(job); + handler.runJob(job); + markJobAsCompleted(job); + + if (handler.sendStatusToWebSocket()) { + sendJobStatusUpdate(job); + } + } catch (BackgroundJobException e) { + markJobAsFailed(job, e); + sendJobStatusUpdate(job); + } + } + + private void markJobAsCompleted(BackgroundJob job) { + job.setStatus(BackgroundJob.Status.COMPLETED); + jobDao.updateJobStatus(job.getId(), BackgroundJob.Status.COMPLETED); + LOG.info( + "Background Job {} completed successfully. Type: {}, Method: {}", + job.getId(), + job.getJobType(), + job.getMethodName()); + } + + private void markJobAsFailed(BackgroundJob job, Exception e) { + job.setStatus(BackgroundJob.Status.FAILED); + jobDao.updateJobStatus(job.getId(), BackgroundJob.Status.FAILED); + LOG.error( + "Background Job {} failed. Type: {}, Method: {}. Error: {}", + job.getId(), + job.getJobType(), + job.getMethodName(), + e.getMessage(), + e); + } + + private void sleep(int seconds) { + try { + Thread.sleep(seconds * 1000L); + } catch (InterruptedException ie) { + LOG.error("Worker interrupted during sleep"); + Thread.currentThread().interrupt(); + } + } + + private void sendJobStatusUpdate(BackgroundJob job) { + String jsonMessage = JsonUtils.pojoToJson(job); + User user = + Entity.getCollectionDAO() + .userDAO() + .findEntityByName(FullyQualifiedName.quoteName(job.getCreatedBy())); + WebSocketManager.getInstance() + .sendToOne(user.getId(), WebSocketManager.BACKGROUND_JOB_CHANNEL, jsonMessage); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jobs/JobDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jobs/JobDAO.java new file mode 100644 index 000000000000..73bb9997133b --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jobs/JobDAO.java @@ -0,0 +1,113 @@ +package org.openmetadata.service.jobs; + +import static org.openmetadata.service.jdbi3.locator.ConnectionType.MYSQL; +import static org.openmetadata.service.jdbi3.locator.ConnectionType.POSTGRES; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Optional; +import lombok.extern.slf4j.Slf4j; +import org.jdbi.v3.core.mapper.RowMapper; +import org.jdbi.v3.core.statement.StatementContext; +import org.jdbi.v3.core.statement.StatementException; +import org.jdbi.v3.sqlobject.config.RegisterRowMapper; +import org.jdbi.v3.sqlobject.customizer.Bind; +import org.jdbi.v3.sqlobject.statement.GetGeneratedKeys; +import org.jdbi.v3.sqlobject.statement.SqlQuery; +import org.openmetadata.schema.jobs.BackgroundJob; +import org.openmetadata.service.jdbi3.locator.ConnectionAwareSqlUpdate; +import org.openmetadata.service.util.JsonUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public interface JobDAO { + + Logger LOG = LoggerFactory.getLogger(JobDAO.class); + + default long insertJob( + BackgroundJob.JobType jobType, JobHandler handler, String jobArgs, String createdBy) { + try { + JsonUtils.readTree(jobArgs); + } catch (Exception e) { + throw new IllegalArgumentException("jobArgs must be a valid JSON string"); + } + return insertJobInternal( + jobType.name(), handler.getClass().getSimpleName(), jobArgs, createdBy); + } + + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO background_jobs (jobType, methodName, jobArgs, createdBy) " + + "VALUES (:jobType, :methodName, :jobArgs, :createdBy)", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO background_jobs (jobType, methodName, jobArgs,createdBy) VALUES (:jobType, :methodName, :jobArgs::jsonb,:createdBy) ", + connectionType = POSTGRES) + @GetGeneratedKeys + long insertJobInternal( + @Bind("jobType") String jobType, + @Bind("methodName") String methodName, + @Bind("jobArgs") String jobArgs, + @Bind("createdBy") String createdBy); + + default Optional fetchPendingJob() throws BackgroundJobException { + return Optional.ofNullable(fetchPendingJobInternal()); + } + + @SqlQuery( + "SELECT id,jobType,methodName,jobArgs,status,createdAt,updatedAt,createdBy FROM background_jobs WHERE status = 'PENDING' ORDER BY createdAt LIMIT 1") + @RegisterRowMapper(BackgroundJobMapper.class) + BackgroundJob fetchPendingJobInternal() throws StatementException; + + @ConnectionAwareSqlUpdate( + value = + "UPDATE background_jobs SET status = :status, updatedAt = (UNIX_TIMESTAMP(NOW(3)) * 1000) WHERE id = :id", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "UPDATE background_jobs SET status = :status, updatedAt = (EXTRACT(EPOCH FROM NOW()) * 1000) WHERE id = :id", + connectionType = POSTGRES) + void updateJobStatusInternal(@Bind("id") long id, @Bind("status") String status); + + default void updateJobStatus(long id, BackgroundJob.Status status) { + updateJobStatusInternal(id, status.name()); + } + + @SqlQuery( + "SELECT id, jobType, methodName, jobArgs, status, createdAt, updatedAt, createdBy FROM background_jobs WHERE id = :id") + @RegisterRowMapper(BackgroundJobMapper.class) + BackgroundJob getJob(@Bind("id") long id) throws StatementException; + + default Optional fetchJobById(long id) { + return Optional.ofNullable(getJob(id)); + } + + @Slf4j + class BackgroundJobMapper implements RowMapper { + + @Override + public BackgroundJob map(ResultSet rs, StatementContext ctx) throws SQLException { + long jobId = rs.getLong("id"); + try { + BackgroundJob job = new BackgroundJob(); + job.setId(jobId); + job.setJobType(BackgroundJob.JobType.fromValue(rs.getString("jobType"))); + job.setMethodName(rs.getString("methodName")); + + String jobArgsJson = rs.getString("jobArgs"); + Object jobArgs = JsonUtils.readValue(jobArgsJson, Object.class); + job.setJobArgs(jobArgs); + + job.setStatus(BackgroundJob.Status.fromValue(rs.getString("status"))); + job.setCreatedAt(rs.getLong("createdAt")); + job.setUpdatedAt(rs.getLong("updatedAt")); + job.setCreatedBy(rs.getString("createdBy")); + + return job; + } catch (Exception e) { + throw new BackgroundJobException(jobId, "Failed to fetch/map pending job.", e); + } + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jobs/JobHandler.java b/openmetadata-service/src/main/java/org/openmetadata/service/jobs/JobHandler.java new file mode 100644 index 000000000000..2088ac786611 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jobs/JobHandler.java @@ -0,0 +1,9 @@ +package org.openmetadata.service.jobs; + +import org.openmetadata.schema.jobs.BackgroundJob; + +public interface JobHandler { + void runJob(BackgroundJob job) throws BackgroundJobException; + + boolean sendStatusToWebSocket(); +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jobs/JobHandlerRegistry.java b/openmetadata-service/src/main/java/org/openmetadata/service/jobs/JobHandlerRegistry.java new file mode 100644 index 000000000000..2b64249742cd --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jobs/JobHandlerRegistry.java @@ -0,0 +1,26 @@ +package org.openmetadata.service.jobs; + +import java.util.HashMap; +import java.util.Map; +import lombok.extern.slf4j.Slf4j; +import org.openmetadata.schema.jobs.BackgroundJob; + +@Slf4j +public class JobHandlerRegistry { + private final Map handlers = new HashMap<>(); + + public void register(String methodName, JobHandler handler) { + LOG.info("Registering background job handler for: {}", handler.getClass().getSimpleName()); + handlers.put(methodName, handler); + } + + public JobHandler getHandler(BackgroundJob job) { + String methodName = job.getMethodName(); + Long jobId = job.getId(); + JobHandler handler = handlers.get(methodName); + if (handler == null) { + throw new BackgroundJobException(jobId, "No handler registered for " + methodName); + } + return handler; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/mapper/EntityMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/mapper/EntityMapper.java new file mode 100644 index 000000000000..a3be330862e4 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/mapper/EntityMapper.java @@ -0,0 +1,45 @@ +package org.openmetadata.service.mapper; + +import static org.openmetadata.schema.type.Include.NON_DELETED; +import static org.openmetadata.service.jdbi3.EntityRepository.validateOwners; +import static org.openmetadata.service.jdbi3.EntityRepository.validateReviewers; +import static org.openmetadata.service.util.EntityUtil.getEntityReferences; + +import java.util.List; +import java.util.UUID; +import org.openmetadata.common.utils.CommonUtil; +import org.openmetadata.schema.CreateEntity; +import org.openmetadata.schema.EntityInterface; +import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.service.Entity; + +public interface EntityMapper { + T createToEntity(C create, String user); + + default T copy(T entity, CreateEntity request, String updatedBy) { + List owners = validateOwners(request.getOwners()); + EntityReference domain = validateDomain(request.getDomain()); + validateReviewers(request.getReviewers()); + entity.setId(UUID.randomUUID()); + entity.setName(request.getName()); + entity.setDisplayName(request.getDisplayName()); + entity.setDescription(request.getDescription()); + entity.setOwners(owners); + entity.setDomain(domain); + entity.setTags(request.getTags()); + entity.setDataProducts(getEntityReferences(Entity.DATA_PRODUCT, request.getDataProducts())); + entity.setLifeCycle(request.getLifeCycle()); + entity.setExtension(request.getExtension()); + entity.setUpdatedBy(updatedBy); + entity.setUpdatedAt(System.currentTimeMillis()); + entity.setReviewers(request.getReviewers()); + return entity; + } + + default EntityReference validateDomain(String domainFqn) { + if (CommonUtil.nullOrEmpty(domainFqn)) { + return null; + } + return Entity.getEntityReferenceByName(Entity.DOMAIN, domainFqn, NON_DELETED); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/mapper/EntityTimeSeriesMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/mapper/EntityTimeSeriesMapper.java new file mode 100644 index 000000000000..242cb8929698 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/mapper/EntityTimeSeriesMapper.java @@ -0,0 +1,7 @@ +package org.openmetadata.service.mapper; + +import org.openmetadata.schema.EntityTimeSeriesInterface; + +public interface EntityTimeSeriesMapper { + T createToEntity(C create, String user); +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/monitoring/EventMonitorConfiguration.java b/openmetadata-service/src/main/java/org/openmetadata/service/monitoring/EventMonitorConfiguration.java index 4e3f17b84c7b..552d34db3daf 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/monitoring/EventMonitorConfiguration.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/monitoring/EventMonitorConfiguration.java @@ -31,4 +31,6 @@ public class EventMonitorConfiguration { private String[] pathPattern; private double[] latency; + + private int servicesHealthCheckInterval = 300; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/EntityResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/EntityResource.java index d2044d444205..74733269e64e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/EntityResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/EntityResource.java @@ -144,7 +144,7 @@ public ResultList listInternal( Fields fields = getFields(fieldsParam); OperationContext listOperationContext = new OperationContext(entityType, getViewOperations(fields)); - + ResourceContext resourceContext = filter.getResourceContext(entityType); return listInternal( uriInfo, securityContext, @@ -154,7 +154,7 @@ public ResultList listInternal( before, after, listOperationContext, - getResourceContext()); + resourceContext); } public ResultList listInternal( @@ -361,7 +361,6 @@ public Response delete( authorizer.authorize(securityContext, operationContext, getResourceContextById(id)); DeleteResponse response = repository.delete(securityContext.getUserPrincipal().getName(), id, recursive, hardDelete); - repository.deleteFromSearch(response.entity(), response.changeType()); if (hardDelete) { limits.invalidateCache(entityType); } @@ -380,7 +379,6 @@ public Response deleteByName( DeleteResponse response = repository.deleteByName( securityContext.getUserPrincipal().getName(), name, recursive, hardDelete); - repository.deleteFromSearch(response.entity(), response.changeType()); addHref(uriInfo, response.entity()); return response.toResponse(); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/analytics/WebAnalyticEventMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/analytics/WebAnalyticEventMapper.java new file mode 100644 index 000000000000..b6880919dd87 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/analytics/WebAnalyticEventMapper.java @@ -0,0 +1,17 @@ +package org.openmetadata.service.resources.analytics; + +import org.openmetadata.schema.analytics.WebAnalyticEvent; +import org.openmetadata.schema.api.tests.CreateWebAnalyticEvent; +import org.openmetadata.service.mapper.EntityMapper; + +public class WebAnalyticEventMapper + implements EntityMapper { + @Override + public WebAnalyticEvent createToEntity(CreateWebAnalyticEvent create, String user) { + return copy(new WebAnalyticEvent(), create, user) + .withName(create.getName()) + .withDisplayName(create.getDisplayName()) + .withDescription(create.getDescription()) + .withEventType(create.getEventType()); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/analytics/WebAnalyticEventResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/analytics/WebAnalyticEventResource.java index 5ce147ef770e..d5ec469b4262 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/analytics/WebAnalyticEventResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/analytics/WebAnalyticEventResource.java @@ -70,6 +70,7 @@ public class WebAnalyticEventResource public static final String COLLECTION_PATH = WebAnalyticEventRepository.COLLECTION_PATH; static final String FIELDS = "owners"; private static final Pattern HTML_PATTERN = Pattern.compile(".*\\<[^>]+>.*", Pattern.DOTALL); + private final WebAnalyticEventMapper mapper = new WebAnalyticEventMapper(); public WebAnalyticEventResource(Authorizer authorizer, Limits limits) { super(Entity.WEB_ANALYTIC_EVENT, authorizer, limits); @@ -169,7 +170,7 @@ public Response create( @Context SecurityContext securityContext, @Valid CreateWebAnalyticEvent create) { WebAnalyticEvent webAnalyticEvent = - getWebAnalyticEvent(create, securityContext.getUserPrincipal().getName()); + mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return create(uriInfo, securityContext, webAnalyticEvent); } @@ -192,7 +193,7 @@ public Response createOrUpdate( @Context SecurityContext securityContext, @Valid CreateWebAnalyticEvent create) { WebAnalyticEvent webAnalyticEvent = - getWebAnalyticEvent(create, securityContext.getUserPrincipal().getName()); + mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return createOrUpdate(uriInfo, securityContext, webAnalyticEvent); } @@ -554,15 +555,6 @@ public ResultList listWebAnalyticEventData( return repository.getWebAnalyticEventData(eventType, startTs, endTs); } - private WebAnalyticEvent getWebAnalyticEvent(CreateWebAnalyticEvent create, String user) { - return repository - .copy(new WebAnalyticEvent(), create, user) - .withName(create.getName()) - .withDisplayName(create.getDisplayName()) - .withDescription(create.getDescription()) - .withEventType(create.getEventType()); - } - public static WebAnalyticEventData sanitizeWebAnalyticEventData( WebAnalyticEventData webAnalyticEventDataInput) { Object inputData = webAnalyticEventDataInput.getEventData(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apis/APICollectionMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apis/APICollectionMapper.java new file mode 100644 index 000000000000..60d3e9c44e77 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apis/APICollectionMapper.java @@ -0,0 +1,20 @@ +package org.openmetadata.service.resources.apis; + +import static org.openmetadata.service.util.EntityUtil.getEntityReference; +import static org.openmetadata.service.util.EntityUtil.getEntityReferences; + +import org.openmetadata.schema.api.data.CreateAPICollection; +import org.openmetadata.schema.entity.data.APICollection; +import org.openmetadata.service.Entity; +import org.openmetadata.service.mapper.EntityMapper; + +public class APICollectionMapper implements EntityMapper { + @Override + public APICollection createToEntity(CreateAPICollection create, String user) { + return copy(new APICollection(), create, user) + .withService(getEntityReference(Entity.API_SERVICE, create.getService())) + .withEndpointURL(create.getEndpointURL()) + .withApiEndpoints(getEntityReferences(Entity.API_ENDPOINT, create.getApiEndpoints())) + .withSourceHash(create.getSourceHash()); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apis/APICollectionResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apis/APICollectionResource.java index 0f2581f1b0cc..b73d02c0c25d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apis/APICollectionResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apis/APICollectionResource.java @@ -73,6 +73,7 @@ @Collection(name = "apiCollections") public class APICollectionResource extends EntityResource { public static final String COLLECTION_PATH = "v1/apiCollections/"; + private final APICollectionMapper mapper = new APICollectionMapper(); static final String FIELDS = "owners,apiEndpoints,tags,extension,domain,sourceHash"; @Override @@ -310,7 +311,7 @@ public Response create( @Context SecurityContext securityContext, @Valid CreateAPICollection create) { APICollection apiCollection = - getAPICollection(create, securityContext.getUserPrincipal().getName()); + mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return create(uriInfo, securityContext, apiCollection); } @@ -392,7 +393,7 @@ public Response createOrUpdate( @Context SecurityContext securityContext, @Valid CreateAPICollection create) { APICollection apiCollection = - getAPICollection(create, securityContext.getUserPrincipal().getName()); + mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return createOrUpdate(uriInfo, securityContext, apiCollection); } @@ -510,13 +511,4 @@ public Response restore( @Valid RestoreEntity restore) { return restoreEntity(uriInfo, securityContext, restore.getId()); } - - private APICollection getAPICollection(CreateAPICollection create, String user) { - return repository - .copy(new APICollection(), create, user) - .withService(getEntityReference(Entity.API_SERVICE, create.getService())) - .withEndpointURL(create.getEndpointURL()) - .withApiEndpoints(getEntityReferences(Entity.API_ENDPOINT, create.getApiEndpoints())) - .withSourceHash(create.getSourceHash()); - } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apis/APIEndpointMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apis/APIEndpointMapper.java new file mode 100644 index 000000000000..f197475efcf0 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apis/APIEndpointMapper.java @@ -0,0 +1,21 @@ +package org.openmetadata.service.resources.apis; + +import static org.openmetadata.service.util.EntityUtil.getEntityReference; + +import org.openmetadata.schema.api.data.CreateAPIEndpoint; +import org.openmetadata.schema.entity.data.APIEndpoint; +import org.openmetadata.service.Entity; +import org.openmetadata.service.mapper.EntityMapper; + +public class APIEndpointMapper implements EntityMapper { + @Override + public APIEndpoint createToEntity(CreateAPIEndpoint create, String user) { + return copy(new APIEndpoint(), create, user) + .withApiCollection(getEntityReference(Entity.API_COLLCECTION, create.getApiCollection())) + .withRequestMethod(create.getRequestMethod()) + .withEndpointURL(create.getEndpointURL()) + .withRequestSchema(create.getRequestSchema()) + .withResponseSchema(create.getResponseSchema()) + .withSourceHash(create.getSourceHash()); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apis/APIEndpointResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apis/APIEndpointResource.java index 5060606c2a2a..40a853aec442 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apis/APIEndpointResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apis/APIEndpointResource.java @@ -69,6 +69,8 @@ @Collection(name = "apiEndpoints") public class APIEndpointResource extends EntityResource { public static final String COLLECTION_PATH = "v1/apiEndpoints/"; + private final APIEndpointMapper mapper = new APIEndpointMapper(); + static final String FIELDS = "owners,followers,tags,extension,domain,dataProducts,sourceHash"; @Override @@ -304,7 +306,8 @@ public Response create( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateAPIEndpoint create) { - APIEndpoint apiEndpoint = getAPIEndpoint(create, securityContext.getUserPrincipal().getName()); + APIEndpoint apiEndpoint = + mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return create(uriInfo, securityContext, apiEndpoint); } @@ -385,7 +388,8 @@ public Response createOrUpdate( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateAPIEndpoint create) { - APIEndpoint apiEndpoint = getAPIEndpoint(create, securityContext.getUserPrincipal().getName()); + APIEndpoint apiEndpoint = + mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return createOrUpdate(uriInfo, securityContext, apiEndpoint); } @@ -552,15 +556,4 @@ public Response restore( @Valid RestoreEntity restore) { return restoreEntity(uriInfo, securityContext, restore.getId()); } - - private APIEndpoint getAPIEndpoint(CreateAPIEndpoint create, String user) { - return repository - .copy(new APIEndpoint(), create, user) - .withApiCollection(getEntityReference(Entity.API_COLLCECTION, create.getApiCollection())) - .withRequestMethod(create.getRequestMethod()) - .withEndpointURL(create.getEndpointURL()) - .withRequestSchema(create.getRequestSchema()) - .withResponseSchema(create.getResponseSchema()) - .withSourceHash(create.getSourceHash()); - } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMapper.java new file mode 100644 index 000000000000..62b941626361 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMapper.java @@ -0,0 +1,72 @@ +package org.openmetadata.service.resources.apps; + +import static org.openmetadata.service.Entity.BOT; +import static org.openmetadata.service.jdbi3.EntityRepository.validateOwners; + +import java.util.List; +import java.util.UUID; +import org.openmetadata.common.utils.CommonUtil; +import org.openmetadata.schema.entity.app.App; +import org.openmetadata.schema.entity.app.AppMarketPlaceDefinition; +import org.openmetadata.schema.entity.app.CreateApp; +import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.type.Include; +import org.openmetadata.service.Entity; +import org.openmetadata.service.jdbi3.AppMarketPlaceRepository; +import org.openmetadata.service.jdbi3.AppRepository; +import org.openmetadata.service.mapper.EntityMapper; +import org.openmetadata.service.util.EntityUtil; + +public class AppMapper implements EntityMapper { + @Override + public App createToEntity(CreateApp createAppRequest, String updatedBy) { + AppMarketPlaceRepository appMarketPlaceRepository = + (AppMarketPlaceRepository) Entity.getEntityRepository(Entity.APP_MARKET_PLACE_DEF); + AppMarketPlaceDefinition marketPlaceDefinition = + appMarketPlaceRepository.getByName( + null, + createAppRequest.getName(), + new EntityUtil.Fields(appMarketPlaceRepository.getAllowedFields())); + List owners = validateOwners(createAppRequest.getOwners()); + App app = + new App() + .withId(UUID.randomUUID()) + .withName(marketPlaceDefinition.getName()) + .withDisplayName(createAppRequest.getDisplayName()) + .withDescription(createAppRequest.getDescription()) + .withOwners(owners) + .withUpdatedBy(updatedBy) + .withUpdatedAt(System.currentTimeMillis()) + .withDeveloper(marketPlaceDefinition.getDeveloper()) + .withDeveloperUrl(marketPlaceDefinition.getDeveloperUrl()) + .withPrivacyPolicyUrl(marketPlaceDefinition.getPrivacyPolicyUrl()) + .withSupportEmail(marketPlaceDefinition.getSupportEmail()) + .withClassName(marketPlaceDefinition.getClassName()) + .withAppType(marketPlaceDefinition.getAppType()) + .withScheduleType(marketPlaceDefinition.getScheduleType()) + .withAppConfiguration(createAppRequest.getAppConfiguration()) + .withRuntime(marketPlaceDefinition.getRuntime()) + .withPermission(marketPlaceDefinition.getPermission()) + .withAppSchedule(createAppRequest.getAppSchedule()) + .withAppLogoUrl(marketPlaceDefinition.getAppLogoUrl()) + .withAppScreenshots(marketPlaceDefinition.getAppScreenshots()) + .withFeatures(marketPlaceDefinition.getFeatures()) + .withSourcePythonClass(marketPlaceDefinition.getSourcePythonClass()) + .withAllowConfiguration(marketPlaceDefinition.getAllowConfiguration()) + .withSystem(marketPlaceDefinition.getSystem()) + .withSupportsInterrupt(marketPlaceDefinition.getSupportsInterrupt()); + + // validate Bot if provided + validateAndAddBot(app, createAppRequest.getBot()); + return app; + } + + private void validateAndAddBot(App app, String botName) { + AppRepository appRepository = (AppRepository) Entity.getEntityRepository(Entity.APPLICATION); + if (!CommonUtil.nullOrEmpty(botName)) { + app.setBot(Entity.getEntityReferenceByName(BOT, botName, Include.NON_DELETED)); + } else { + app.setBot(appRepository.createNewAppBot(app)); + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMarketPlaceMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMarketPlaceMapper.java new file mode 100644 index 000000000000..532cf057b101 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMarketPlaceMapper.java @@ -0,0 +1,68 @@ +package org.openmetadata.service.resources.apps; + +import javax.ws.rs.BadRequestException; +import org.openmetadata.schema.entity.app.AppMarketPlaceDefinition; +import org.openmetadata.schema.entity.app.AppType; +import org.openmetadata.schema.entity.app.CreateAppMarketPlaceDefinitionReq; +import org.openmetadata.schema.entity.services.ingestionPipelines.PipelineServiceClientResponse; +import org.openmetadata.sdk.PipelineServiceClientInterface; +import org.openmetadata.service.mapper.EntityMapper; +import org.openmetadata.service.util.JsonUtils; + +public class AppMarketPlaceMapper + implements EntityMapper { + private PipelineServiceClientInterface pipelineServiceClient; + + public AppMarketPlaceMapper(PipelineServiceClientInterface pipelineServiceClient) { + this.pipelineServiceClient = pipelineServiceClient; + } + + @Override + public AppMarketPlaceDefinition createToEntity( + CreateAppMarketPlaceDefinitionReq create, String user) { + AppMarketPlaceDefinition app = + copy(new AppMarketPlaceDefinition(), create, user) + .withDeveloper(create.getDeveloper()) + .withDeveloperUrl(create.getDeveloperUrl()) + .withSupportEmail(create.getSupportEmail()) + .withPrivacyPolicyUrl(create.getPrivacyPolicyUrl()) + .withClassName(create.getClassName()) + .withAppType(create.getAppType()) + .withScheduleType(create.getScheduleType()) + .withRuntime(create.getRuntime()) + .withAppConfiguration(create.getAppConfiguration()) + .withPermission(create.getPermission()) + .withAppLogoUrl(create.getAppLogoUrl()) + .withAppScreenshots(create.getAppScreenshots()) + .withFeatures(create.getFeatures()) + .withSourcePythonClass(create.getSourcePythonClass()) + .withAllowConfiguration(create.getAllowConfiguration()) + .withSystem(create.getSystem()) + .withSupportsInterrupt(create.getSupportsInterrupt()); + + // Validate App + validateApplication(app); + return app; + } + + private void validateApplication(AppMarketPlaceDefinition app) { + // Check if the className Exists in classPath + if (app.getAppType().equals(AppType.Internal)) { + // Check class name exists + try { + Class.forName(app.getClassName()); + } catch (ClassNotFoundException e) { + throw new BadRequestException( + "Application Cannot be registered, because the classname cannot be found on the Classpath."); + } + } else { + PipelineServiceClientResponse response = pipelineServiceClient.validateAppRegistration(app); + if (response.getCode() != 200) { + throw new BadRequestException( + String.format( + "Application Cannot be registered, Error from Pipeline Service Client. Status Code : %s , Reponse : %s", + response.getCode(), JsonUtils.pojoToJson(response))); + } + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMarketPlaceResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMarketPlaceResource.java index 11540d291828..0400e2dc9f21 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMarketPlaceResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMarketPlaceResource.java @@ -1,8 +1,5 @@ package org.openmetadata.service.resources.apps; -import static org.openmetadata.service.Entity.APPLICATION; -import static org.openmetadata.service.jdbi3.EntityRepository.getEntitiesFromSeedData; - import io.swagger.v3.oas.annotations.ExternalDocumentation; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; @@ -12,13 +9,11 @@ import io.swagger.v3.oas.annotations.parameters.RequestBody; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.tags.Tag; -import java.util.List; import java.util.UUID; import javax.json.JsonPatch; import javax.validation.Valid; import javax.validation.constraints.Max; import javax.validation.constraints.Min; -import javax.ws.rs.BadRequestException; import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; import javax.ws.rs.DefaultValue; @@ -39,9 +34,7 @@ import org.openmetadata.schema.api.data.RestoreEntity; import org.openmetadata.schema.entity.app.App; import org.openmetadata.schema.entity.app.AppMarketPlaceDefinition; -import org.openmetadata.schema.entity.app.AppType; import org.openmetadata.schema.entity.app.CreateAppMarketPlaceDefinitionReq; -import org.openmetadata.schema.entity.services.ingestionPipelines.PipelineServiceClientResponse; import org.openmetadata.schema.type.EntityHistory; import org.openmetadata.schema.type.Include; import org.openmetadata.sdk.PipelineServiceClientInterface; @@ -55,7 +48,7 @@ import org.openmetadata.service.resources.Collection; import org.openmetadata.service.resources.EntityResource; import org.openmetadata.service.security.Authorizer; -import org.openmetadata.service.util.JsonUtils; +import org.openmetadata.service.util.AppMarketPlaceUtil; import org.openmetadata.service.util.ResultList; @Path("/v1/apps/marketplace") @@ -69,29 +62,17 @@ public class AppMarketPlaceResource extends EntityResource { public static final String COLLECTION_PATH = "/v1/apps/marketplace/"; - private PipelineServiceClientInterface pipelineServiceClient; - + private AppMarketPlaceMapper mapper; static final String FIELDS = "owners,tags"; @Override public void initialize(OpenMetadataApplicationConfig config) { try { - this.pipelineServiceClient = + PipelineServiceClientInterface pipelineServiceClient = PipelineServiceClientFactory.createPipelineServiceClient( config.getPipelineServiceClientConfiguration()); - - // Initialize Default Installed Applications - List createAppMarketPlaceDefinitionReqs = - getEntitiesFromSeedData( - APPLICATION, - String.format(".*json/data/%s/.*\\.json$", entityType), - CreateAppMarketPlaceDefinitionReq.class); - for (CreateAppMarketPlaceDefinitionReq definitionReq : createAppMarketPlaceDefinitionReqs) { - AppMarketPlaceDefinition definition = getApplicationDefinition(definitionReq, "admin"); - // Update Fully Qualified Name - repository.setFullyQualifiedName(definition); - this.repository.createOrUpdate(null, definition); - } + mapper = new AppMarketPlaceMapper(pipelineServiceClient); + AppMarketPlaceUtil.createAppMarketPlaceDefinitions(repository, mapper); } catch (Exception ex) { LOG.error("Failed in initializing App MarketPlace Resource", ex); } @@ -309,7 +290,7 @@ public Response create( @Context SecurityContext securityContext, @Valid CreateAppMarketPlaceDefinitionReq create) { AppMarketPlaceDefinition app = - getApplicationDefinition(create, securityContext.getUserPrincipal().getName()); + mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return create(uriInfo, securityContext, app); } @@ -389,7 +370,7 @@ public Response createOrUpdate( @Context SecurityContext securityContext, @Valid CreateAppMarketPlaceDefinitionReq create) { AppMarketPlaceDefinition app = - getApplicationDefinition(create, securityContext.getUserPrincipal().getName()); + mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return createOrUpdate(uriInfo, securityContext, app); } @@ -459,53 +440,4 @@ public Response restoreApp( @Valid RestoreEntity restore) { return restoreEntity(uriInfo, securityContext, restore.getId()); } - - private AppMarketPlaceDefinition getApplicationDefinition( - CreateAppMarketPlaceDefinitionReq create, String updatedBy) { - AppMarketPlaceDefinition app = - repository - .copy(new AppMarketPlaceDefinition(), create, updatedBy) - .withDeveloper(create.getDeveloper()) - .withDeveloperUrl(create.getDeveloperUrl()) - .withSupportEmail(create.getSupportEmail()) - .withPrivacyPolicyUrl(create.getPrivacyPolicyUrl()) - .withClassName(create.getClassName()) - .withAppType(create.getAppType()) - .withScheduleType(create.getScheduleType()) - .withRuntime(create.getRuntime()) - .withAppConfiguration(create.getAppConfiguration()) - .withPermission(create.getPermission()) - .withAppLogoUrl(create.getAppLogoUrl()) - .withAppScreenshots(create.getAppScreenshots()) - .withFeatures(create.getFeatures()) - .withSourcePythonClass(create.getSourcePythonClass()) - .withAllowConfiguration(create.getAllowConfiguration()) - .withSystem(create.getSystem()) - .withSupportsInterrupt(create.getSupportsInterrupt()); - - // Validate App - validateApplication(app); - return app; - } - - private void validateApplication(AppMarketPlaceDefinition app) { - // Check if the className Exists in classPath - if (app.getAppType().equals(AppType.Internal)) { - // Check class name exists - try { - Class.forName(app.getClassName()); - } catch (ClassNotFoundException e) { - throw new BadRequestException( - "Application Cannot be registered, because the classname cannot be found on the Classpath."); - } - } else { - PipelineServiceClientResponse response = pipelineServiceClient.validateAppRegistration(app); - if (response.getCode() != 200) { - throw new BadRequestException( - String.format( - "Application Cannot be registered, Error from Pipeline Service Client. Status Code : %s , Reponse : %s", - response.getCode(), JsonUtils.pojoToJson(response))); - } - } - } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java index 21a88f23d942..3da3526b6443 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java @@ -3,7 +3,6 @@ import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; import static org.openmetadata.schema.type.Include.ALL; import static org.openmetadata.service.Entity.APPLICATION; -import static org.openmetadata.service.Entity.BOT; import static org.openmetadata.service.Entity.FIELD_OWNERS; import static org.openmetadata.service.jdbi3.EntityRepository.getEntitiesFromSeedData; @@ -43,12 +42,10 @@ import javax.ws.rs.core.SecurityContext; import javax.ws.rs.core.UriInfo; import lombok.extern.slf4j.Slf4j; -import org.openmetadata.common.utils.CommonUtil; import org.openmetadata.schema.ServiceEntityInterface; import org.openmetadata.schema.api.data.RestoreEntity; import org.openmetadata.schema.entity.app.App; import org.openmetadata.schema.entity.app.AppExtension; -import org.openmetadata.schema.entity.app.AppMarketPlaceDefinition; import org.openmetadata.schema.entity.app.AppRunRecord; import org.openmetadata.schema.entity.app.AppType; import org.openmetadata.schema.entity.app.CreateApp; @@ -105,6 +102,7 @@ public class AppResource extends EntityResource { public static final List SCHEDULED_TYPES = List.of(ScheduleType.Scheduled, ScheduleType.ScheduledOrManual, ScheduleType.NoSchedule); public static final String SLACK_APPLICATION = "SlackApplication"; + private final AppMapper mapper = new AppMapper(); @Override public void initialize(OpenMetadataApplicationConfig config) { @@ -133,18 +131,10 @@ private void loadDefaultApplications(List defaultAppCreateRequests) { // Get Create App Requests for (CreateApp createApp : defaultAppCreateRequests) { try { - AppMarketPlaceDefinition definition = - repository - .getMarketPlace() - .getByName( - null, - createApp.getName(), - new EntityUtil.Fields(repository.getMarketPlace().getAllowedFields())); App app = getAppForInit(createApp.getName()); if (app == null) { app = - getApplication(definition, createApp, "admin") - .withFullyQualifiedName(createApp.getName()); + mapper.createToEntity(createApp, "admin").withFullyQualifiedName(createApp.getName()); repository.initializeEntity(app); } @@ -259,7 +249,7 @@ public ResultList list( mediaType = "application/json", schema = @Schema(implementation = AppRunList.class))) }) - public Response listAppRuns( + public ResultList listAppRuns( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Parameter(description = "Name of the App", schema = @Schema(type = "string")) @@ -289,9 +279,7 @@ public Response listAppRuns( Long endTs) { App installation = repository.getByName(uriInfo, name, repository.getFields("id,pipelines")); if (installation.getAppType().equals(AppType.Internal)) { - return Response.status(Response.Status.OK) - .entity(repository.listAppRuns(installation, limitParam, offset)) - .build(); + return repository.listAppRuns(installation, limitParam, offset); } if (!installation.getPipelines().isEmpty()) { EntityReference pipelineRef = installation.getPipelines().get(0); @@ -300,13 +288,27 @@ public Response listAppRuns( IngestionPipeline ingestionPipeline = ingestionPipelineRepository.get( uriInfo, pipelineRef.getId(), ingestionPipelineRepository.getFields(FIELD_OWNERS)); - return Response.ok( - ingestionPipelineRepository.listPipelineStatus( - ingestionPipeline.getFullyQualifiedName(), startTs, endTs), - MediaType.APPLICATION_JSON_TYPE) - .build(); + return ingestionPipelineRepository + .listPipelineStatus(ingestionPipeline.getFullyQualifiedName(), startTs, endTs) + .map(pipelineStatus -> convertPipelineStatus(installation, pipelineStatus)); } - throw new IllegalArgumentException("App does not have an associated pipeline."); + throw new IllegalArgumentException("App does not have a scheduled deployment"); + } + + private static AppRunRecord convertPipelineStatus(App app, PipelineStatus pipelineStatus) { + return new AppRunRecord() + .withAppId(app.getId()) + .withAppName(app.getName()) + .withExecutionTime(pipelineStatus.getStartDate()) + .withEndTime(pipelineStatus.getEndDate()) + .withStatus( + switch (pipelineStatus.getPipelineState()) { + case QUEUED -> AppRunRecord.Status.PENDING; + case SUCCESS -> AppRunRecord.Status.SUCCESS; + case FAILED, PARTIAL_SUCCESS -> AppRunRecord.Status.FAILED; + case RUNNING -> AppRunRecord.Status.RUNNING; + }) + .withConfig(pipelineStatus.getConfig()); } @GET @@ -621,19 +623,11 @@ public App getVersion( }) public Response create( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateApp create) { - - AppMarketPlaceDefinition definition = - repository - .getMarketPlace() - .getByName( - uriInfo, - create.getName(), - new EntityUtil.Fields(repository.getMarketPlace().getAllowedFields())); - App app = getApplication(definition, create, securityContext.getUserPrincipal().getName()); + App app = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); limits.enforceLimits( securityContext, getResourceContext(), - new OperationContext(Entity.APPLICATION, MetadataOperation.CREATE)); + new OperationContext(APPLICATION, MetadataOperation.CREATE)); if (SCHEDULED_TYPES.contains(app.getScheduleType())) { ApplicationHandler.getInstance() .installApplication(app, Entity.getCollectionDAO(), searchRepository); @@ -749,14 +743,7 @@ public Response patchApplication( public Response createOrUpdate( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateApp create) throws SchedulerException { - AppMarketPlaceDefinition definition = - repository - .getMarketPlace() - .getByName( - uriInfo, - create.getName(), - new EntityUtil.Fields(repository.getMarketPlace().getAllowedFields())); - App app = getApplication(definition, create, securityContext.getUserPrincipal().getName()); + App app = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); AppScheduler.getInstance().deleteScheduledApplication(app); if (SCHEDULED_TYPES.contains(app.getScheduleType())) { ApplicationHandler.getInstance() @@ -1100,52 +1087,6 @@ private void decryptOrNullify( } } - private App getApplication( - AppMarketPlaceDefinition marketPlaceDefinition, - CreateApp createAppRequest, - String updatedBy) { - List owners = repository.validateOwners(createAppRequest.getOwners()); - App app = - new App() - .withId(UUID.randomUUID()) - .withName(marketPlaceDefinition.getName()) - .withDisplayName(createAppRequest.getDisplayName()) - .withDescription(createAppRequest.getDescription()) - .withOwners(owners) - .withUpdatedBy(updatedBy) - .withUpdatedAt(System.currentTimeMillis()) - .withDeveloper(marketPlaceDefinition.getDeveloper()) - .withDeveloperUrl(marketPlaceDefinition.getDeveloperUrl()) - .withPrivacyPolicyUrl(marketPlaceDefinition.getPrivacyPolicyUrl()) - .withSupportEmail(marketPlaceDefinition.getSupportEmail()) - .withClassName(marketPlaceDefinition.getClassName()) - .withAppType(marketPlaceDefinition.getAppType()) - .withScheduleType(marketPlaceDefinition.getScheduleType()) - .withAppConfiguration(createAppRequest.getAppConfiguration()) - .withRuntime(marketPlaceDefinition.getRuntime()) - .withPermission(marketPlaceDefinition.getPermission()) - .withAppSchedule(createAppRequest.getAppSchedule()) - .withAppLogoUrl(marketPlaceDefinition.getAppLogoUrl()) - .withAppScreenshots(marketPlaceDefinition.getAppScreenshots()) - .withFeatures(marketPlaceDefinition.getFeatures()) - .withSourcePythonClass(marketPlaceDefinition.getSourcePythonClass()) - .withAllowConfiguration(marketPlaceDefinition.getAllowConfiguration()) - .withSystem(marketPlaceDefinition.getSystem()) - .withSupportsInterrupt(marketPlaceDefinition.getSupportsInterrupt()); - - // validate Bot if provided - validateAndAddBot(app, createAppRequest.getBot()); - return app; - } - - private void validateAndAddBot(App app, String botName) { - if (!CommonUtil.nullOrEmpty(botName)) { - app.setBot(Entity.getEntityReferenceByName(BOT, botName, Include.NON_DELETED)); - } else { - app.setBot(repository.createNewAppBot(app)); - } - } - private IngestionPipeline getIngestionPipeline( UriInfo uriInfo, SecurityContext securityContext, App app) { EntityReference pipelineRef = app.getPipelines().get(0); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/automations/WorkflowMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/automations/WorkflowMapper.java new file mode 100644 index 000000000000..2aa3af07e0d6 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/automations/WorkflowMapper.java @@ -0,0 +1,31 @@ +package org.openmetadata.service.resources.automations; + +import org.openmetadata.schema.entity.automations.CreateWorkflow; +import org.openmetadata.schema.entity.automations.Workflow; +import org.openmetadata.schema.services.connections.metadata.OpenMetadataConnection; +import org.openmetadata.service.OpenMetadataApplicationConfig; +import org.openmetadata.service.mapper.EntityMapper; +import org.openmetadata.service.util.OpenMetadataConnectionBuilder; + +public class WorkflowMapper implements EntityMapper { + private final OpenMetadataApplicationConfig openMetadataApplicationConfig; + + public WorkflowMapper(OpenMetadataApplicationConfig config) { + this.openMetadataApplicationConfig = config; + } + + @Override + public Workflow createToEntity(CreateWorkflow create, String user) { + OpenMetadataConnection openMetadataServerConnection = + new OpenMetadataConnectionBuilder(openMetadataApplicationConfig).build(); + return copy(new Workflow(), create, user) + .withDescription(create.getDescription()) + .withRequest(create.getRequest()) + .withWorkflowType(create.getWorkflowType()) + .withDisplayName(create.getDisplayName()) + .withResponse(create.getResponse()) + .withStatus(create.getStatus()) + .withOpenMetadataServerConnection(openMetadataServerConnection) + .withName(create.getName()); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/automations/WorkflowResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/automations/WorkflowResource.java index 0ec7efa9df40..01f22137ccad 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/automations/WorkflowResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/automations/WorkflowResource.java @@ -83,7 +83,7 @@ public class WorkflowResource extends EntityResource { public static final String COLLECTION_PATH = "/v1/automations/workflows"; static final String FIELDS = "owners"; - + private WorkflowMapper mapper; private PipelineServiceClientInterface pipelineServiceClient; private OpenMetadataApplicationConfig openMetadataApplicationConfig; @@ -94,7 +94,7 @@ public WorkflowResource(Authorizer authorizer, Limits limits) { @Override public void initialize(OpenMetadataApplicationConfig config) { this.openMetadataApplicationConfig = config; - + this.mapper = new WorkflowMapper(config); this.pipelineServiceClient = PipelineServiceClientFactory.createPipelineServiceClient( config.getPipelineServiceClientConfiguration()); @@ -331,7 +331,7 @@ public Response create( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateWorkflow create) { - Workflow workflow = getWorkflow(create, securityContext.getUserPrincipal().getName()); + Workflow workflow = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); Response response = create(uriInfo, securityContext, unmask(workflow)); return Response.fromResponse(response) .entity(decryptOrNullify(securityContext, (Workflow) response.getEntity())) @@ -452,7 +452,7 @@ public Response createOrUpdate( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateWorkflow create) { - Workflow workflow = getWorkflow(create, securityContext.getUserPrincipal().getName()); + Workflow workflow = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); workflow = unmask(workflow); Response response = createOrUpdate(uriInfo, securityContext, workflow); return Response.fromResponse(response) @@ -539,21 +539,6 @@ public Response restoreWorkflow( .build(); } - private Workflow getWorkflow(CreateWorkflow create, String user) { - OpenMetadataConnection openMetadataServerConnection = - new OpenMetadataConnectionBuilder(openMetadataApplicationConfig).build(); - return repository - .copy(new Workflow(), create, user) - .withDescription(create.getDescription()) - .withRequest(create.getRequest()) - .withWorkflowType(create.getWorkflowType()) - .withDisplayName(create.getDisplayName()) - .withResponse(create.getResponse()) - .withStatus(create.getStatus()) - .withOpenMetadataServerConnection(openMetadataServerConnection) - .withName(create.getName()); - } - private Workflow unmask(Workflow workflow) { repository.setFullyQualifiedName(workflow); Workflow originalWorkflow; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/bots/BotMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/bots/BotMapper.java new file mode 100644 index 000000000000..170dbbcc7ec1 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/bots/BotMapper.java @@ -0,0 +1,85 @@ +package org.openmetadata.service.resources.bots; + +import static org.openmetadata.service.util.EntityUtil.getEntityReference; + +import java.util.List; +import org.openmetadata.schema.EntityInterface; +import org.openmetadata.schema.api.CreateBot; +import org.openmetadata.schema.entity.Bot; +import org.openmetadata.schema.entity.teams.User; +import org.openmetadata.schema.type.Include; +import org.openmetadata.schema.type.Relationship; +import org.openmetadata.service.Entity; +import org.openmetadata.service.exception.CatalogExceptionMessage; +import org.openmetadata.service.jdbi3.BotRepository; +import org.openmetadata.service.jdbi3.CollectionDAO; +import org.openmetadata.service.jdbi3.EntityRepository; +import org.openmetadata.service.mapper.EntityMapper; +import org.openmetadata.service.util.EntityUtil; + +public class BotMapper implements EntityMapper { + @Override + public Bot createToEntity(CreateBot create, String user) { + BotRepository repository = (BotRepository) Entity.getEntityRepository(Entity.BOT); + Bot bot = getBot(create, user); + Bot originalBot = retrieveBot(bot.getName()); + User botUser = retrieveUser(bot); + if (botUser != null && !Boolean.TRUE.equals(botUser.getIsBot())) { + throw new IllegalArgumentException( + String.format("User [%s] is not a bot user", botUser.getName())); + } + if (userHasRelationshipWithAnyBot(botUser, originalBot)) { + List userBotRelationship = + retrieveBotRelationshipsFor(botUser); + bot = + repository.get( + null, + userBotRelationship.stream().findFirst().orElseThrow().getId(), + EntityUtil.Fields.EMPTY_FIELDS); + throw new IllegalArgumentException( + CatalogExceptionMessage.userAlreadyBot(botUser.getName(), bot.getName())); + } + // TODO: review this flow on https://github.com/open-metadata/OpenMetadata/issues/8321 + if (originalBot != null) { + bot.setProvider(originalBot.getProvider()); + } + return bot; + } + + private Bot getBot(CreateBot create, String user) { + return copy(new Bot(), create, user) + .withBotUser(getEntityReference(Entity.USER, create.getBotUser())) + .withProvider(create.getProvider()) + .withFullyQualifiedName(create.getName()); + } + + private boolean userHasRelationshipWithAnyBot(User user, Bot botUser) { + if (user == null) { + return false; + } + List userBotRelationship = + retrieveBotRelationshipsFor(user); + return !userBotRelationship.isEmpty() + && (botUser == null + || userBotRelationship.stream() + .anyMatch(relationship -> !relationship.getId().equals(botUser.getId()))); + } + + private List retrieveBotRelationshipsFor(User user) { + BotRepository repository = (BotRepository) Entity.getEntityRepository(Entity.BOT); + return repository.findFromRecords(user.getId(), Entity.USER, Relationship.CONTAINS, Entity.BOT); + } + + private User retrieveUser(Bot bot) { + EntityRepository userRepository = + Entity.getEntityRepository(Entity.USER); + return (User) + userRepository.findByNameOrNull( + bot.getBotUser().getFullyQualifiedName(), Include.NON_DELETED); + } + + private Bot retrieveBot(String botName) { + BotRepository repository = (BotRepository) Entity.getEntityRepository(Entity.BOT); + return repository.findByNameOrNull(botName, Include.NON_DELETED); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/bots/BotResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/bots/BotResource.java index 457485dcf575..eaf240ec5618 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/bots/BotResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/bots/BotResource.java @@ -48,30 +48,24 @@ import javax.ws.rs.core.SecurityContext; import javax.ws.rs.core.UriInfo; import lombok.extern.slf4j.Slf4j; -import org.openmetadata.schema.EntityInterface; import org.openmetadata.schema.api.CreateBot; import org.openmetadata.schema.api.data.RestoreEntity; import org.openmetadata.schema.entity.Bot; +import org.openmetadata.schema.entity.teams.Role; import org.openmetadata.schema.entity.teams.User; import org.openmetadata.schema.type.EntityHistory; import org.openmetadata.schema.type.Include; -import org.openmetadata.schema.type.Relationship; import org.openmetadata.schema.utils.EntityInterfaceUtil; import org.openmetadata.service.Entity; import org.openmetadata.service.OpenMetadataApplicationConfig; -import org.openmetadata.service.exception.CatalogExceptionMessage; import org.openmetadata.service.jdbi3.BotRepository; -import org.openmetadata.service.jdbi3.CollectionDAO.EntityRelationshipRecord; -import org.openmetadata.service.jdbi3.EntityRepository; import org.openmetadata.service.jdbi3.ListFilter; import org.openmetadata.service.jdbi3.UserRepository; import org.openmetadata.service.limits.Limits; import org.openmetadata.service.resources.Collection; import org.openmetadata.service.resources.EntityResource; -import org.openmetadata.service.resources.teams.RoleResource; import org.openmetadata.service.security.Authorizer; import org.openmetadata.service.security.SecurityUtil; -import org.openmetadata.service.util.EntityUtil; import org.openmetadata.service.util.ResultList; import org.openmetadata.service.util.UserUtil; @@ -87,6 +81,7 @@ @Collection(name = "bots", order = 4, requiredForOps = true) // initialize after user resource public class BotResource extends EntityResource { public static final String COLLECTION_PATH = "/v1/bots/"; + private final BotMapper mapper = new BotMapper(); public BotResource(Authorizer authorizer, Limits limits) { super(Entity.BOT, authorizer, limits); @@ -105,7 +100,17 @@ public void initialize(OpenMetadataApplicationConfig config) throws IOException .withIsAdmin(false); user.setRoles( listOrEmpty(botUser.getRoles()).stream() - .map(entityReference -> RoleResource.getRole(entityReference.getName())) + .map( + entityReference -> { + Role role = + Entity.getEntityByName( + Entity.ROLE, + entityReference.getName(), + "id", + Include.NON_DELETED, + true); + return role.getEntityReference(); + }) .toList()); // Add or update User Bot UserUtil.addOrUpdateBotUser(user); @@ -300,7 +305,7 @@ public Bot getVersion( }) public Response create( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateBot create) { - Bot bot = getBot(securityContext, create); + Bot bot = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return create(uriInfo, securityContext, bot); } @@ -321,7 +326,7 @@ public Response create( }) public Response createOrUpdate( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateBot create) { - Bot bot = getBot(securityContext, create); + Bot bot = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return createOrUpdate(uriInfo, securityContext, bot); } @@ -449,64 +454,4 @@ public Response restoreBot( @Valid RestoreEntity restore) { return restoreEntity(uriInfo, securityContext, restore.getId()); } - - private Bot getBot(CreateBot create, String user) { - return repository - .copy(new Bot(), create, user) - .withBotUser(getEntityReference(Entity.USER, create.getBotUser())) - .withProvider(create.getProvider()) - .withFullyQualifiedName(create.getName()); - } - - private boolean userHasRelationshipWithAnyBot(User user, Bot botUser) { - if (user == null) { - return false; - } - List userBotRelationship = retrieveBotRelationshipsFor(user); - return !userBotRelationship.isEmpty() - && (botUser == null - || userBotRelationship.stream() - .anyMatch(relationship -> !relationship.getId().equals(botUser.getId()))); - } - - private List retrieveBotRelationshipsFor(User user) { - return repository.findFromRecords(user.getId(), Entity.USER, Relationship.CONTAINS, Entity.BOT); - } - - private Bot getBot(SecurityContext securityContext, CreateBot create) { - Bot bot = getBot(create, securityContext.getUserPrincipal().getName()); - Bot originalBot = retrieveBot(bot.getName()); - User botUser = retrieveUser(bot); - if (botUser != null && !Boolean.TRUE.equals(botUser.getIsBot())) { - throw new IllegalArgumentException( - String.format("User [%s] is not a bot user", botUser.getName())); - } - if (userHasRelationshipWithAnyBot(botUser, originalBot)) { - List userBotRelationship = retrieveBotRelationshipsFor(botUser); - bot = - repository.get( - null, - userBotRelationship.stream().findFirst().orElseThrow().getId(), - EntityUtil.Fields.EMPTY_FIELDS); - throw new IllegalArgumentException( - CatalogExceptionMessage.userAlreadyBot(botUser.getName(), bot.getName())); - } - // TODO: review this flow on https://github.com/open-metadata/OpenMetadata/issues/8321 - if (originalBot != null) { - bot.setProvider(originalBot.getProvider()); - } - return bot; - } - - private User retrieveUser(Bot bot) { - EntityRepository userRepository = - Entity.getEntityRepository(Entity.USER); - return (User) - userRepository.findByNameOrNull( - bot.getBotUser().getFullyQualifiedName(), Include.NON_DELETED); - } - - private Bot retrieveBot(String botName) { - return repository.findByNameOrNull(botName, Include.NON_DELETED); - } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/charts/ChartMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/charts/ChartMapper.java new file mode 100644 index 000000000000..095077f0a8e5 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/charts/ChartMapper.java @@ -0,0 +1,21 @@ +package org.openmetadata.service.resources.charts; + +import static org.openmetadata.service.util.EntityUtil.getEntityReferences; + +import org.openmetadata.schema.api.data.CreateChart; +import org.openmetadata.schema.entity.data.Chart; +import org.openmetadata.service.Entity; +import org.openmetadata.service.mapper.EntityMapper; +import org.openmetadata.service.util.EntityUtil; + +public class ChartMapper implements EntityMapper { + @Override + public Chart createToEntity(CreateChart create, String user) { + return copy(new Chart(), create, user) + .withService(EntityUtil.getEntityReference(Entity.DASHBOARD_SERVICE, create.getService())) + .withChartType(create.getChartType()) + .withSourceUrl(create.getSourceUrl()) + .withSourceHash(create.getSourceHash()) + .withDashboards(getEntityReferences(Entity.DASHBOARD, create.getDashboards())); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/charts/ChartResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/charts/ChartResource.java index 31dc3da0797d..b9d8ca6268b0 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/charts/ChartResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/charts/ChartResource.java @@ -61,7 +61,6 @@ import org.openmetadata.service.resources.Collection; import org.openmetadata.service.resources.EntityResource; import org.openmetadata.service.security.Authorizer; -import org.openmetadata.service.util.EntityUtil; import org.openmetadata.service.util.ResultList; @Path("/v1/charts") @@ -74,6 +73,7 @@ @Collection(name = "charts") public class ChartResource extends EntityResource { public static final String COLLECTION_PATH = "v1/charts/"; + private final ChartMapper mapper = new ChartMapper(); static final String FIELDS = "owners,followers,tags,domain,dataProducts,sourceHash,dashboards"; @Override @@ -300,7 +300,7 @@ public Response create( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateChart create) { - Chart chart = getChart(create, securityContext.getUserPrincipal().getName()); + Chart chart = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return create(uriInfo, securityContext, chart); } @@ -379,7 +379,7 @@ public Response createOrUpdate( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateChart create) { - Chart chart = getChart(create, securityContext.getUserPrincipal().getName()); + Chart chart = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return createOrUpdate(uriInfo, securityContext, chart); } @@ -523,14 +523,4 @@ public Response restoreChart( @Valid RestoreEntity restore) { return restoreEntity(uriInfo, securityContext, restore.getId()); } - - private Chart getChart(CreateChart create, String user) { - return repository - .copy(new Chart(), create, user) - .withService(EntityUtil.getEntityReference(Entity.DASHBOARD_SERVICE, create.getService())) - .withChartType(create.getChartType()) - .withSourceUrl(create.getSourceUrl()) - .withSourceHash(create.getSourceHash()) - .withDashboards(getEntityReferences(Entity.DASHBOARD, create.getDashboards())); - } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dashboards/DashboardMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dashboards/DashboardMapper.java new file mode 100644 index 000000000000..c7ba5a1084a5 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dashboards/DashboardMapper.java @@ -0,0 +1,23 @@ +package org.openmetadata.service.resources.dashboards; + +import static org.openmetadata.service.util.EntityUtil.getEntityReference; +import static org.openmetadata.service.util.EntityUtil.getEntityReferences; + +import org.openmetadata.schema.api.data.CreateDashboard; +import org.openmetadata.schema.entity.data.Dashboard; +import org.openmetadata.service.Entity; +import org.openmetadata.service.mapper.EntityMapper; + +public class DashboardMapper implements EntityMapper { + @Override + public Dashboard createToEntity(CreateDashboard create, String user) { + return copy(new Dashboard(), create, user) + .withService(getEntityReference(Entity.DASHBOARD_SERVICE, create.getService())) + .withCharts(getEntityReferences(Entity.CHART, create.getCharts())) + .withDataModels(getEntityReferences(Entity.DASHBOARD_DATA_MODEL, create.getDataModels())) + .withSourceUrl(create.getSourceUrl()) + .withDashboardType(create.getDashboardType()) + .withProject(create.getProject()) + .withSourceHash(create.getSourceHash()); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dashboards/DashboardResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dashboards/DashboardResource.java index 19aea5208efa..bc26f5a1138d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dashboards/DashboardResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dashboards/DashboardResource.java @@ -76,6 +76,7 @@ public class DashboardResource extends EntityResource { + @Override + public Database createToEntity(CreateDatabase create, String user) { + return copy(new Database(), create, user) + .withService(getEntityReference(Entity.DATABASE_SERVICE, create.getService())) + .withSourceUrl(create.getSourceUrl()) + .withRetentionPeriod(create.getRetentionPeriod()) + .withSourceHash(create.getSourceHash()); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/DatabaseResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/DatabaseResource.java index e46ce51b8ff7..646f0b0ee53b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/DatabaseResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/DatabaseResource.java @@ -77,6 +77,7 @@ @Collection(name = "databases") public class DatabaseResource extends EntityResource { public static final String COLLECTION_PATH = "v1/databases/"; + private final DatabaseMapper mapper = new DatabaseMapper(); static final String FIELDS = "owners,databaseSchemas,usageSummary,location,tags,extension,domain,sourceHash"; @@ -310,7 +311,7 @@ public Response create( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateDatabase create) { - Database database = getDatabase(create, securityContext.getUserPrincipal().getName()); + Database database = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return create(uriInfo, securityContext, database); } @@ -390,7 +391,7 @@ public Response createOrUpdate( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateDatabase create) { - Database database = getDatabase(create, securityContext.getUserPrincipal().getName()); + Database database = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return createOrUpdate(uriInfo, securityContext, database); } @@ -705,13 +706,4 @@ public Database deleteDataProfilerConfig( Database database = repository.deleteDatabaseProfilerConfig(id); return addHref(uriInfo, database); } - - private Database getDatabase(CreateDatabase create, String user) { - return repository - .copy(new Database(), create, user) - .withService(getEntityReference(Entity.DATABASE_SERVICE, create.getService())) - .withSourceUrl(create.getSourceUrl()) - .withRetentionPeriod(create.getRetentionPeriod()) - .withSourceHash(create.getSourceHash()); - } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/DatabaseSchemaMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/DatabaseSchemaMapper.java new file mode 100644 index 000000000000..b6d9e9c6fc43 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/DatabaseSchemaMapper.java @@ -0,0 +1,19 @@ +package org.openmetadata.service.resources.databases; + +import static org.openmetadata.service.util.EntityUtil.getEntityReference; + +import org.openmetadata.schema.api.data.CreateDatabaseSchema; +import org.openmetadata.schema.entity.data.DatabaseSchema; +import org.openmetadata.service.Entity; +import org.openmetadata.service.mapper.EntityMapper; + +public class DatabaseSchemaMapper implements EntityMapper { + @Override + public DatabaseSchema createToEntity(CreateDatabaseSchema create, String user) { + return copy(new DatabaseSchema(), create, user) + .withDatabase(getEntityReference(Entity.DATABASE, create.getDatabase())) + .withSourceUrl(create.getSourceUrl()) + .withRetentionPeriod(create.getRetentionPeriod()) + .withSourceHash(create.getSourceHash()); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/DatabaseSchemaResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/DatabaseSchemaResource.java index 72bbb666f508..fc0e4becc212 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/DatabaseSchemaResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/DatabaseSchemaResource.java @@ -75,6 +75,7 @@ @Collection(name = "databaseSchemas") public class DatabaseSchemaResource extends EntityResource { + private final DatabaseSchemaMapper mapper = new DatabaseSchemaMapper(); public static final String COLLECTION_PATH = "v1/databaseSchemas/"; static final String FIELDS = "owners,tables,usageSummary,tags,extension,domain,sourceHash"; @@ -309,7 +310,8 @@ public Response create( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateDatabaseSchema create) { - DatabaseSchema schema = getDatabaseSchema(create, securityContext.getUserPrincipal().getName()); + DatabaseSchema schema = + mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return create(uriInfo, securityContext, schema); } @@ -390,7 +392,8 @@ public Response createOrUpdate( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateDatabaseSchema create) { - DatabaseSchema schema = getDatabaseSchema(create, securityContext.getUserPrincipal().getName()); + DatabaseSchema schema = + mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return createOrUpdate(uriInfo, securityContext, schema); } @@ -739,13 +742,4 @@ public Response searchSchemaEntityRelationship( return Entity.getSearchRepository() .searchSchemaEntityRelationship(fqn, upstreamDepth, downstreamDepth, queryFilter, deleted); } - - private DatabaseSchema getDatabaseSchema(CreateDatabaseSchema create, String user) { - return repository - .copy(new DatabaseSchema(), create, user) - .withDatabase(getEntityReference(Entity.DATABASE, create.getDatabase())) - .withSourceUrl(create.getSourceUrl()) - .withRetentionPeriod(create.getRetentionPeriod()) - .withSourceHash(create.getSourceHash()); - } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/StoredProcedureMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/StoredProcedureMapper.java new file mode 100644 index 000000000000..ca1013b8566c --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/StoredProcedureMapper.java @@ -0,0 +1,20 @@ +package org.openmetadata.service.resources.databases; + +import static org.openmetadata.service.util.EntityUtil.getEntityReference; + +import org.openmetadata.schema.api.data.CreateStoredProcedure; +import org.openmetadata.schema.entity.data.StoredProcedure; +import org.openmetadata.service.Entity; +import org.openmetadata.service.mapper.EntityMapper; + +public class StoredProcedureMapper implements EntityMapper { + @Override + public StoredProcedure createToEntity(CreateStoredProcedure create, String user) { + return copy(new StoredProcedure(), create, user) + .withDatabaseSchema(getEntityReference(Entity.DATABASE_SCHEMA, create.getDatabaseSchema())) + .withStoredProcedureCode(create.getStoredProcedureCode()) + .withStoredProcedureType(create.getStoredProcedureType()) + .withSourceUrl(create.getSourceUrl()) + .withSourceHash(create.getSourceHash()); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/StoredProcedureResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/StoredProcedureResource.java index 84626a53062b..f83976818d33 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/StoredProcedureResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/StoredProcedureResource.java @@ -42,6 +42,7 @@ @Collection(name = "storedProcedures") public class StoredProcedureResource extends EntityResource { + private final StoredProcedureMapper mapper = new StoredProcedureMapper(); public static final String COLLECTION_PATH = "v1/storedProcedures/"; static final String FIELDS = "owners,tags,followers,extension,domain,sourceHash"; @@ -271,7 +272,7 @@ public Response create( @Context SecurityContext securityContext, @Valid CreateStoredProcedure create) { StoredProcedure storedProcedure = - getStoredProcedure(create, securityContext.getUserPrincipal().getName()); + mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return create(uriInfo, securityContext, storedProcedure); } @@ -353,7 +354,7 @@ public Response createOrUpdate( @Context SecurityContext securityContext, @Valid CreateStoredProcedure create) { StoredProcedure storedProcedure = - getStoredProcedure(create, securityContext.getUserPrincipal().getName()); + mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return createOrUpdate(uriInfo, securityContext, storedProcedure); } @@ -527,14 +528,4 @@ public Response restoreStoredProcedure( @Valid RestoreEntity restore) { return restoreEntity(uriInfo, securityContext, restore.getId()); } - - private StoredProcedure getStoredProcedure(CreateStoredProcedure create, String user) { - return repository - .copy(new StoredProcedure(), create, user) - .withDatabaseSchema(getEntityReference(Entity.DATABASE_SCHEMA, create.getDatabaseSchema())) - .withStoredProcedureCode(create.getStoredProcedureCode()) - .withStoredProcedureType(create.getStoredProcedureType()) - .withSourceUrl(create.getSourceUrl()) - .withSourceHash(create.getSourceHash()); - } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableMapper.java new file mode 100644 index 000000000000..48b609724064 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableMapper.java @@ -0,0 +1,53 @@ +package org.openmetadata.service.resources.databases; + +import static org.openmetadata.service.util.EntityUtil.getEntityReference; + +import java.util.UUID; +import org.openmetadata.schema.api.data.CreateTable; +import org.openmetadata.schema.api.tests.CreateCustomMetric; +import org.openmetadata.schema.entity.data.Table; +import org.openmetadata.schema.tests.CustomMetric; +import org.openmetadata.service.Entity; +import org.openmetadata.service.mapper.EntityMapper; + +public class TableMapper implements EntityMapper { + @Override + public Table createToEntity(CreateTable create, String user) { + return validateNewTable( + copy(new Table(), create, user) + .withColumns(create.getColumns()) + .withSourceUrl(create.getSourceUrl()) + .withLocationPath(create.getLocationPath()) + .withTableConstraints(create.getTableConstraints()) + .withTablePartition(create.getTablePartition()) + .withTableType(create.getTableType()) + .withFileFormat(create.getFileFormat()) + .withSchemaDefinition(create.getSchemaDefinition()) + .withTableProfilerConfig(create.getTableProfilerConfig()) + .withDatabaseSchema( + getEntityReference(Entity.DATABASE_SCHEMA, create.getDatabaseSchema()))) + .withDatabaseSchema(getEntityReference(Entity.DATABASE_SCHEMA, create.getDatabaseSchema())) + .withRetentionPeriod(create.getRetentionPeriod()) + .withSourceHash(create.getSourceHash()); + } + + public CustomMetric createCustomMetricToEntity(CreateCustomMetric create, String user) { + return new CustomMetric() + .withId(UUID.randomUUID()) + .withDescription(create.getDescription()) + .withName(create.getName()) + .withColumnName(create.getColumnName()) + .withOwners(create.getOwners()) + .withExpression(create.getExpression()) + .withUpdatedBy(user) + .withUpdatedAt(System.currentTimeMillis()); + } + + public static Table validateNewTable(Table table) { + table.setId(UUID.randomUUID()); + DatabaseUtil.validateConstraints(table.getColumns(), table.getTableConstraints()); + DatabaseUtil.validateTablePartition(table.getColumns(), table.getTablePartition()); + DatabaseUtil.validateColumns(table.getColumns()); + return table; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java index 529aa0769776..d74e7b13587c 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java @@ -90,6 +90,7 @@ @Consumes(MediaType.APPLICATION_JSON) @Collection(name = "tables") public class TableResource extends EntityResource { + private final TableMapper mapper = new TableMapper(); public static final String COLLECTION_PATH = "v1/tables/"; static final String FIELDS = "tableConstraints,tablePartition,usageSummary,owners,customMetrics,columns," @@ -367,7 +368,7 @@ public Response create( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateTable create) { - Table table = getTable(create, securityContext.getUserPrincipal().getName()); + Table table = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return create(uriInfo, securityContext, table); } @@ -391,7 +392,7 @@ public Response createOrUpdate( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateTable create) { - Table table = getTable(create, securityContext.getUserPrincipal().getName()); + Table table = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return createOrUpdate(uriInfo, securityContext, table); } @@ -1152,7 +1153,9 @@ public Table addCustomMetric( OperationContext operationContext = new OperationContext(entityType, MetadataOperation.EDIT_DATA_PROFILE); authorizer.authorize(securityContext, operationContext, getResourceContextById(id)); - CustomMetric customMetric = getCustomMetric(securityContext, createCustomMetric); + CustomMetric customMetric = + mapper.createCustomMetricToEntity( + createCustomMetric, securityContext.getUserPrincipal().getName()); Table table = repository.addCustomMetric(id, customMetric); return addHref(uriInfo, table); } @@ -1312,44 +1315,4 @@ public Response searchEntityRelationship( return Entity.getSearchRepository() .searchEntityRelationship(fqn, upstreamDepth, downstreamDepth, queryFilter, deleted); } - - public static Table validateNewTable(Table table) { - table.setId(UUID.randomUUID()); - DatabaseUtil.validateConstraints(table.getColumns(), table.getTableConstraints()); - DatabaseUtil.validateTablePartition(table.getColumns(), table.getTablePartition()); - DatabaseUtil.validateColumns(table.getColumns()); - return table; - } - - private Table getTable(CreateTable create, String user) { - return validateNewTable( - repository - .copy(new Table(), create, user) - .withColumns(create.getColumns()) - .withSourceUrl(create.getSourceUrl()) - .withLocationPath(create.getLocationPath()) - .withTableConstraints(create.getTableConstraints()) - .withTablePartition(create.getTablePartition()) - .withTableType(create.getTableType()) - .withFileFormat(create.getFileFormat()) - .withSchemaDefinition(create.getSchemaDefinition()) - .withTableProfilerConfig(create.getTableProfilerConfig()) - .withDatabaseSchema( - getEntityReference(Entity.DATABASE_SCHEMA, create.getDatabaseSchema()))) - .withDatabaseSchema(getEntityReference(Entity.DATABASE_SCHEMA, create.getDatabaseSchema())) - .withRetentionPeriod(create.getRetentionPeriod()) - .withSourceHash(create.getSourceHash()); - } - - private CustomMetric getCustomMetric(SecurityContext securityContext, CreateCustomMetric create) { - return new CustomMetric() - .withId(UUID.randomUUID()) - .withDescription(create.getDescription()) - .withName(create.getName()) - .withColumnName(create.getColumnName()) - .withOwners(create.getOwners()) - .withExpression(create.getExpression()) - .withUpdatedBy(securityContext.getUserPrincipal().getName()) - .withUpdatedAt(System.currentTimeMillis()); - } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/datainsight/DataInsightChartMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/datainsight/DataInsightChartMapper.java new file mode 100644 index 000000000000..8d3523e47133 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/datainsight/DataInsightChartMapper.java @@ -0,0 +1,19 @@ +package org.openmetadata.service.resources.datainsight; + +import org.openmetadata.schema.api.dataInsight.CreateDataInsightChart; +import org.openmetadata.schema.dataInsight.DataInsightChart; +import org.openmetadata.service.mapper.EntityMapper; + +public class DataInsightChartMapper + implements EntityMapper { + @Override + public DataInsightChart createToEntity(CreateDataInsightChart create, String user) { + return copy(new DataInsightChart(), create, user) + .withName(create.getName()) + .withDescription(create.getDescription()) + .withDataIndexType(create.getDataIndexType()) + .withDimensions(create.getDimensions()) + .withMetrics(create.getMetrics()) + .withDisplayName(create.getDisplayName()); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/datainsight/DataInsightChartResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/datainsight/DataInsightChartResource.java index da081c044bab..50a9c365f963 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/datainsight/DataInsightChartResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/datainsight/DataInsightChartResource.java @@ -65,6 +65,7 @@ public class DataInsightChartResource extends EntityResource { + private final DataInsightChartMapper mapper = new DataInsightChartMapper(); public static final String COLLECTION_PATH = DataInsightChartRepository.COLLECTION_PATH; public static final String FIELDS = "owners"; private final SearchRepository searchRepository; @@ -299,7 +300,7 @@ public Response create( @Context SecurityContext securityContext, @Valid CreateDataInsightChart create) { DataInsightChart dataInsightChart = - getDataInsightChart(create, securityContext.getUserPrincipal().getName()); + mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return create(uriInfo, securityContext, dataInsightChart); } @@ -381,7 +382,7 @@ public Response createOrUpdate( @Context SecurityContext securityContext, @Valid CreateDataInsightChart create) { DataInsightChart dataInsightChart = - getDataInsightChart(create, securityContext.getUserPrincipal().getName()); + mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return createOrUpdate(uriInfo, securityContext, dataInsightChart); } @@ -536,15 +537,4 @@ public Response listDataInsightChartResult( return searchRepository.listDataInsightChartResult( startTs, endTs, tier, team, dataInsightChartName, size, from, queryFilter, dataReportIndex); } - - private DataInsightChart getDataInsightChart(CreateDataInsightChart create, String user) { - return repository - .copy(new DataInsightChart(), create, user) - .withName(create.getName()) - .withDescription(create.getDescription()) - .withDataIndexType(create.getDataIndexType()) - .withDimensions(create.getDimensions()) - .withMetrics(create.getMetrics()) - .withDisplayName(create.getDisplayName()); - } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/datamodels/DashboardDataModelMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/datamodels/DashboardDataModelMapper.java new file mode 100644 index 000000000000..3785f1500b01 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/datamodels/DashboardDataModelMapper.java @@ -0,0 +1,25 @@ +package org.openmetadata.service.resources.datamodels; + +import org.openmetadata.schema.api.data.CreateDashboardDataModel; +import org.openmetadata.schema.entity.data.DashboardDataModel; +import org.openmetadata.service.Entity; +import org.openmetadata.service.mapper.EntityMapper; +import org.openmetadata.service.resources.databases.DatabaseUtil; +import org.openmetadata.service.util.EntityUtil; + +public class DashboardDataModelMapper + implements EntityMapper { + @Override + public DashboardDataModel createToEntity(CreateDashboardDataModel create, String user) { + DatabaseUtil.validateColumns(create.getColumns()); + return copy(new DashboardDataModel(), create, user) + .withService(EntityUtil.getEntityReference(Entity.DASHBOARD_SERVICE, create.getService())) + .withDataModelType(create.getDataModelType()) + .withSql(create.getSql()) + .withDataModelType(create.getDataModelType()) + .withServiceType(create.getServiceType()) + .withColumns(create.getColumns()) + .withProject(create.getProject()) + .withSourceHash(create.getSourceHash()); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/datamodels/DashboardDataModelResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/datamodels/DashboardDataModelResource.java index 5be6a779a363..4250b3304b5b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/datamodels/DashboardDataModelResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/datamodels/DashboardDataModelResource.java @@ -56,9 +56,7 @@ import org.openmetadata.service.limits.Limits; import org.openmetadata.service.resources.Collection; import org.openmetadata.service.resources.EntityResource; -import org.openmetadata.service.resources.databases.DatabaseUtil; import org.openmetadata.service.security.Authorizer; -import org.openmetadata.service.util.EntityUtil; import org.openmetadata.service.util.ResultList; @Path("/v1/dashboard/datamodels") @@ -71,6 +69,7 @@ @Collection(name = "datamodels") public class DashboardDataModelResource extends EntityResource { + private final DashboardDataModelMapper mapper = new DashboardDataModelMapper(); public static final String COLLECTION_PATH = "/v1/dashboard/datamodels"; protected static final String FIELDS = "owners,tags,followers,domain,sourceHash,extension"; @@ -300,7 +299,7 @@ public Response create( @Context SecurityContext securityContext, @Valid CreateDashboardDataModel create) { DashboardDataModel dashboardDataModel = - getDataModel(create, securityContext.getUserPrincipal().getName()); + mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return create(uriInfo, securityContext, dashboardDataModel); } @@ -382,7 +381,7 @@ public Response createOrUpdate( @Context SecurityContext securityContext, @Valid CreateDashboardDataModel create) { DashboardDataModel dashboardDataModel = - getDataModel(create, securityContext.getUserPrincipal().getName()); + mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return createOrUpdate(uriInfo, securityContext, dashboardDataModel); } @@ -541,18 +540,4 @@ public Response restoreDataModel( @Valid RestoreEntity restore) { return restoreEntity(uriInfo, securityContext, restore.getId()); } - - private DashboardDataModel getDataModel(CreateDashboardDataModel create, String user) { - DatabaseUtil.validateColumns(create.getColumns()); - return repository - .copy(new DashboardDataModel(), create, user) - .withService(EntityUtil.getEntityReference(Entity.DASHBOARD_SERVICE, create.getService())) - .withDataModelType(create.getDataModelType()) - .withSql(create.getSql()) - .withDataModelType(create.getDataModelType()) - .withServiceType(create.getServiceType()) - .withColumns(create.getColumns()) - .withProject(create.getProject()) - .withSourceHash(create.getSourceHash()); - } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/docstore/DocStoreMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/docstore/DocStoreMapper.java new file mode 100644 index 000000000000..7f1e259c6e57 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/docstore/DocStoreMapper.java @@ -0,0 +1,44 @@ +package org.openmetadata.service.resources.docstore; + +import javax.ws.rs.core.Response; +import org.openmetadata.schema.email.EmailTemplate; +import org.openmetadata.schema.email.TemplateValidationResponse; +import org.openmetadata.schema.entities.docStore.CreateDocument; +import org.openmetadata.schema.entities.docStore.Document; +import org.openmetadata.service.Entity; +import org.openmetadata.service.exception.CustomExceptionMessage; +import org.openmetadata.service.jdbi3.DocumentRepository; +import org.openmetadata.service.mapper.EntityMapper; +import org.openmetadata.service.security.Authorizer; +import org.openmetadata.service.util.JsonUtils; +import org.openmetadata.service.util.email.DefaultTemplateProvider; + +public class DocStoreMapper implements EntityMapper { + public DocStoreMapper(Authorizer authorizer) { + this.authorizer = authorizer; + } + + private final Authorizer authorizer; + + @Override + public Document createToEntity(CreateDocument create, String user) { + DocumentRepository documentRepository = + (DocumentRepository) Entity.getEntityRepository(Entity.DOCUMENT); + // Validate email template + if (create.getEntityType().equals(DefaultTemplateProvider.ENTITY_TYPE_EMAIL_TEMPLATE)) { + // Only Admins Can do these operations + authorizer.authorizeAdmin(user); + String content = JsonUtils.convertValue(create.getData(), EmailTemplate.class).getTemplate(); + TemplateValidationResponse validationResp = + documentRepository.validateEmailTemplate(create.getName(), content); + if (Boolean.FALSE.equals(validationResp.getIsValid())) { + throw new CustomExceptionMessage( + Response.status(400).entity(validationResp).build(), validationResp.getMessage()); + } + } + return copy(new Document(), create, user) + .withFullyQualifiedName(create.getFullyQualifiedName()) + .withData(create.getData()) + .withEntityType(create.getEntityType()); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/docstore/DocStoreResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/docstore/DocStoreResource.java index fb69f12fec28..a0420f7122db 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/docstore/DocStoreResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/docstore/DocStoreResource.java @@ -57,16 +57,13 @@ import org.openmetadata.schema.type.MetadataOperation; import org.openmetadata.service.Entity; import org.openmetadata.service.OpenMetadataApplicationConfig; -import org.openmetadata.service.exception.CustomExceptionMessage; import org.openmetadata.service.jdbi3.DocumentRepository; import org.openmetadata.service.jdbi3.ListFilter; import org.openmetadata.service.limits.Limits; import org.openmetadata.service.resources.Collection; import org.openmetadata.service.resources.EntityResource; import org.openmetadata.service.security.Authorizer; -import org.openmetadata.service.util.JsonUtils; import org.openmetadata.service.util.ResultList; -import org.openmetadata.service.util.email.DefaultTemplateProvider; @Slf4j @Path("/v1/docStore") @@ -76,6 +73,7 @@ @Collection(name = "knowledgePanel", order = 2) public class DocStoreResource extends EntityResource { public static final String COLLECTION_PATH = "/v1/docStore"; + private DocStoreMapper mapper; @Override public Document addHref(UriInfo uriInfo, Document doc) { @@ -91,6 +89,7 @@ protected List getEntitySpecificOperations() { public DocStoreResource(Authorizer authorizer, Limits limits) { super(Entity.DOCUMENT, authorizer, limits); + this.mapper = new DocStoreMapper(authorizer); } public static class DocumentList extends ResultList { @@ -301,7 +300,7 @@ public Response create( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateDocument cd) { - Document doc = getDocument(cd, securityContext); + Document doc = mapper.createToEntity(cd, securityContext.getUserPrincipal().getName()); return create(uriInfo, securityContext, doc); } @@ -324,7 +323,7 @@ public Response createOrUpdate( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateDocument cd) { - Document doc = getDocument(cd, securityContext); + Document doc = mapper.createToEntity(cd, securityContext.getUserPrincipal().getName()); return createOrUpdate(uriInfo, securityContext, doc); } @@ -489,24 +488,4 @@ public Response resetEmailTemplate( .build(); } } - - private Document getDocument(CreateDocument cd, SecurityContext securityContext) { - // Validate email template - if (cd.getEntityType().equals(DefaultTemplateProvider.ENTITY_TYPE_EMAIL_TEMPLATE)) { - // Only Admins Can do these operations - authorizer.authorizeAdmin(securityContext); - String content = JsonUtils.convertValue(cd.getData(), EmailTemplate.class).getTemplate(); - TemplateValidationResponse validationResp = - repository.validateEmailTemplate(cd.getName(), content); - if (Boolean.FALSE.equals(validationResp.getIsValid())) { - throw new CustomExceptionMessage( - Response.status(400).entity(validationResp).build(), validationResp.getMessage()); - } - } - return repository - .copy(new Document(), cd, securityContext.getUserPrincipal().getName()) - .withFullyQualifiedName(cd.getFullyQualifiedName()) - .withData(cd.getData()) - .withEntityType(cd.getEntityType()); - } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/domains/DataProductMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/domains/DataProductMapper.java new file mode 100644 index 000000000000..f91693a86367 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/domains/DataProductMapper.java @@ -0,0 +1,34 @@ +package org.openmetadata.service.resources.domains; + +import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; +import static org.openmetadata.service.util.EntityUtil.getEntityReferences; + +import java.util.ArrayList; +import java.util.List; +import org.openmetadata.schema.api.domains.CreateDataProduct; +import org.openmetadata.schema.entity.domains.DataProduct; +import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.type.Include; +import org.openmetadata.service.Entity; +import org.openmetadata.service.mapper.EntityMapper; +import org.openmetadata.service.util.EntityUtil; + +public class DataProductMapper implements EntityMapper { + @Override + public DataProduct createToEntity(CreateDataProduct create, String user) { + List experts = create.getExperts(); + DataProduct dataProduct = + copy(new DataProduct(), create, user) + .withFullyQualifiedName(create.getName()) + .withStyle(create.getStyle()) + .withExperts( + EntityUtil.populateEntityReferences(getEntityReferences(Entity.USER, experts))); + dataProduct.withAssets(new ArrayList<>()); + for (EntityReference asset : listOrEmpty(create.getAssets())) { + asset = Entity.getEntityReference(asset, Include.NON_DELETED); + dataProduct.getAssets().add(asset); + dataProduct.getAssets().sort(EntityUtil.compareEntityReference); + } + return dataProduct; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/domains/DataProductResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/domains/DataProductResource.java index f5a20bbe770a..1794f49d42c8 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/domains/DataProductResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/domains/DataProductResource.java @@ -13,7 +13,6 @@ package org.openmetadata.service.resources.domains; -import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; import io.swagger.v3.oas.annotations.ExternalDocumentation; @@ -25,8 +24,6 @@ import io.swagger.v3.oas.annotations.parameters.RequestBody; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.tags.Tag; -import java.util.ArrayList; -import java.util.List; import java.util.UUID; import javax.json.JsonPatch; import javax.validation.Valid; @@ -64,7 +61,6 @@ import org.openmetadata.service.resources.Collection; import org.openmetadata.service.resources.EntityResource; import org.openmetadata.service.security.Authorizer; -import org.openmetadata.service.util.EntityUtil; import org.openmetadata.service.util.ResultList; @Slf4j @@ -79,6 +75,7 @@ @Collection(name = "dataProducts", order = 4) // initialize after user resource public class DataProductResource extends EntityResource { public static final String COLLECTION_PATH = "/v1/dataProducts/"; + private final DataProductMapper mapper = new DataProductMapper(); static final String FIELDS = "domain,owners,experts,assets,extension"; public DataProductResource(Authorizer authorizer, Limits limits) { @@ -285,7 +282,8 @@ public Response create( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateDataProduct create) { - DataProduct dataProduct = getDataProduct(create, securityContext.getUserPrincipal().getName()); + DataProduct dataProduct = + mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return create(uriInfo, securityContext, dataProduct); } @@ -309,7 +307,8 @@ public Response createOrUpdate( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateDataProduct create) { - DataProduct dataProduct = getDataProduct(create, securityContext.getUserPrincipal().getName()); + DataProduct dataProduct = + mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return createOrUpdate(uriInfo, securityContext, dataProduct); } @@ -464,22 +463,4 @@ public Response delete( String name) { return deleteByName(uriInfo, securityContext, name, true, true); } - - private DataProduct getDataProduct(CreateDataProduct create, String user) { - List experts = create.getExperts(); - DataProduct dataProduct = - repository - .copy(new DataProduct(), create, user) - .withFullyQualifiedName(create.getName()) - .withStyle(create.getStyle()) - .withExperts( - EntityUtil.populateEntityReferences(getEntityReferences(Entity.USER, experts))); - dataProduct.withAssets(new ArrayList<>()); - for (EntityReference asset : listOrEmpty(create.getAssets())) { - asset = Entity.getEntityReference(asset, Include.NON_DELETED); - dataProduct.getAssets().add(asset); - dataProduct.getAssets().sort(EntityUtil.compareEntityReference); - } - return dataProduct; - } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/domains/DomainMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/domains/DomainMapper.java new file mode 100644 index 000000000000..fb4ef61c6cd6 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/domains/DomainMapper.java @@ -0,0 +1,28 @@ +package org.openmetadata.service.resources.domains; + +import static org.openmetadata.service.util.EntityUtil.getEntityReference; +import static org.openmetadata.service.util.EntityUtil.getEntityReferences; + +import java.util.List; +import org.openmetadata.schema.api.domains.CreateDomain; +import org.openmetadata.schema.entity.domains.Domain; +import org.openmetadata.schema.type.Include; +import org.openmetadata.service.Entity; +import org.openmetadata.service.mapper.EntityMapper; +import org.openmetadata.service.util.EntityUtil; + +public class DomainMapper implements EntityMapper { + @Override + public Domain createToEntity(CreateDomain create, String user) { + List experts = create.getExperts(); + return copy(new Domain(), create, user) + .withStyle(create.getStyle()) + .withDomainType(create.getDomainType()) + .withFullyQualifiedName(create.getName()) + .withParent( + Entity.getEntityReference( + getEntityReference(Entity.DOMAIN, create.getParent()), Include.NON_DELETED)) + .withExperts( + EntityUtil.populateEntityReferences(getEntityReferences(Entity.USER, experts))); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/domains/DomainResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/domains/DomainResource.java index b416b2e35f69..c335f6833de6 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/domains/DomainResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/domains/DomainResource.java @@ -22,7 +22,6 @@ import io.swagger.v3.oas.annotations.parameters.RequestBody; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.tags.Tag; -import java.util.List; import java.util.UUID; import javax.json.JsonPatch; import javax.validation.Valid; @@ -46,10 +45,10 @@ import javax.ws.rs.core.UriInfo; import lombok.extern.slf4j.Slf4j; import org.openmetadata.schema.api.domains.CreateDomain; +import org.openmetadata.schema.entity.data.EntityHierarchy; import org.openmetadata.schema.entity.domains.Domain; import org.openmetadata.schema.type.ChangeEvent; import org.openmetadata.schema.type.EntityHistory; -import org.openmetadata.schema.type.Include; import org.openmetadata.schema.type.api.BulkAssets; import org.openmetadata.schema.type.api.BulkOperationResult; import org.openmetadata.service.Entity; @@ -59,7 +58,7 @@ import org.openmetadata.service.resources.Collection; import org.openmetadata.service.resources.EntityResource; import org.openmetadata.service.security.Authorizer; -import org.openmetadata.service.util.EntityUtil; +import org.openmetadata.service.util.EntityHierarchyList; import org.openmetadata.service.util.ResultList; @Slf4j @@ -73,6 +72,7 @@ @Collection(name = "domains", order = 4) // initialize after user resource public class DomainResource extends EntityResource { public static final String COLLECTION_PATH = "/v1/domains/"; + private final DomainMapper mapper = new DomainMapper(); static final String FIELDS = "children,owners,experts"; public DomainResource(Authorizer authorizer, Limits limits) { @@ -262,7 +262,7 @@ public Response create( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateDomain create) { - Domain domain = getDomain(create, securityContext.getUserPrincipal().getName()); + Domain domain = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return create(uriInfo, securityContext, domain); } @@ -286,7 +286,7 @@ public Response createOrUpdate( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateDomain create) { - Domain domain = getDomain(create, securityContext.getUserPrincipal().getName()); + Domain domain = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return createOrUpdate(uriInfo, securityContext, domain); } @@ -436,17 +436,31 @@ public Response delete( return deleteByName(uriInfo, securityContext, name, true, true); } - private Domain getDomain(CreateDomain create, String user) { - List experts = create.getExperts(); - return repository - .copy(new Domain(), create, user) - .withStyle(create.getStyle()) - .withDomainType(create.getDomainType()) - .withFullyQualifiedName(create.getName()) - .withParent( - Entity.getEntityReference( - getEntityReference(Entity.DOMAIN, create.getParent()), Include.NON_DELETED)) - .withExperts( - EntityUtil.populateEntityReferences(getEntityReferences(Entity.USER, experts))); + @GET + @Path("/hierarchy") + @Operation( + operationId = "listDomainsHierarchy", + summary = "List domains in hierarchical order", + description = "Get a list of Domains in hierarchical order.", + responses = { + @ApiResponse( + responseCode = "200", + description = "List of Domains in hierarchical order", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = EntityHierarchyList.class))) + }) + public ResultList listHierarchy( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter( + description = "Fields requested in the returned resource", + schema = @Schema(type = "string", example = FIELDS)) + @QueryParam("fields") + String fieldsParam, + @DefaultValue("10") @Min(0) @Max(1000000) @QueryParam("limit") int limitParam) { + + return new EntityHierarchyList(repository.buildHierarchy(fieldsParam, limitParam)); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseMapper.java new file mode 100644 index 000000000000..a23a13486305 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseMapper.java @@ -0,0 +1,27 @@ +package org.openmetadata.service.resources.dqtests; + +import static org.openmetadata.service.util.EntityUtil.getEntityReference; + +import org.openmetadata.schema.api.tests.CreateTestCase; +import org.openmetadata.schema.tests.TestCase; +import org.openmetadata.service.Entity; +import org.openmetadata.service.mapper.EntityMapper; +import org.openmetadata.service.resources.feeds.MessageParser; + +public class TestCaseMapper implements EntityMapper { + @Override + public TestCase createToEntity(CreateTestCase create, String user) { + MessageParser.EntityLink entityLink = MessageParser.EntityLink.parse(create.getEntityLink()); + return copy(new TestCase(), create, user) + .withDescription(create.getDescription()) + .withName(create.getName()) + .withDisplayName(create.getDisplayName()) + .withParameterValues(create.getParameterValues()) + .withEntityLink(create.getEntityLink()) + .withComputePassedFailedRowCount(create.getComputePassedFailedRowCount()) + .withUseDynamicAssertion(create.getUseDynamicAssertion()) + .withEntityFQN(entityLink.getFullyQualifiedFieldValue()) + .withTestSuite(getEntityReference(Entity.TEST_SUITE, create.getTestSuite())) + .withTestDefinition(getEntityReference(Entity.TEST_DEFINITION, create.getTestDefinition())); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResolutionStatusMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResolutionStatusMapper.java new file mode 100644 index 000000000000..d62017b1d194 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResolutionStatusMapper.java @@ -0,0 +1,28 @@ +package org.openmetadata.service.resources.dqtests; + +import org.openmetadata.schema.api.tests.CreateTestCaseResolutionStatus; +import org.openmetadata.schema.entity.teams.User; +import org.openmetadata.schema.tests.TestCase; +import org.openmetadata.schema.tests.type.TestCaseResolutionStatus; +import org.openmetadata.schema.type.Include; +import org.openmetadata.service.Entity; +import org.openmetadata.service.mapper.EntityTimeSeriesMapper; + +public class TestCaseResolutionStatusMapper + implements EntityTimeSeriesMapper { + @Override + public TestCaseResolutionStatus createToEntity( + CreateTestCaseResolutionStatus create, String user) { + TestCase testCaseEntity = + Entity.getEntityByName(Entity.TEST_CASE, create.getTestCaseReference(), null, Include.ALL); + User userEntity = Entity.getEntityByName(Entity.USER, user, null, Include.ALL); + + return new TestCaseResolutionStatus() + .withTimestamp(System.currentTimeMillis()) + .withTestCaseResolutionStatusType(create.getTestCaseResolutionStatusType()) + .withTestCaseResolutionStatusDetails(create.getTestCaseResolutionStatusDetails()) + .withUpdatedBy(userEntity.getEntityReference()) + .withUpdatedAt(System.currentTimeMillis()) + .withTestCaseReference(testCaseEntity.getEntityReference()); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResolutionStatusResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResolutionStatusResource.java index 4a12f5eff632..1fa08546cd17 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResolutionStatusResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResolutionStatusResource.java @@ -33,11 +33,8 @@ import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import org.openmetadata.schema.api.tests.CreateTestCaseResolutionStatus; -import org.openmetadata.schema.entity.teams.User; -import org.openmetadata.schema.tests.TestCase; import org.openmetadata.schema.tests.type.TestCaseResolutionStatus; import org.openmetadata.schema.tests.type.TestCaseResolutionStatusTypes; -import org.openmetadata.schema.type.Include; import org.openmetadata.schema.type.MetadataOperation; import org.openmetadata.service.Entity; import org.openmetadata.service.jdbi3.ListFilter; @@ -63,6 +60,7 @@ public class TestCaseResolutionStatusResource extends EntityTimeSeriesResource { public static final String COLLECTION_PATH = "/v1/dataQuality/testCases/testCaseIncidentStatus"; + private TestCaseResolutionStatusMapper mapper = new TestCaseResolutionStatusMapper(); public TestCaseResolutionStatusResource(Authorizer authorizer) { super(Entity.TEST_CASE_RESOLUTION_STATUS, authorizer); @@ -232,20 +230,12 @@ public Response create( new OperationContext(Entity.TEST_CASE, MetadataOperation.EDIT_TESTS); ResourceContextInterface resourceContext = ReportDataContext.builder().build(); authorizer.authorize(securityContext, operationContext, resourceContext); - - TestCase testCaseEntity = - Entity.getEntityByName( - Entity.TEST_CASE, - createTestCaseResolutionStatus.getTestCaseReference(), - null, - Include.ALL); TestCaseResolutionStatus testCaseResolutionStatus = - getTestCaseResolutionStatus( - testCaseEntity, - createTestCaseResolutionStatus, - securityContext.getUserPrincipal().getName()); - - return create(testCaseResolutionStatus, testCaseEntity.getFullyQualifiedName()); + mapper.createToEntity( + createTestCaseResolutionStatus, securityContext.getUserPrincipal().getName()); + return create( + testCaseResolutionStatus, + testCaseResolutionStatus.getTestCaseReference().getFullyQualifiedName()); } @PATCH @@ -284,21 +274,4 @@ public Response patch( repository.patch(id, patch, securityContext.getUserPrincipal().getName()); return response.toResponse(); } - - private TestCaseResolutionStatus getTestCaseResolutionStatus( - TestCase testCaseEntity, - CreateTestCaseResolutionStatus createTestCaseResolutionStatus, - String userName) { - User userEntity = Entity.getEntityByName(Entity.USER, userName, null, Include.ALL); - - return new TestCaseResolutionStatus() - .withTimestamp(System.currentTimeMillis()) - .withTestCaseResolutionStatusType( - createTestCaseResolutionStatus.getTestCaseResolutionStatusType()) - .withTestCaseResolutionStatusDetails( - createTestCaseResolutionStatus.getTestCaseResolutionStatusDetails()) - .withUpdatedBy(userEntity.getEntityReference()) - .withUpdatedAt(System.currentTimeMillis()) - .withTestCaseReference(testCaseEntity.getEntityReference()); - } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java index 4e96e18eba93..4f0add675282 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java @@ -128,8 +128,8 @@ public static class TestCaseResultList extends ResultList { "Get a list of test. Use `fields` " + "parameter to get only necessary fields. Use cursor-based pagination to limit the number " + "entries in the list using `limit` and `before` or `after` query params." - + "Use the `testSuite` field to get the executable Test Suite linked to this test case " - + "or use the `testSuites` field to list test suites (executable and logical) linked.", + + "Use the `testSuite` field to get the Basic Test Suite linked to this test case " + + "or use the `testSuites` field to list test suites (Basic and Logical) linked.", responses = { @ApiResponse( responseCode = "200", @@ -240,8 +240,8 @@ public ResultList list( "Get a list of test cases using the search service. Use `fields` " + "parameter to get only necessary fields. Use offset/limit pagination to limit the number " + "entries in the list using `limit` and `offset` query params." - + "Use the `testSuite` field to get the executable Test Suite linked to this test case " - + "or use the `testSuites` field to list test suites (executable and logical) linked.", + + "Use the `testSuite` field to get the Basic Test Suite linked to this test case " + + "or use the `testSuites` field to list test suites (Basic and Logical) linked.", responses = { @ApiResponse( responseCode = "200", @@ -645,7 +645,7 @@ public Response create( new CreateResourceContext<>(entityType, test), new OperationContext(Entity.TEST_CASE, MetadataOperation.EDIT_TESTS)); authorizer.authorize(securityContext, operationContext, resourceContext); - repository.isTestSuiteExecutable(create.getTestSuite()); + repository.isTestSuiteBasic(create.getTestSuite()); test = addHref(uriInfo, repository.create(uriInfo, test)); return Response.created(test.getHref()).entity(test).build(); } @@ -760,7 +760,7 @@ public Response createOrUpdate( new OperationContext(Entity.TABLE, MetadataOperation.EDIT_TESTS); authorizer.authorize(securityContext, operationContext, resourceContext); TestCase test = getTestCase(create, securityContext.getUserPrincipal().getName(), entityLink); - repository.isTestSuiteExecutable(create.getTestSuite()); + repository.isTestSuiteBasic(create.getTestSuite()); repository.prepareInternal(test, true); PutResponse response = repository.createOrUpdate(uriInfo, test); addHref(uriInfo, response.getEntity()); @@ -916,6 +916,7 @@ public Response addTestCaseResult( repository.deleteTestCaseFailedRowsSample(testCase.getId()); } RestUtil.validateTimestampMilliseconds(testCaseResult.getTimestamp()); + testCaseResult.withId(UUID.randomUUID()).withTestCaseFQN(fqn); return repository .addTestCaseResult( securityContext.getUserPrincipal().getName(), uriInfo, fqn, testCaseResult) @@ -1146,9 +1147,8 @@ public Response addTestCasesToLogicalTestSuite( ResourceContextInterface resourceContext = TestCaseResourceContext.builder().entity(testSuite).build(); authorizer.authorize(securityContext, operationContext, resourceContext); - if (Boolean.TRUE.equals(testSuite.getExecutable())) { - throw new IllegalArgumentException( - "You are trying to add test cases to an executable test suite."); + if (Boolean.TRUE.equals(testSuite.getBasic())) { + throw new IllegalArgumentException("You are trying to add test cases to a basic test suite."); } List testCaseIds = createLogicalTestCases.getTestCaseIds(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResultMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResultMapper.java new file mode 100644 index 000000000000..b38fd6964d8f --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResultMapper.java @@ -0,0 +1,36 @@ +package org.openmetadata.service.resources.dqtests; + +import static org.openmetadata.service.Entity.TEST_CASE; + +import java.util.UUID; +import org.openmetadata.schema.api.tests.CreateTestCaseResult; +import org.openmetadata.schema.tests.TestCase; +import org.openmetadata.schema.tests.type.TestCaseResult; +import org.openmetadata.schema.type.Include; +import org.openmetadata.service.Entity; +import org.openmetadata.service.mapper.EntityTimeSeriesMapper; +import org.openmetadata.service.util.RestUtil; + +public class TestCaseResultMapper + implements EntityTimeSeriesMapper { + @Override + public TestCaseResult createToEntity(CreateTestCaseResult create, String user) { + TestCase testCase = Entity.getEntityByName(TEST_CASE, create.getFqn(), "", Include.ALL); + RestUtil.validateTimestampMilliseconds(create.getTimestamp()); + return new TestCaseResult() + .withId(UUID.randomUUID()) + .withTestCaseFQN(testCase.getFullyQualifiedName()) + .withTimestamp(create.getTimestamp()) + .withTestCaseStatus(create.getTestCaseStatus()) + .withResult(create.getResult()) + .withSampleData(create.getSampleData()) + .withTestResultValue(create.getTestResultValue()) + .withPassedRows(create.getPassedRows()) + .withFailedRows(create.getFailedRows()) + .withPassedRowsPercentage(create.getPassedRowsPercentage()) + .withFailedRowsPercentage(create.getFailedRowsPercentage()) + .withIncidentId(create.getIncidentId()) + .withMaxBound(create.getMaxBound()) + .withMinBound(create.getMinBound()); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResultResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResultResource.java index 8561d314b3f9..080b7ddf1d1f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResultResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResultResource.java @@ -67,6 +67,7 @@ @Collection(name = "TestCaseResults") public class TestCaseResultResource extends EntityTimeSeriesResource { + private final TestCaseResultMapper mapper = new TestCaseResultMapper(); static final String FIELDS = "testCase,testDefinition"; public TestCaseResultResource(Authorizer authorizer) { @@ -101,12 +102,17 @@ public Response addTestCaseResult( @PathParam("fqn") String fqn, @Valid CreateTestCaseResult createTestCaseResults) { + // Needed in further validation to check if the testCase exists + createTestCaseResults.withFqn(fqn); ResourceContextInterface resourceContext = TestCaseResourceContext.builder().name(fqn).build(); OperationContext operationContext = new OperationContext(Entity.TABLE, MetadataOperation.EDIT_TESTS); authorizer.authorize(securityContext, operationContext, resourceContext); return repository.addTestCaseResult( - securityContext.getUserPrincipal().getName(), uriInfo, fqn, createTestCaseResults); + securityContext.getUserPrincipal().getName(), + uriInfo, + fqn, + mapper.createToEntity(createTestCaseResults, securityContext.getUserPrincipal().getName())); } @GET diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestDefinitionMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestDefinitionMapper.java new file mode 100644 index 000000000000..1b587fd77474 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestDefinitionMapper.java @@ -0,0 +1,19 @@ +package org.openmetadata.service.resources.dqtests; + +import org.openmetadata.schema.api.tests.CreateTestDefinition; +import org.openmetadata.schema.tests.TestDefinition; +import org.openmetadata.service.mapper.EntityMapper; + +public class TestDefinitionMapper implements EntityMapper { + @Override + public TestDefinition createToEntity(CreateTestDefinition create, String user) { + return copy(new TestDefinition(), create, user) + .withDescription(create.getDescription()) + .withEntityType(create.getEntityType()) + .withTestPlatforms(create.getTestPlatforms()) + .withSupportedDataTypes(create.getSupportedDataTypes()) + .withDisplayName(create.getDisplayName()) + .withParameterDefinition(create.getParameterDefinition()) + .withName(create.getName()); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestDefinitionResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestDefinitionResource.java index 9cd9126e1910..d59de3ccc2a1 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestDefinitionResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestDefinitionResource.java @@ -63,6 +63,7 @@ @Collection(name = "TestDefinitions") public class TestDefinitionResource extends EntityResource { + private final TestDefinitionMapper mapper = new TestDefinitionMapper(); public static final String COLLECTION_PATH = "/v1/dataQuality/testDefinitions"; static final String FIELDS = "owners"; @@ -314,7 +315,7 @@ public Response create( @Context SecurityContext securityContext, @Valid CreateTestDefinition create) { TestDefinition testDefinition = - getTestDefinition(create, securityContext.getUserPrincipal().getName()); + mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return create(uriInfo, securityContext, testDefinition); } @@ -367,7 +368,7 @@ public Response createOrUpdate( @Context SecurityContext securityContext, @Valid CreateTestDefinition create) { TestDefinition testDefinition = - getTestDefinition(create, securityContext.getUserPrincipal().getName()); + mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return createOrUpdate(uriInfo, securityContext, testDefinition); } @@ -452,16 +453,4 @@ public Response restoreTestDefinition( @Valid RestoreEntity restore) { return restoreEntity(uriInfo, securityContext, restore.getId()); } - - private TestDefinition getTestDefinition(CreateTestDefinition create, String user) { - return repository - .copy(new TestDefinition(), create, user) - .withDescription(create.getDescription()) - .withEntityType(create.getEntityType()) - .withTestPlatforms(create.getTestPlatforms()) - .withSupportedDataTypes(create.getSupportedDataTypes()) - .withDisplayName(create.getDisplayName()) - .withParameterDefinition(create.getParameterDefinition()) - .withName(create.getName()); - } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestSuiteMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestSuiteMapper.java new file mode 100644 index 000000000000..f5bb30c2f6e2 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestSuiteMapper.java @@ -0,0 +1,31 @@ +package org.openmetadata.service.resources.dqtests; + +import org.openmetadata.schema.api.tests.CreateTestSuite; +import org.openmetadata.schema.entity.data.Table; +import org.openmetadata.schema.tests.TestSuite; +import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.service.Entity; +import org.openmetadata.service.mapper.EntityMapper; + +public class TestSuiteMapper implements EntityMapper { + @Override + public TestSuite createToEntity(CreateTestSuite create, String user) { + TestSuite testSuite = + copy(new TestSuite(), create, user) + .withDescription(create.getDescription()) + .withDisplayName(create.getDisplayName()) + .withName(create.getName()); + if (create.getBasicEntityReference() != null) { + Table table = + Entity.getEntityByName(Entity.TABLE, create.getBasicEntityReference(), null, null); + EntityReference entityReference = + new EntityReference() + .withId(table.getId()) + .withFullyQualifiedName(table.getFullyQualifiedName()) + .withName(table.getName()) + .withType(Entity.TABLE); + testSuite.setBasicEntityReference(entityReference); + } + return testSuite; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestSuiteResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestSuiteResource.java index bc4887c1d5d0..03e7c799b9b5 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestSuiteResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestSuiteResource.java @@ -73,10 +73,12 @@ @Collection(name = "TestSuites") public class TestSuiteResource extends EntityResource { public static final String COLLECTION_PATH = "/v1/dataQuality/testSuites"; - public static final String EXECUTABLE_TEST_SUITE_DELETION_ERROR = + public static final String BASIC_TEST_SUITE_DELETION_ERROR = "Cannot delete logical test suite. To delete logical test suite, use DELETE /v1/dataQuality/testSuites/<...>"; - public static final String NON_EXECUTABLE_TEST_SUITE_DELETION_ERROR = - "Cannot delete executable test suite. To delete executable test suite, use DELETE /v1/dataQuality/testSuites/executable/<...>"; + public static final String NON_BASIC_TEST_SUITE_DELETION_ERROR = + "Cannot delete executable test suite. To delete executable test suite, use DELETE /v1/dataQuality/testSuites/basic/<...>"; + public static final String BASIC_TEST_SUITE_WITHOUT_REF_ERROR = + "Cannot create a basic test suite without the BasicEntityReference field informed."; static final String FIELDS = "owners,tests,summary"; static final String SEARCH_FIELDS_EXCLUDE = "table,database,databaseSchema,service"; @@ -131,7 +133,7 @@ public ResultList list( @Parameter( description = "Returns executable or logical test suites. If omitted, returns all test suites.", - schema = @Schema(type = "string", example = "executable")) + schema = @Schema(type = "string", example = "basic")) @QueryParam("testSuiteType") String testSuiteType, @Parameter( @@ -554,10 +556,10 @@ public Response create( @Context SecurityContext securityContext, @Valid CreateTestSuite create) { create = - create.withExecutableEntityReference( + create.withBasicEntityReference( null); // entity reference is not applicable for logical test suites TestSuite testSuite = getTestSuite(create, securityContext.getUserPrincipal().getName()); - testSuite.setExecutable(false); + testSuite.setBasic(false); return create(uriInfo, securityContext, testSuite); } @@ -580,9 +582,46 @@ public Response create( public Response createExecutable( @Context UriInfo uriInfo, @Context SecurityContext securityContext, + @Context HttpServletResponse response, @Valid CreateTestSuite create) { TestSuite testSuite = getTestSuite(create, securityContext.getUserPrincipal().getName()); - testSuite.setExecutable(true); + if (testSuite.getBasicEntityReference() == null) { + throw new IllegalArgumentException(BASIC_TEST_SUITE_WITHOUT_REF_ERROR); + } + testSuite.setBasic(true); + // Set the deprecation header based on draft specification from IETF + // https://datatracker.ietf.org/doc/html/draft-ietf-httpapi-deprecation-header-02 + response.setHeader("Deprecation", "Monday, March 24, 2025"); + response.setHeader("Link", "api/v1/dataQuality/testSuites/basic; rel=\"alternate\""); + return create(uriInfo, securityContext, testSuite); + } + + @POST + @Path("/basic") + @Operation( + operationId = "createBasicTestSuite", + summary = "Create a basic test suite", + description = "Create a basic test suite.", + responses = { + @ApiResponse( + responseCode = "200", + description = "Basic test suite", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = TestSuite.class))), + @ApiResponse(responseCode = "400", description = "Bad request") + }) + public Response createBasic( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Context HttpServletResponse response, + @Valid CreateTestSuite create) { + TestSuite testSuite = getTestSuite(create, securityContext.getUserPrincipal().getName()); + if (testSuite.getBasicEntityReference() == null) { + throw new IllegalArgumentException(BASIC_TEST_SUITE_WITHOUT_REF_ERROR); + } + testSuite.setBasic(true); return create(uriInfo, securityContext, testSuite); } @@ -635,10 +674,10 @@ public Response createOrUpdate( @Context SecurityContext securityContext, @Valid CreateTestSuite create) { create = - create.withExecutableEntityReference( + create.withBasicEntityReference( null); // entity reference is not applicable for logical test suites TestSuite testSuite = getTestSuite(create, securityContext.getUserPrincipal().getName()); - testSuite.setExecutable(false); + testSuite.setBasic(false); return createOrUpdate(uriInfo, securityContext, testSuite); } @@ -659,11 +698,40 @@ public Response createOrUpdate( schema = @Schema(implementation = TestSuite.class))) }) public Response createOrUpdateExecutable( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Context HttpServletResponse response, + @Valid CreateTestSuite create) { + TestSuite testSuite = getTestSuite(create, securityContext.getUserPrincipal().getName()); + testSuite.setBasic(true); + // Set the deprecation header based on draft specification from IETF + // https://datatracker.ietf.org/doc/html/draft-ietf-httpapi-deprecation-header-02 + response.setHeader("Deprecation", "Monday, March 24, 2025"); + response.setHeader("Link", "api/v1/dataQuality/testSuites/basic; rel=\"alternate\""); + return createOrUpdate(uriInfo, securityContext, testSuite); + } + + @PUT + @Path("/basic") + @Operation( + operationId = "createOrUpdateBasicTestSuite", + summary = "Create or Update Basic test suite", + description = "Create a Basic TestSuite if it does not exist or update an existing one.", + responses = { + @ApiResponse( + responseCode = "200", + description = "The updated test definition ", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = TestSuite.class))) + }) + public Response createOrUpdateBasic( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateTestSuite create) { TestSuite testSuite = getTestSuite(create, securityContext.getUserPrincipal().getName()); - testSuite.setExecutable(true); + testSuite.setBasic(true); return createOrUpdate(uriInfo, securityContext, testSuite); } @@ -692,12 +760,12 @@ public Response delete( OperationContext operationContext = new OperationContext(entityType, MetadataOperation.DELETE); authorizer.authorize(securityContext, operationContext, getResourceContextById(id)); TestSuite testSuite = Entity.getEntity(Entity.TEST_SUITE, id, "*", ALL); - if (Boolean.TRUE.equals(testSuite.getExecutable())) { - throw new IllegalArgumentException(NON_EXECUTABLE_TEST_SUITE_DELETION_ERROR); + if (Boolean.TRUE.equals(testSuite.getBasic())) { + throw new IllegalArgumentException(NON_BASIC_TEST_SUITE_DELETION_ERROR); } RestUtil.DeleteResponse response = repository.deleteLogicalTestSuite(securityContext, testSuite, hardDelete); - repository.deleteFromSearch(response.entity(), response.changeType()); + repository.deleteFromSearch(response.entity(), hardDelete); addHref(uriInfo, response.entity()); return response.toResponse(); } @@ -727,8 +795,8 @@ public Response delete( OperationContext operationContext = new OperationContext(entityType, MetadataOperation.DELETE); authorizer.authorize(securityContext, operationContext, getResourceContextByName(name)); TestSuite testSuite = Entity.getEntityByName(Entity.TEST_SUITE, name, "*", ALL); - if (Boolean.TRUE.equals(testSuite.getExecutable())) { - throw new IllegalArgumentException(NON_EXECUTABLE_TEST_SUITE_DELETION_ERROR); + if (Boolean.TRUE.equals(testSuite.getBasic())) { + throw new IllegalArgumentException(NON_BASIC_TEST_SUITE_DELETION_ERROR); } RestUtil.DeleteResponse response = repository.deleteLogicalTestSuite(securityContext, testSuite, hardDelete); @@ -749,6 +817,47 @@ public Response delete( description = "Test suite for instance {name} is not found") }) public Response deleteExecutable( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Context HttpServletResponse response, + @Parameter( + description = "Recursively delete this entity and it's children. (Default `false`)") + @DefaultValue("false") + @QueryParam("recursive") + boolean recursive, + @Parameter(description = "Hard delete the entity. (Default = `false`)") + @QueryParam("hardDelete") + @DefaultValue("false") + boolean hardDelete, + @Parameter(description = "Name of the test suite", schema = @Schema(type = "string")) + @PathParam("name") + String name) { + OperationContext operationContext = new OperationContext(entityType, MetadataOperation.DELETE); + authorizer.authorize(securityContext, operationContext, getResourceContextByName(name)); + TestSuite testSuite = Entity.getEntityByName(Entity.TEST_SUITE, name, "*", ALL); + if (Boolean.FALSE.equals(testSuite.getBasic())) { + throw new IllegalArgumentException(BASIC_TEST_SUITE_DELETION_ERROR); + } + // Set the deprecation header based on draft specification from IETF + // https://datatracker.ietf.org/doc/html/draft-ietf-httpapi-deprecation-header-02 + response.setHeader("Deprecation", "Monday, March 24, 2025"); + response.setHeader("Link", "api/v1/dataQuality/testSuites/basic; rel=\"alternate\""); + return deleteByName(uriInfo, securityContext, name, recursive, hardDelete); + } + + @DELETE + @Path("/basic/name/{name}") + @Operation( + operationId = "deleteTestSuiteByName", + summary = "Delete a test suite", + description = "Delete a test suite by `name`.", + responses = { + @ApiResponse(responseCode = "200", description = "OK"), + @ApiResponse( + responseCode = "404", + description = "Test suite for instance {name} is not found") + }) + public Response deleteBasic( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Parameter( @@ -766,8 +875,8 @@ public Response deleteExecutable( OperationContext operationContext = new OperationContext(entityType, MetadataOperation.DELETE); authorizer.authorize(securityContext, operationContext, getResourceContextByName(name)); TestSuite testSuite = Entity.getEntityByName(Entity.TEST_SUITE, name, "*", ALL); - if (Boolean.FALSE.equals(testSuite.getExecutable())) { - throw new IllegalArgumentException(EXECUTABLE_TEST_SUITE_DELETION_ERROR); + if (Boolean.FALSE.equals(testSuite.getBasic())) { + throw new IllegalArgumentException(BASIC_TEST_SUITE_DELETION_ERROR); } return deleteByName(uriInfo, securityContext, name, recursive, hardDelete); } @@ -785,6 +894,47 @@ public Response deleteExecutable( description = "Test suite for instance {id} is not found") }) public Response deleteExecutable( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Context HttpServletResponse response, + @Parameter( + description = "Recursively delete this entity and it's children. (Default `false`)") + @DefaultValue("false") + @QueryParam("recursive") + boolean recursive, + @Parameter(description = "Hard delete the entity. (Default = `false`)") + @QueryParam("hardDelete") + @DefaultValue("false") + boolean hardDelete, + @Parameter(description = "Id of the test suite", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + OperationContext operationContext = new OperationContext(entityType, MetadataOperation.DELETE); + authorizer.authorize(securityContext, operationContext, getResourceContextById(id)); + TestSuite testSuite = Entity.getEntity(Entity.TEST_SUITE, id, "*", ALL); + if (Boolean.FALSE.equals(testSuite.getBasic())) { + throw new IllegalArgumentException(BASIC_TEST_SUITE_DELETION_ERROR); + } + // Set the deprecation header based on draft specification from IETF + // https://datatracker.ietf.org/doc/html/draft-ietf-httpapi-deprecation-header-02 + response.setHeader("Deprecation", "Monday, March 24, 2025"); + response.setHeader("Link", "api/v1/dataQuality/testSuites/basic; rel=\"alternate\""); + return delete(uriInfo, securityContext, id, recursive, hardDelete); + } + + @DELETE + @Path("/basic/{id}") + @Operation( + operationId = "deleteTestSuite", + summary = "Delete a test suite", + description = "Delete a test suite by `Id`.", + responses = { + @ApiResponse(responseCode = "200", description = "OK"), + @ApiResponse( + responseCode = "404", + description = "Test suite for instance {id} is not found") + }) + public Response deleteBasic( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Parameter( @@ -802,8 +952,8 @@ public Response deleteExecutable( OperationContext operationContext = new OperationContext(entityType, MetadataOperation.DELETE); authorizer.authorize(securityContext, operationContext, getResourceContextById(id)); TestSuite testSuite = Entity.getEntity(Entity.TEST_SUITE, id, "*", ALL); - if (Boolean.FALSE.equals(testSuite.getExecutable())) { - throw new IllegalArgumentException(EXECUTABLE_TEST_SUITE_DELETION_ERROR); + if (Boolean.FALSE.equals(testSuite.getBasic())) { + throw new IllegalArgumentException(BASIC_TEST_SUITE_DELETION_ERROR); } return delete(uriInfo, securityContext, id, recursive, hardDelete); } @@ -837,16 +987,16 @@ private TestSuite getTestSuite(CreateTestSuite create, String user) { .withDescription(create.getDescription()) .withDisplayName(create.getDisplayName()) .withName(create.getName()); - if (create.getExecutableEntityReference() != null) { + if (create.getBasicEntityReference() != null) { Table table = - Entity.getEntityByName(Entity.TABLE, create.getExecutableEntityReference(), null, null); + Entity.getEntityByName(Entity.TABLE, create.getBasicEntityReference(), null, null); EntityReference entityReference = new EntityReference() .withId(table.getId()) .withFullyQualifiedName(table.getFullyQualifiedName()) .withName(table.getName()) .withType(Entity.TABLE); - testSuite.setExecutableEntityReference(entityReference); + testSuite.setBasicEntityReference(entityReference); } return testSuite; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/events/subscription/EventSubscriptionMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/events/subscription/EventSubscriptionMapper.java new file mode 100644 index 000000000000..2892c2488030 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/events/subscription/EventSubscriptionMapper.java @@ -0,0 +1,46 @@ +package org.openmetadata.service.resources.events.subscription; + +import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; +import static org.openmetadata.service.events.subscription.AlertUtil.validateAndBuildFilteringConditions; +import static org.openmetadata.service.fernet.Fernet.encryptWebhookSecretKey; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import org.openmetadata.schema.api.events.CreateEventSubscription; +import org.openmetadata.schema.entity.events.EventSubscription; +import org.openmetadata.schema.entity.events.SubscriptionDestination; +import org.openmetadata.service.mapper.EntityMapper; + +public class EventSubscriptionMapper + implements EntityMapper { + @Override + public EventSubscription createToEntity(CreateEventSubscription create, String user) { + return copy(new EventSubscription(), create, user) + .withAlertType(create.getAlertType()) + .withTrigger(create.getTrigger()) + .withEnabled(create.getEnabled()) + .withBatchSize(create.getBatchSize()) + .withFilteringRules( + validateAndBuildFilteringConditions( + create.getResources(), create.getAlertType(), create.getInput())) + .withDestinations(encryptWebhookSecretKey(getSubscriptions(create.getDestinations()))) + .withProvider(create.getProvider()) + .withRetries(create.getRetries()) + .withPollInterval(create.getPollInterval()) + .withInput(create.getInput()); + } + + private List getSubscriptions( + List subscriptions) { + List result = new ArrayList<>(); + subscriptions.forEach( + subscription -> { + if (nullOrEmpty(subscription.getId())) { + subscription.withId(UUID.randomUUID()); + } + result.add(subscription); + }); + return result; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/events/subscription/EventSubscriptionResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/events/subscription/EventSubscriptionResource.java index 51d943faa3b4..2d455581519b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/events/subscription/EventSubscriptionResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/events/subscription/EventSubscriptionResource.java @@ -16,8 +16,6 @@ import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; import static org.openmetadata.schema.api.events.CreateEventSubscription.AlertType.NOTIFICATION; -import static org.openmetadata.service.events.subscription.AlertUtil.validateAndBuildFilteringConditions; -import static org.openmetadata.service.fernet.Fernet.encryptWebhookSecretKey; import io.swagger.v3.oas.annotations.ExternalDocumentation; import io.swagger.v3.oas.annotations.Operation; @@ -108,6 +106,7 @@ public class EventSubscriptionResource extends EntityResource { public static final String COLLECTION_PATH = "/v1/events/subscriptions"; public static final String FIELDS = "owners,filteringRules"; + private final EventSubscriptionMapper mapper = new EventSubscriptionMapper(); public EventSubscriptionResource(Authorizer authorizer, Limits limits) { super(Entity.EVENT_SUBSCRIPTION, authorizer, limits); @@ -296,7 +295,7 @@ public Response createEventSubscription( @Valid CreateEventSubscription request) throws SchedulerException { EventSubscription eventSub = - getEventSubscription(request, securityContext.getUserPrincipal().getName()); + mapper.createToEntity(request, securityContext.getUserPrincipal().getName()); // Only one Creation is allowed Response response = create(uriInfo, securityContext, eventSub); EventSubscriptionScheduler.getInstance().addSubscriptionPublisher(eventSub); @@ -323,7 +322,7 @@ public Response createOrUpdateEventSubscription( @Context SecurityContext securityContext, @Valid CreateEventSubscription create) { EventSubscription eventSub = - getEventSubscription(create, securityContext.getUserPrincipal().getName()); + mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); Response response = createOrUpdate(uriInfo, securityContext, eventSub); EventSubscriptionScheduler.getInstance() .updateEventSubscription((EventSubscription) response.getEntity()); @@ -534,8 +533,6 @@ public SubscriptionStatus getEventSubscriptionStatusByName( OperationContext operationContext = new OperationContext(entityType, MetadataOperation.VIEW_ALL); authorizer.authorize(securityContext, operationContext, getResourceContextByName(name)); - - authorizer.authorizeAdmin(securityContext); EventSubscription sub = repository.getByName(null, name, repository.getFields("name")); return EventSubscriptionScheduler.getInstance() .getStatusForEventSubscription(sub.getId(), destinationId); @@ -1314,6 +1311,39 @@ public List getAllDestinationStatusesForSubscriptionByN return EventSubscriptionScheduler.getInstance().listAlertDestinations(sub.getId()); } + @PUT + @Path("name/{eventSubscriptionName}/syncOffset") + @Valid + @Operation( + operationId = "syncOffsetForEventSubscriptionByName", + summary = "Sync Offset for a specific Event Subscription by its name", + description = "Sync Offset for a specific Event Subscription by its name", + responses = { + @ApiResponse( + responseCode = "200", + description = "Returns the destinations for the Event Subscription", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = SubscriptionDestination.class))), + @ApiResponse( + responseCode = "404", + description = "Event Subscription with the name {fqn} is not found") + }) + public Response syncOffsetForEventSubscription( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Name of the Event Subscription", schema = @Schema(type = "string")) + @PathParam("eventSubscriptionName") + String name) { + OperationContext operationContext = + new OperationContext(entityType, MetadataOperation.EDIT_ALL); + authorizer.authorize(securityContext, operationContext, getResourceContextByName(name)); + return Response.status(Response.Status.OK) + .entity(repository.syncEventSubscriptionOffset(name)) + .build(); + } + @POST @Path("/testDestination") @Operation( @@ -1354,36 +1384,6 @@ public Response sendTestMessageAlert( return Response.ok(resultDestinations).build(); } - private EventSubscription getEventSubscription(CreateEventSubscription create, String user) { - return repository - .copy(new EventSubscription(), create, user) - .withAlertType(create.getAlertType()) - .withTrigger(create.getTrigger()) - .withEnabled(create.getEnabled()) - .withBatchSize(create.getBatchSize()) - .withFilteringRules( - validateAndBuildFilteringConditions( - create.getResources(), create.getAlertType(), create.getInput())) - .withDestinations(encryptWebhookSecretKey(getSubscriptions(create.getDestinations()))) - .withProvider(create.getProvider()) - .withRetries(create.getRetries()) - .withPollInterval(create.getPollInterval()) - .withInput(create.getInput()); - } - - private List getSubscriptions( - List subscriptions) { - List result = new ArrayList<>(); - subscriptions.forEach( - subscription -> { - if (nullOrEmpty(subscription.getId())) { - subscription.withId(UUID.randomUUID()); - } - result.add(subscription); - }); - return result; - } - public static List getNotificationsFilterDescriptors() throws IOException { List entityNotificationDescriptors = diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/feeds/FeedMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/feeds/FeedMapper.java new file mode 100644 index 000000000000..541096507d11 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/feeds/FeedMapper.java @@ -0,0 +1,56 @@ +package org.openmetadata.service.resources.feeds; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import org.openmetadata.schema.api.CreateTaskDetails; +import org.openmetadata.schema.api.feed.CreateThread; +import org.openmetadata.schema.entity.feed.Thread; +import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.type.TaskDetails; +import org.openmetadata.schema.type.TaskStatus; +import org.openmetadata.service.Entity; + +public class FeedMapper { + public Thread createToEntity(CreateThread create, String user) { + UUID randomUUID = UUID.randomUUID(); + return new Thread() + .withId(randomUUID) + .withThreadTs(System.currentTimeMillis()) + .withMessage(create.getMessage()) + .withCreatedBy(create.getFrom()) + .withAbout(create.getAbout()) + .withAddressedTo(create.getAddressedTo()) + .withReactions(Collections.emptyList()) + .withType(create.getType()) + .withTask(getTaskDetails(create.getTaskDetails())) + .withAnnouncement(create.getAnnouncementDetails()) + .withChatbot(create.getChatbotDetails()) + .withUpdatedBy(user) + .withUpdatedAt(System.currentTimeMillis()) + .withEntityRef(new EntityReference().withId(randomUUID).withType(Entity.THREAD)) + .withGeneratedBy(Thread.GeneratedBy.USER); + } + + private TaskDetails getTaskDetails(CreateTaskDetails create) { + if (create != null) { + return new TaskDetails() + .withAssignees(formatAssignees(create.getAssignees())) + .withType(create.getType()) + .withStatus(TaskStatus.Open) + .withOldValue(create.getOldValue()) + .withSuggestion(create.getSuggestion()); + } + return null; + } + + public static List formatAssignees(List assignees) { + List result = new ArrayList<>(); + assignees.forEach( + assignee -> + result.add( + new EntityReference().withId(assignee.getId()).withType(assignee.getType()))); + return result; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/feeds/FeedResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/feeds/FeedResource.java index a1f6565bfdaa..b998185addcd 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/feeds/FeedResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/feeds/FeedResource.java @@ -28,8 +28,6 @@ import io.swagger.v3.oas.annotations.parameters.RequestBody; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.tags.Tag; -import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.UUID; import javax.json.JsonPatch; @@ -52,7 +50,6 @@ import javax.ws.rs.core.Response; import javax.ws.rs.core.SecurityContext; import javax.ws.rs.core.UriInfo; -import org.openmetadata.schema.api.CreateTaskDetails; import org.openmetadata.schema.api.feed.CloseTask; import org.openmetadata.schema.api.feed.CreatePost; import org.openmetadata.schema.api.feed.CreateThread; @@ -62,7 +59,6 @@ import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.MetadataOperation; import org.openmetadata.schema.type.Post; -import org.openmetadata.schema.type.TaskDetails; import org.openmetadata.schema.type.TaskStatus; import org.openmetadata.schema.type.ThreadType; import org.openmetadata.service.Entity; @@ -90,6 +86,8 @@ @Collection(name = "feeds") public class FeedResource { public static final String COLLECTION_PATH = "/v1/feed/"; + private final FeedMapper mapper = new FeedMapper(); + private final PostMapper postMapper = new PostMapper(); private final FeedRepository dao; private final Authorizer authorizer; @@ -421,7 +419,7 @@ public Response createThread( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateThread create) { - Thread thread = getThread(securityContext, create); + Thread thread = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); addHref(uriInfo, dao.create(thread)); return Response.created(thread.getHref()) .entity(thread) @@ -452,7 +450,7 @@ public Response addPost( @PathParam("id") UUID id, @Valid CreatePost createPost) { - Post post = getPost(createPost); + Post post = postMapper.createToEntity(createPost, securityContext.getUserPrincipal().getName()); Thread thread = addHref( uriInfo, dao.addPostToThread(id, post, securityContext.getUserPrincipal().getName())); @@ -588,54 +586,4 @@ public ResultList getPosts( UUID id) { return new ResultList<>(dao.listPosts(id)); } - - private Thread getThread(SecurityContext securityContext, CreateThread create) { - UUID randomUUID = UUID.randomUUID(); - return new Thread() - .withId(randomUUID) - .withThreadTs(System.currentTimeMillis()) - .withMessage(create.getMessage()) - .withCreatedBy(create.getFrom()) - .withAbout(create.getAbout()) - .withAddressedTo(create.getAddressedTo()) - .withReactions(Collections.emptyList()) - .withType(create.getType()) - .withTask(getTaskDetails(create.getTaskDetails())) - .withAnnouncement(create.getAnnouncementDetails()) - .withChatbot(create.getChatbotDetails()) - .withUpdatedBy(securityContext.getUserPrincipal().getName()) - .withUpdatedAt(System.currentTimeMillis()) - .withEntityRef(new EntityReference().withId(randomUUID).withType(Entity.THREAD)) - .withGeneratedBy(Thread.GeneratedBy.USER); - } - - private Post getPost(CreatePost create) { - return new Post() - .withId(UUID.randomUUID()) - .withMessage(create.getMessage()) - .withFrom(create.getFrom()) - .withReactions(Collections.emptyList()) - .withPostTs(System.currentTimeMillis()); - } - - private TaskDetails getTaskDetails(CreateTaskDetails create) { - if (create != null) { - return new TaskDetails() - .withAssignees(formatAssignees(create.getAssignees())) - .withType(create.getType()) - .withStatus(TaskStatus.Open) - .withOldValue(create.getOldValue()) - .withSuggestion(create.getSuggestion()); - } - return null; - } - - public static List formatAssignees(List assignees) { - List result = new ArrayList<>(); - assignees.forEach( - assignee -> - result.add( - new EntityReference().withId(assignee.getId()).withType(assignee.getType()))); - return result; - } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/feeds/PostMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/feeds/PostMapper.java new file mode 100644 index 000000000000..2e281168f9d8 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/feeds/PostMapper.java @@ -0,0 +1,17 @@ +package org.openmetadata.service.resources.feeds; + +import java.util.Collections; +import java.util.UUID; +import org.openmetadata.schema.api.feed.CreatePost; +import org.openmetadata.schema.type.Post; + +public class PostMapper { + public Post createToEntity(CreatePost create, String user) { + return new Post() + .withId(UUID.randomUUID()) + .withMessage(create.getMessage()) + .withFrom(create.getFrom()) + .withReactions(Collections.emptyList()) + .withPostTs(System.currentTimeMillis()); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/feeds/SuggestionMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/feeds/SuggestionMapper.java new file mode 100644 index 000000000000..c032cfa578a3 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/feeds/SuggestionMapper.java @@ -0,0 +1,72 @@ +package org.openmetadata.service.resources.feeds; + +import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; + +import java.util.UUID; +import javax.ws.rs.core.Response; +import org.openmetadata.schema.api.feed.CreateSuggestion; +import org.openmetadata.schema.entity.feed.Suggestion; +import org.openmetadata.schema.type.Include; +import org.openmetadata.schema.type.SuggestionStatus; +import org.openmetadata.schema.type.SuggestionType; +import org.openmetadata.schema.type.TagLabel; +import org.openmetadata.sdk.exception.SuggestionException; +import org.openmetadata.service.Entity; +import org.openmetadata.service.resources.tags.TagLabelUtil; +import org.openmetadata.service.util.UserUtil; + +public class SuggestionMapper { + private final String INVALID_SUGGESTION_REQUEST = "INVALID_SUGGESTION_REQUEST"; + + public Suggestion createToEntity(CreateSuggestion create, String user) { + validate(create); + return new Suggestion() + .withId(UUID.randomUUID()) + .withDescription(create.getDescription()) + .withEntityLink(create.getEntityLink()) + .withType(create.getType()) + .withDescription(create.getDescription()) + .withTagLabels(create.getTagLabels()) + .withStatus(SuggestionStatus.Open) + .withCreatedBy(UserUtil.getUserOrBot(user)) + .withCreatedAt(System.currentTimeMillis()) + .withUpdatedBy(user) + .withUpdatedAt(System.currentTimeMillis()); + } + + private void validate(CreateSuggestion suggestion) { + if (suggestion.getEntityLink() == null) { + throw new SuggestionException( + Response.Status.BAD_REQUEST, + INVALID_SUGGESTION_REQUEST, + "Suggestion's entityLink cannot be null."); + } + MessageParser.EntityLink entityLink = + MessageParser.EntityLink.parse(suggestion.getEntityLink()); + Entity.getEntityReferenceByName( + entityLink.getEntityType(), entityLink.getEntityFQN(), Include.NON_DELETED); + + if (suggestion.getType() == SuggestionType.SuggestDescription) { + if (suggestion.getDescription() == null || suggestion.getDescription().isEmpty()) { + throw new SuggestionException( + Response.Status.BAD_REQUEST, + INVALID_SUGGESTION_REQUEST, + "Suggestion's description cannot be empty."); + } + } else if (suggestion.getType() == SuggestionType.SuggestTagLabel) { + if (suggestion.getTagLabels().isEmpty()) { + throw new SuggestionException( + Response.Status.BAD_REQUEST, + INVALID_SUGGESTION_REQUEST, + "Suggestion's tag label's cannot be empty."); + } else { + for (TagLabel label : listOrEmpty(suggestion.getTagLabels())) { + TagLabelUtil.applyTagCommonFields(label); + } + } + } else { + throw new SuggestionException( + Response.Status.BAD_REQUEST, INVALID_SUGGESTION_REQUEST, "Invalid Suggestion Type."); + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/feeds/SuggestionsResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/feeds/SuggestionsResource.java index 8a411075783e..131493dd36b5 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/feeds/SuggestionsResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/feeds/SuggestionsResource.java @@ -13,7 +13,6 @@ package org.openmetadata.service.resources.feeds; -import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; import static org.openmetadata.schema.type.EventType.SUGGESTION_CREATED; import static org.openmetadata.schema.type.EventType.SUGGESTION_REJECTED; @@ -54,20 +53,16 @@ import org.openmetadata.schema.type.MetadataOperation; import org.openmetadata.schema.type.SuggestionStatus; import org.openmetadata.schema.type.SuggestionType; -import org.openmetadata.schema.type.TagLabel; -import org.openmetadata.sdk.exception.SuggestionException; import org.openmetadata.service.Entity; import org.openmetadata.service.jdbi3.SuggestionFilter; import org.openmetadata.service.jdbi3.SuggestionRepository; import org.openmetadata.service.resources.Collection; -import org.openmetadata.service.resources.tags.TagLabelUtil; import org.openmetadata.service.security.Authorizer; import org.openmetadata.service.security.policyevaluator.OperationContext; import org.openmetadata.service.security.policyevaluator.PostResourceContext; import org.openmetadata.service.security.policyevaluator.ResourceContextInterface; import org.openmetadata.service.util.RestUtil; import org.openmetadata.service.util.ResultList; -import org.openmetadata.service.util.UserUtil; @Path("/v1/suggestions") @Tag( @@ -79,9 +74,9 @@ @Collection(name = "suggestions") public class SuggestionsResource { public static final String COLLECTION_PATH = "/v1/suggestions/"; + private final SuggestionMapper mapper = new SuggestionMapper(); private final SuggestionRepository dao; private final Authorizer authorizer; - private final String INVALID_SUGGESTION_REQUEST = "INVALID_SUGGESTION_REQUEST"; public static void addHref(UriInfo uriInfo, List suggestions) { if (uriInfo != null) { @@ -410,7 +405,8 @@ public Response createSuggestion( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateSuggestion create) { - Suggestion suggestion = getSuggestion(securityContext, create); + Suggestion suggestion = + mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); addHref(uriInfo, dao.create(suggestion)); return Response.created(suggestion.getHref()) .entity(suggestion) @@ -479,56 +475,4 @@ public Response deleteSuggestions( return dao.deleteSuggestionsForAnEntity(entity, securityContext.getUserPrincipal().getName()) .toResponse(); } - - private Suggestion getSuggestion(SecurityContext securityContext, CreateSuggestion create) { - validate(create); - return new Suggestion() - .withId(UUID.randomUUID()) - .withDescription(create.getDescription()) - .withEntityLink(create.getEntityLink()) - .withType(create.getType()) - .withDescription(create.getDescription()) - .withTagLabels(create.getTagLabels()) - .withStatus(SuggestionStatus.Open) - .withCreatedBy(UserUtil.getUserOrBot(securityContext.getUserPrincipal().getName())) - .withCreatedAt(System.currentTimeMillis()) - .withUpdatedBy(securityContext.getUserPrincipal().getName()) - .withUpdatedAt(System.currentTimeMillis()); - } - - private void validate(CreateSuggestion suggestion) { - if (suggestion.getEntityLink() == null) { - throw new SuggestionException( - Response.Status.BAD_REQUEST, - INVALID_SUGGESTION_REQUEST, - "Suggestion's entityLink cannot be null."); - } - MessageParser.EntityLink entityLink = - MessageParser.EntityLink.parse(suggestion.getEntityLink()); - Entity.getEntityReferenceByName( - entityLink.getEntityType(), entityLink.getEntityFQN(), Include.NON_DELETED); - - if (suggestion.getType() == SuggestionType.SuggestDescription) { - if (suggestion.getDescription() == null || suggestion.getDescription().isEmpty()) { - throw new SuggestionException( - Response.Status.BAD_REQUEST, - INVALID_SUGGESTION_REQUEST, - "Suggestion's description cannot be empty."); - } - } else if (suggestion.getType() == SuggestionType.SuggestTagLabel) { - if (suggestion.getTagLabels().isEmpty()) { - throw new SuggestionException( - Response.Status.BAD_REQUEST, - INVALID_SUGGESTION_REQUEST, - "Suggestion's tag label's cannot be empty."); - } else { - for (TagLabel label : listOrEmpty(suggestion.getTagLabels())) { - TagLabelUtil.applyTagCommonFields(label); - } - } - } else { - throw new SuggestionException( - Response.Status.BAD_REQUEST, INVALID_SUGGESTION_REQUEST, "Invalid Suggestion Type."); - } - } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryMapper.java new file mode 100644 index 000000000000..0a5a6f950d97 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryMapper.java @@ -0,0 +1,14 @@ +package org.openmetadata.service.resources.glossary; + +import org.openmetadata.schema.api.data.CreateGlossary; +import org.openmetadata.schema.entity.data.Glossary; +import org.openmetadata.service.mapper.EntityMapper; + +public class GlossaryMapper implements EntityMapper { + @Override + public Glossary createToEntity(CreateGlossary create, String user) { + return copy(new Glossary(), create, user) + .withProvider(create.getProvider()) + .withMutuallyExclusive(create.getMutuallyExclusive()); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryResource.java index 2947dd330924..169a30a34c7a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryResource.java @@ -79,6 +79,7 @@ public class GlossaryResource extends EntityResource { public static final String COLLECTION_PATH = "v1/glossaries/"; static final String FIELDS = "owners,tags,reviewers,usageCount,termCount,domain,extension"; + private final GlossaryMapper mapper = new GlossaryMapper(); public GlossaryResource(Authorizer authorizer, Limits limits) { super(Entity.GLOSSARY, authorizer, limits); @@ -296,7 +297,7 @@ public Response create( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateGlossary create) { - Glossary glossary = getGlossary(create, securityContext.getUserPrincipal().getName()); + Glossary glossary = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return create(uriInfo, securityContext, glossary); } @@ -377,7 +378,7 @@ public Response createOrUpdate( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateGlossary create) { - Glossary glossary = getGlossary(create, securityContext.getUserPrincipal().getName()); + Glossary glossary = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return createOrUpdate(uriInfo, securityContext, glossary); } @@ -607,16 +608,4 @@ public Response importCsvAsync( boolean dryRun) { return importCsvInternalAsync(securityContext, name, csv, dryRun); } - - private Glossary getGlossary(CreateGlossary create, String user) { - return getGlossary(repository, create, user); - } - - public static Glossary getGlossary( - GlossaryRepository repository, CreateGlossary create, String updatedBy) { - return repository - .copy(new Glossary(), create, updatedBy) - .withProvider(create.getProvider()) - .withMutuallyExclusive(create.getMutuallyExclusive()); - } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryTermMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryTermMapper.java new file mode 100644 index 000000000000..47c6e494725d --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryTermMapper.java @@ -0,0 +1,24 @@ +package org.openmetadata.service.resources.glossary; + +import static org.openmetadata.service.util.EntityUtil.getEntityReference; +import static org.openmetadata.service.util.EntityUtil.getEntityReferences; + +import org.openmetadata.schema.api.data.CreateGlossaryTerm; +import org.openmetadata.schema.entity.data.GlossaryTerm; +import org.openmetadata.service.Entity; +import org.openmetadata.service.mapper.EntityMapper; + +public class GlossaryTermMapper implements EntityMapper { + @Override + public GlossaryTerm createToEntity(CreateGlossaryTerm create, String user) { + return copy(new GlossaryTerm(), create, user) + .withSynonyms(create.getSynonyms()) + .withStyle(create.getStyle()) + .withGlossary(getEntityReference(Entity.GLOSSARY, create.getGlossary())) + .withParent(getEntityReference(Entity.GLOSSARY_TERM, create.getParent())) + .withRelatedTerms(getEntityReferences(Entity.GLOSSARY_TERM, create.getRelatedTerms())) + .withReferences(create.getReferences()) + .withProvider(create.getProvider()) + .withMutuallyExclusive(create.getMutuallyExclusive()); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryTermResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryTermResource.java index e7673154bc39..11ffd396f8b9 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryTermResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryTermResource.java @@ -88,6 +88,8 @@ name = "glossaryTerms", order = 7) // Initialized after Glossary, Classification, and Tags public class GlossaryTermResource extends EntityResource { + private final GlossaryTermMapper mapper = new GlossaryTermMapper(); + private final GlossaryMapper glossaryMapper = new GlossaryMapper(); public static final String COLLECTION_PATH = "v1/glossaryTerms/"; static final String FIELDS = "children,relatedTerms,reviewers,owners,tags,usageCount,domain,extension,childrenCount"; @@ -126,8 +128,7 @@ public void initialize(OpenMetadataApplicationConfig config) throws IOException GLOSSARY, ".*json/data/glossary/.*Glossary\\.json$", LoadGlossary.class); for (LoadGlossary loadGlossary : loadGlossaries) { Glossary glossary = - GlossaryResource.getGlossary( - glossaryRepository, loadGlossary.getCreateGlossary(), ADMIN_USER_NAME); + glossaryMapper.createToEntity(loadGlossary.getCreateGlossary(), ADMIN_USER_NAME); glossary.setFullyQualifiedName(glossary.getName()); glossaryRepository.initializeEntity(glossary); @@ -135,7 +136,7 @@ public void initialize(OpenMetadataApplicationConfig config) throws IOException for (CreateGlossaryTerm createTerm : loadGlossary.getCreateTerms()) { createTerm.withGlossary(glossary.getName()); createTerm.withProvider(glossary.getProvider()); - GlossaryTerm term = getGlossaryTerm(createTerm, ADMIN_USER_NAME); + GlossaryTerm term = mapper.createToEntity(createTerm, ADMIN_USER_NAME); repository.setFullyQualifiedName(term); // FQN required for ordering tags based on hierarchy termsToCreate.add(term); } @@ -411,7 +412,7 @@ public Response create( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateGlossaryTerm create) { - GlossaryTerm term = getGlossaryTerm(create, securityContext.getUserPrincipal().getName()); + GlossaryTerm term = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return create(uriInfo, securityContext, term); } @@ -493,7 +494,7 @@ public Response createOrUpdate( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateGlossaryTerm create) { - GlossaryTerm term = getGlossaryTerm(create, securityContext.getUserPrincipal().getName()); + GlossaryTerm term = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return createOrUpdate(uriInfo, securityContext, term); } @@ -681,17 +682,4 @@ public Response restoreTable( @Valid RestoreEntity restore) { return restoreEntity(uriInfo, securityContext, restore.getId()); } - - private GlossaryTerm getGlossaryTerm(CreateGlossaryTerm create, String user) { - return repository - .copy(new GlossaryTerm(), create, user) - .withSynonyms(create.getSynonyms()) - .withStyle(create.getStyle()) - .withGlossary(getEntityReference(Entity.GLOSSARY, create.getGlossary())) - .withParent(getEntityReference(Entity.GLOSSARY_TERM, create.getParent())) - .withRelatedTerms(getEntityReferences(Entity.GLOSSARY_TERM, create.getRelatedTerms())) - .withReferences(create.getReferences()) - .withProvider(create.getProvider()) - .withMutuallyExclusive(create.getMutuallyExclusive()); - } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/governance/WorkflowDefinitionMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/governance/WorkflowDefinitionMapper.java new file mode 100644 index 000000000000..452ca49309f4 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/governance/WorkflowDefinitionMapper.java @@ -0,0 +1,18 @@ +package org.openmetadata.service.resources.governance; + +import org.openmetadata.schema.api.governance.CreateWorkflowDefinition; +import org.openmetadata.schema.governance.workflows.WorkflowDefinition; +import org.openmetadata.service.mapper.EntityMapper; + +public class WorkflowDefinitionMapper + implements EntityMapper { + @Override + public WorkflowDefinition createToEntity(CreateWorkflowDefinition create, String user) { + return copy(new WorkflowDefinition(), create, user) + .withFullyQualifiedName(create.getName()) + .withType(WorkflowDefinition.Type.fromValue(create.getType().toString())) + .withTrigger(create.getTrigger()) + .withNodes(create.getNodes()) + .withEdges(create.getEdges()); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/governance/WorkflowDefinitionResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/governance/WorkflowDefinitionResource.java index 7cbc8e066a97..73832c1313e9 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/governance/WorkflowDefinitionResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/governance/WorkflowDefinitionResource.java @@ -59,6 +59,7 @@ public class WorkflowDefinitionResource extends EntityResource { public static final String COLLECTION_PATH = "v1/governance/workflowDefinitions/"; static final String FIELDS = "owners"; + private final WorkflowDefinitionMapper mapper = new WorkflowDefinitionMapper(); public WorkflowDefinitionResource(Authorizer authorizer, Limits limits) { super(Entity.WORKFLOW_DEFINITION, authorizer, limits); @@ -70,7 +71,6 @@ public static class WorkflowDefinitionList extends ResultList { + @Override + public Kpi createToEntity(CreateKpiRequest create, String user) { + return copy(new Kpi(), create, user) + .withStartDate(create.getStartDate()) + .withEndDate(create.getEndDate()) + .withTargetValue(create.getTargetValue()) + .withDataInsightChart( + getEntityReference( + Entity.DATA_INSIGHT_CUSTOM_CHART, create.getDataInsightChart().value())) + .withMetricType(create.getMetricType()); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/kpi/KpiResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/kpi/KpiResource.java index d47001dc6402..0f2dcdd69788 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/kpi/KpiResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/kpi/KpiResource.java @@ -62,6 +62,7 @@ @Collection(name = "kpi") public class KpiResource extends EntityResource { public static final String COLLECTION_PATH = "/v1/kpi"; + private final KpiMapper mapper = new KpiMapper(); static final String FIELDS = "owners,dataInsightChart,kpiResult"; @Override @@ -284,7 +285,7 @@ public Response create( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateKpiRequest create) { - Kpi kpi = getKpi(create, securityContext.getUserPrincipal().getName()); + Kpi kpi = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); // TODO fix this // dao.validateDataInsightChartOneToOneMapping(kpi.getDataInsightChart().getId()); return create(uriInfo, securityContext, kpi); @@ -365,7 +366,7 @@ public Response createOrUpdate( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateKpiRequest create) { - Kpi kpi = getKpi(create, securityContext.getUserPrincipal().getName()); + Kpi kpi = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return createOrUpdate(uriInfo, securityContext, kpi); } @@ -507,16 +508,4 @@ public KpiResult listKpiResults( String name) { return repository.getKpiResult(name); } - - private Kpi getKpi(CreateKpiRequest create, String user) { - return repository - .copy(new Kpi(), create, user) - .withStartDate(create.getStartDate()) - .withEndDate(create.getEndDate()) - .withTargetValue(create.getTargetValue()) - .withDataInsightChart( - getEntityReference( - Entity.DATA_INSIGHT_CUSTOM_CHART, create.getDataInsightChart().value())) - .withMetricType(create.getMetricType()); - } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/lineage/LineageResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/lineage/LineageResource.java index b5ace8f9ab59..cd71dfe5fe8d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/lineage/LineageResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/lineage/LineageResource.java @@ -60,6 +60,7 @@ import org.openmetadata.service.resources.Collection; import org.openmetadata.service.security.Authorizer; import org.openmetadata.service.security.policyevaluator.OperationContext; +import org.openmetadata.service.security.policyevaluator.ResourceContext; import org.openmetadata.service.security.policyevaluator.ResourceContextInterface; import org.openmetadata.service.util.AsyncService; import org.openmetadata.service.util.CSVExportMessage; @@ -349,8 +350,20 @@ public Response addLineage( @Valid AddLineage addLineage) { authorizer.authorize( securityContext, - new OperationContext(LINEAGE_FIELD, MetadataOperation.EDIT_LINEAGE), - new LineageResourceContext()); + new OperationContext( + addLineage.getEdge().getFromEntity().getType(), MetadataOperation.EDIT_LINEAGE), + new ResourceContext<>( + addLineage.getEdge().getFromEntity().getType(), + addLineage.getEdge().getFromEntity().getId(), + addLineage.getEdge().getFromEntity().getName())); + authorizer.authorize( + securityContext, + new OperationContext( + addLineage.getEdge().getToEntity().getType(), MetadataOperation.EDIT_LINEAGE), + new ResourceContext<>( + addLineage.getEdge().getToEntity().getType(), + addLineage.getEdge().getToEntity().getId(), + addLineage.getEdge().getToEntity().getName())); dao.addLineage(addLineage); return Response.status(Status.OK).build(); } @@ -426,8 +439,12 @@ public Response patchLineageEdge( JsonPatch patch) { authorizer.authorize( securityContext, - new OperationContext(LINEAGE_FIELD, MetadataOperation.EDIT_LINEAGE), - new LineageResourceContext()); + new OperationContext(fromEntity, MetadataOperation.EDIT_LINEAGE), + new ResourceContext<>(fromEntity, fromId, null)); + authorizer.authorize( + securityContext, + new OperationContext(toEntity, MetadataOperation.EDIT_LINEAGE), + new ResourceContext<>(toEntity, toId, null)); return dao.patchLineageEdge(fromEntity, fromId, toEntity, toId, patch); } @@ -467,8 +484,12 @@ public Response deleteLineage( String toId) { authorizer.authorize( securityContext, - new OperationContext(LINEAGE_FIELD, MetadataOperation.EDIT_LINEAGE), - new LineageResourceContext()); + new OperationContext(fromEntity, MetadataOperation.EDIT_LINEAGE), + new ResourceContext<>(fromEntity, UUID.fromString(fromId), null)); + authorizer.authorize( + securityContext, + new OperationContext(toEntity, MetadataOperation.EDIT_LINEAGE), + new ResourceContext<>(toEntity, UUID.fromString(toId), null)); boolean deleted = dao.deleteLineage(fromEntity, fromId, toEntity, toId); if (!deleted) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/metrics/MetricMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/metrics/MetricMapper.java new file mode 100644 index 000000000000..473783f50634 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/metrics/MetricMapper.java @@ -0,0 +1,20 @@ +package org.openmetadata.service.resources.metrics; + +import static org.openmetadata.service.util.EntityUtil.getEntityReferences; + +import org.openmetadata.schema.api.data.CreateMetric; +import org.openmetadata.schema.entity.data.Metric; +import org.openmetadata.service.Entity; +import org.openmetadata.service.mapper.EntityMapper; + +public class MetricMapper implements EntityMapper { + @Override + public Metric createToEntity(CreateMetric create, String user) { + return copy(new Metric(), create, user) + .withMetricExpression(create.getMetricExpression()) + .withGranularity(create.getGranularity()) + .withRelatedMetrics(getEntityReferences(Entity.METRIC, create.getRelatedMetrics())) + .withMetricType(create.getMetricType()) + .withUnitOfMeasurement(create.getUnitOfMeasurement()); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/metrics/MetricResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/metrics/MetricResource.java index 4be491b81a58..b2e644498d7e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/metrics/MetricResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/metrics/MetricResource.java @@ -73,6 +73,7 @@ @Collection(name = "metrics") public class MetricResource extends EntityResource { public static final String COLLECTION_PATH = "v1/metrics/"; + private final MetricMapper mapper = new MetricMapper(); static final String FIELDS = "owners,relatedMetrics,followers,tags,extension,domain,dataProducts"; public MetricResource(Authorizer authorizer, Limits limits) { @@ -279,7 +280,7 @@ public Response create( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateMetric create) { - Metric metric = getMetric(create, securityContext.getUserPrincipal().getName()); + Metric metric = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return create(uriInfo, securityContext, metric); } @@ -302,7 +303,7 @@ public Response createOrUpdate( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateMetric create) { - Metric metric = getMetric(create, securityContext.getUserPrincipal().getName()); + Metric metric = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return createOrUpdate(uriInfo, securityContext, metric); } @@ -519,14 +520,4 @@ public Response restore( @Valid RestoreEntity restore) { return restoreEntity(uriInfo, securityContext, restore.getId()); } - - private Metric getMetric(CreateMetric create, String user) { - return repository - .copy(new Metric(), create, user) - .withMetricExpression(create.getMetricExpression()) - .withGranularity(create.getGranularity()) - .withRelatedMetrics(getEntityReferences(Entity.METRIC, create.getRelatedMetrics())) - .withMetricType(create.getMetricType()) - .withUnitOfMeasurement(create.getUnitOfMeasurement()); - } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/mlmodels/MlModelMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/mlmodels/MlModelMapper.java new file mode 100644 index 000000000000..29f3358bf827 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/mlmodels/MlModelMapper.java @@ -0,0 +1,25 @@ +package org.openmetadata.service.resources.mlmodels; + +import static org.openmetadata.service.util.EntityUtil.getEntityReference; + +import org.openmetadata.schema.api.data.CreateMlModel; +import org.openmetadata.schema.entity.data.MlModel; +import org.openmetadata.service.Entity; +import org.openmetadata.service.mapper.EntityMapper; + +public class MlModelMapper implements EntityMapper { + @Override + public MlModel createToEntity(CreateMlModel create, String user) { + return copy(new MlModel(), create, user) + .withService(getEntityReference(Entity.MLMODEL_SERVICE, create.getService())) + .withDashboard(getEntityReference(Entity.DASHBOARD, create.getDashboard())) + .withAlgorithm(create.getAlgorithm()) + .withMlFeatures(create.getMlFeatures()) + .withMlHyperParameters(create.getMlHyperParameters()) + .withMlStore(create.getMlStore()) + .withServer(create.getServer()) + .withTarget(create.getTarget()) + .withSourceUrl(create.getSourceUrl()) + .withSourceHash(create.getSourceHash()); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/mlmodels/MlModelResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/mlmodels/MlModelResource.java index d0ed303ac86c..092e6c7489a9 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/mlmodels/MlModelResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/mlmodels/MlModelResource.java @@ -73,6 +73,7 @@ @Collection(name = "mlmodels") public class MlModelResource extends EntityResource { public static final String COLLECTION_PATH = "v1/mlmodels/"; + private final MlModelMapper mapper = new MlModelMapper(); static final String FIELDS = "owners,dashboard,followers,tags,usageSummary,extension,domain,sourceHash"; @@ -250,7 +251,7 @@ public Response create( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateMlModel create) { - MlModel mlModel = getMlModel(create, securityContext.getUserPrincipal().getName()); + MlModel mlModel = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return create(uriInfo, securityContext, mlModel); } @@ -331,7 +332,7 @@ public Response createOrUpdate( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateMlModel create) { - MlModel mlModel = getMlModel(create, securityContext.getUserPrincipal().getName()); + MlModel mlModel = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return createOrUpdate(uriInfo, securityContext, mlModel); } @@ -552,19 +553,4 @@ public Response restoreMlModel( @Valid RestoreEntity restore) { return restoreEntity(uriInfo, securityContext, restore.getId()); } - - private MlModel getMlModel(CreateMlModel create, String user) { - return repository - .copy(new MlModel(), create, user) - .withService(getEntityReference(Entity.MLMODEL_SERVICE, create.getService())) - .withDashboard(getEntityReference(Entity.DASHBOARD, create.getDashboard())) - .withAlgorithm(create.getAlgorithm()) - .withMlFeatures(create.getMlFeatures()) - .withMlHyperParameters(create.getMlHyperParameters()) - .withMlStore(create.getMlStore()) - .withServer(create.getServer()) - .withTarget(create.getTarget()) - .withSourceUrl(create.getSourceUrl()) - .withSourceHash(create.getSourceHash()); - } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/pipelines/PipelineMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/pipelines/PipelineMapper.java new file mode 100644 index 000000000000..78143bafdb0a --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/pipelines/PipelineMapper.java @@ -0,0 +1,23 @@ +package org.openmetadata.service.resources.pipelines; + +import static org.openmetadata.service.util.EntityUtil.getEntityReference; + +import org.openmetadata.schema.api.data.CreatePipeline; +import org.openmetadata.schema.entity.data.Pipeline; +import org.openmetadata.service.Entity; +import org.openmetadata.service.mapper.EntityMapper; + +public class PipelineMapper implements EntityMapper { + @Override + public Pipeline createToEntity(CreatePipeline create, String user) { + return copy(new Pipeline(), create, user) + .withService(getEntityReference(Entity.PIPELINE_SERVICE, create.getService())) + .withTasks(create.getTasks()) + .withSourceUrl(create.getSourceUrl()) + .withConcurrency(create.getConcurrency()) + .withStartDate(create.getStartDate()) + .withPipelineLocation(create.getPipelineLocation()) + .withScheduleInterval(create.getScheduleInterval()) + .withSourceHash(create.getSourceHash()); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/pipelines/PipelineResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/pipelines/PipelineResource.java index ee9228690c63..d0e97492496d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/pipelines/PipelineResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/pipelines/PipelineResource.java @@ -76,6 +76,7 @@ @Collection(name = "pipelines") public class PipelineResource extends EntityResource { public static final String COLLECTION_PATH = "v1/pipelines/"; + private final PipelineMapper mapper = new PipelineMapper(); static final String FIELDS = "owners,tasks,pipelineStatus,followers,tags,extension,scheduleInterval,domain,sourceHash"; @@ -311,7 +312,7 @@ public Response create( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreatePipeline create) { - Pipeline pipeline = getPipeline(create, securityContext.getUserPrincipal().getName()); + Pipeline pipeline = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return create(uriInfo, securityContext, pipeline); } @@ -392,7 +393,7 @@ public Response createOrUpdate( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreatePipeline create) { - Pipeline pipeline = getPipeline(create, securityContext.getUserPrincipal().getName()); + Pipeline pipeline = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return createOrUpdate(uriInfo, securityContext, pipeline); } @@ -424,7 +425,7 @@ public Response addPipelineStatus( OperationContext operationContext = new OperationContext(entityType, MetadataOperation.EDIT_STATUS); authorizer.authorize(securityContext, operationContext, getResourceContextByName(fqn)); - return repository.addPipelineStatus(uriInfo, fqn, pipelineStatus).toResponse(); + return repository.addPipelineStatus(fqn, pipelineStatus).toResponse(); } @GET @@ -669,17 +670,4 @@ public Response restorePipeline( @Valid RestoreEntity restore) { return restoreEntity(uriInfo, securityContext, restore.getId()); } - - private Pipeline getPipeline(CreatePipeline create, String user) { - return repository - .copy(new Pipeline(), create, user) - .withService(getEntityReference(Entity.PIPELINE_SERVICE, create.getService())) - .withTasks(create.getTasks()) - .withSourceUrl(create.getSourceUrl()) - .withConcurrency(create.getConcurrency()) - .withStartDate(create.getStartDate()) - .withPipelineLocation(create.getPipelineLocation()) - .withScheduleInterval(create.getScheduleInterval()) - .withSourceHash(create.getSourceHash()); - } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/policies/PolicyMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/policies/PolicyMapper.java new file mode 100644 index 000000000000..60b5aa6322d5 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/policies/PolicyMapper.java @@ -0,0 +1,20 @@ +package org.openmetadata.service.resources.policies; + +import org.openmetadata.schema.api.policies.CreatePolicy; +import org.openmetadata.schema.entity.policies.Policy; +import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.service.mapper.EntityMapper; + +public class PolicyMapper implements EntityMapper { + @Override + public Policy createToEntity(CreatePolicy create, String user) { + Policy policy = + copy(new Policy(), create, user) + .withRules(create.getRules()) + .withEnabled(create.getEnabled()); + if (create.getLocation() != null) { + policy = policy.withLocation(new EntityReference().withId(create.getLocation())); + } + return policy; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/policies/PolicyResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/policies/PolicyResource.java index 663ff9c556f7..411a489899b2 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/policies/PolicyResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/policies/PolicyResource.java @@ -51,7 +51,6 @@ import org.openmetadata.schema.entity.policies.Policy; import org.openmetadata.schema.entity.policies.accessControl.Rule; import org.openmetadata.schema.type.EntityHistory; -import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.Function; import org.openmetadata.schema.type.Include; import org.openmetadata.schema.type.MetadataOperation; @@ -82,6 +81,7 @@ @Consumes(MediaType.APPLICATION_JSON) @Collection(name = "policies", order = 0, requiredForOps = true) public class PolicyResource extends EntityResource { + private final PolicyMapper mapper = new PolicyMapper(); public static final String COLLECTION_PATH = "v1/policies/"; public static final String FIELDS = "owners,location,teams,roles"; @@ -359,7 +359,7 @@ public Response create( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreatePolicy create) { - Policy policy = getPolicy(create, securityContext.getUserPrincipal().getName()); + Policy policy = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return create(uriInfo, securityContext, policy); } @@ -439,7 +439,7 @@ public Response createOrUpdate( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreatePolicy create) { - Policy policy = getPolicy(create, securityContext.getUserPrincipal().getName()); + Policy policy = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return createOrUpdate(uriInfo, securityContext, policy); } @@ -531,16 +531,4 @@ public void validateCondition( authorizer.authorizeAdmin(securityContext); CompiledRule.validateExpression(expression, Boolean.class); } - - private Policy getPolicy(CreatePolicy create, String user) { - Policy policy = - repository - .copy(new Policy(), create, user) - .withRules(create.getRules()) - .withEnabled(create.getEnabled()); - if (create.getLocation() != null) { - policy = policy.withLocation(new EntityReference().withId(create.getLocation())); - } - return policy; - } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/query/QueryMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/query/QueryMapper.java new file mode 100644 index 000000000000..581e81679780 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/query/QueryMapper.java @@ -0,0 +1,29 @@ +package org.openmetadata.service.resources.query; + +import static org.openmetadata.service.Entity.USER; +import static org.openmetadata.service.util.EntityUtil.getEntityReference; +import static org.openmetadata.service.util.EntityUtil.getEntityReferences; + +import org.openmetadata.schema.api.data.CreateQuery; +import org.openmetadata.schema.entity.data.Query; +import org.openmetadata.schema.type.Votes; +import org.openmetadata.service.Entity; +import org.openmetadata.service.mapper.EntityMapper; +import org.openmetadata.service.util.EntityUtil; + +public class QueryMapper implements EntityMapper { + @Override + public Query createToEntity(CreateQuery create, String user) { + return copy(new Query(), create, user) + .withQuery(create.getQuery()) + .withChecksum(EntityUtil.hash(create.getQuery())) + .withService(getEntityReference(Entity.DATABASE_SERVICE, create.getService())) + .withDuration(create.getDuration()) + .withVotes(new Votes().withUpVotes(0).withDownVotes(0)) + .withUsers(getEntityReferences(USER, create.getUsers())) + .withQueryUsedIn(EntityUtil.populateEntityReferences(create.getQueryUsedIn())) + .withQueryDate(create.getQueryDate()) + .withTriggeredBy(create.getTriggeredBy()) + .withProcessedLineage(create.getProcessedLineage()); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/query/QueryResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/query/QueryResource.java index 1839fa8d4928..c82743da06e0 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/query/QueryResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/query/QueryResource.java @@ -1,7 +1,5 @@ package org.openmetadata.service.resources.query; -import static org.openmetadata.service.Entity.USER; - import io.swagger.v3.oas.annotations.ExternalDocumentation; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; @@ -42,7 +40,6 @@ import org.openmetadata.schema.type.EntityHistory; import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.MetadataOperation; -import org.openmetadata.schema.type.Votes; import org.openmetadata.service.Entity; import org.openmetadata.service.jdbi3.ListFilter; import org.openmetadata.service.jdbi3.QueryRepository; @@ -52,7 +49,6 @@ import org.openmetadata.service.security.Authorizer; import org.openmetadata.service.security.mask.PIIMasker; import org.openmetadata.service.security.policyevaluator.OperationContext; -import org.openmetadata.service.util.EntityUtil; import org.openmetadata.service.util.ResultList; @Path("/v1/queries") @@ -64,6 +60,7 @@ @Consumes(MediaType.APPLICATION_JSON) @Collection(name = "queries") public class QueryResource extends EntityResource { + private final QueryMapper mapper = new QueryMapper(); public static final String COLLECTION_PATH = "v1/queries/"; static final String FIELDS = "owners,followers,users,votes,tags,queryUsedIn"; @@ -285,7 +282,7 @@ public Response create( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateQuery create) { - Query query = getQuery(create, securityContext.getUserPrincipal().getName()); + Query query = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return create(uriInfo, securityContext, query); } @@ -309,7 +306,7 @@ public Response createOrUpdate( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateQuery create) { - Query query = getQuery(create, securityContext.getUserPrincipal().getName()); + Query query = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return createOrUpdate(uriInfo, securityContext, query); } @@ -633,19 +630,4 @@ public Response delete( String fqn) { return deleteByName(uriInfo, securityContext, fqn, false, true); } - - private Query getQuery(CreateQuery create, String user) { - return repository - .copy(new Query(), create, user) - .withQuery(create.getQuery()) - .withChecksum(EntityUtil.hash(create.getQuery())) - .withService(getEntityReference(Entity.DATABASE_SERVICE, create.getService())) - .withDuration(create.getDuration()) - .withVotes(new Votes().withUpVotes(0).withDownVotes(0)) - .withUsers(getEntityReferences(USER, create.getUsers())) - .withQueryUsedIn(EntityUtil.populateEntityReferences(create.getQueryUsedIn())) - .withQueryDate(create.getQueryDate()) - .withTriggeredBy(create.getTriggeredBy()) - .withProcessedLineage(create.getProcessedLineage()); - } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/searchindex/SearchIndexMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/searchindex/SearchIndexMapper.java new file mode 100644 index 000000000000..b7df4b11e337 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/searchindex/SearchIndexMapper.java @@ -0,0 +1,20 @@ +package org.openmetadata.service.resources.searchindex; + +import static org.openmetadata.service.util.EntityUtil.getEntityReference; + +import org.openmetadata.schema.api.data.CreateSearchIndex; +import org.openmetadata.schema.entity.data.SearchIndex; +import org.openmetadata.service.Entity; +import org.openmetadata.service.mapper.EntityMapper; + +public class SearchIndexMapper implements EntityMapper { + @Override + public SearchIndex createToEntity(CreateSearchIndex create, String user) { + return copy(new SearchIndex(), create, user) + .withService(getEntityReference(Entity.SEARCH_SERVICE, create.getService())) + .withFields(create.getFields()) + .withSearchIndexSettings(create.getSearchIndexSettings()) + .withSourceHash(create.getSourceHash()) + .withIndexType(create.getIndexType()); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/searchindex/SearchIndexResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/searchindex/SearchIndexResource.java index 24831f21253e..e65dea4cf4ad 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/searchindex/SearchIndexResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/searchindex/SearchIndexResource.java @@ -75,6 +75,7 @@ @Consumes(MediaType.APPLICATION_JSON) @Collection(name = "searchIndexes") public class SearchIndexResource extends EntityResource { + private final SearchIndexMapper mapper = new SearchIndexMapper(); public static final String COLLECTION_PATH = "v1/searchIndexes/"; static final String FIELDS = "owners,followers,tags,extension,domain,dataProducts,sourceHash"; @@ -309,7 +310,8 @@ public Response create( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateSearchIndex create) { - SearchIndex searchIndex = getSearchIndex(create, securityContext.getUserPrincipal().getName()); + SearchIndex searchIndex = + mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return create(uriInfo, securityContext, searchIndex); } @@ -389,7 +391,8 @@ public Response createOrUpdate( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateSearchIndex create) { - SearchIndex searchIndex = getSearchIndex(create, securityContext.getUserPrincipal().getName()); + SearchIndex searchIndex = + mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return createOrUpdate(uriInfo, securityContext, searchIndex); } @@ -626,16 +629,4 @@ public Response restoreSearchIndex( @Valid RestoreEntity restore) { return restoreEntity(uriInfo, securityContext, restore.getId()); } - - private SearchIndex getSearchIndex(CreateSearchIndex create, String user) { - SearchIndex searchIndex = - repository - .copy(new SearchIndex(), create, user) - .withService(getEntityReference(Entity.SEARCH_SERVICE, create.getService())) - .withFields(create.getFields()) - .withSearchIndexSettings(create.getSearchIndexSettings()) - .withSourceHash(create.getSourceHash()) - .withIndexType(create.getIndexType()); - return searchIndex; - } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/apiservices/APIServiceMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/apiservices/APIServiceMapper.java new file mode 100644 index 000000000000..9e5187875f61 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/apiservices/APIServiceMapper.java @@ -0,0 +1,14 @@ +package org.openmetadata.service.resources.services.apiservices; + +import org.openmetadata.schema.api.services.CreateApiService; +import org.openmetadata.schema.entity.services.ApiService; +import org.openmetadata.service.mapper.EntityMapper; + +public class APIServiceMapper implements EntityMapper { + @Override + public ApiService createToEntity(CreateApiService create, String user) { + return copy(new ApiService(), create, user) + .withServiceType(create.getServiceType()) + .withConnection(create.getConnection()); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/apiservices/APIServiceResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/apiservices/APIServiceResource.java index 1a23c9b1c9a9..47f2345b06b0 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/apiservices/APIServiceResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/apiservices/APIServiceResource.java @@ -76,6 +76,7 @@ @Collection(name = "apiServices") public class APIServiceResource extends ServiceEntityResource { + private final APIServiceMapper mapper = new APIServiceMapper(); public static final String COLLECTION_PATH = "v1/services/apiServices/"; static final String FIELDS = "pipelines,owners,tags,domain"; @@ -345,7 +346,8 @@ public Response create( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateApiService create) { - ApiService service = getService(create, securityContext.getUserPrincipal().getName()); + ApiService service = + mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); Response response = create(uriInfo, securityContext, service); decryptOrNullify(securityContext, (ApiService) response.getEntity()); return response; @@ -370,7 +372,8 @@ public Response createOrUpdate( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateApiService update) { - ApiService service = getService(update, securityContext.getUserPrincipal().getName()); + ApiService service = + mapper.createToEntity(update, securityContext.getUserPrincipal().getName()); Response response = createOrUpdate(uriInfo, securityContext, unmask(service)); decryptOrNullify(securityContext, (ApiService) response.getEntity()); return response; @@ -513,13 +516,6 @@ public Response restoreAPIService( return restoreEntity(uriInfo, securityContext, restore.getId()); } - private ApiService getService(CreateApiService create, String user) { - return repository - .copy(new ApiService(), create, user) - .withServiceType(create.getServiceType()) - .withConnection(create.getConnection()); - } - @Override protected ApiService nullifyConnection(ApiService service) { return service.withConnection(null); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/dashboard/DashboardServiceMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/dashboard/DashboardServiceMapper.java new file mode 100644 index 000000000000..3ec9cab15750 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/dashboard/DashboardServiceMapper.java @@ -0,0 +1,15 @@ +package org.openmetadata.service.resources.services.dashboard; + +import org.openmetadata.schema.api.services.CreateDashboardService; +import org.openmetadata.schema.entity.services.DashboardService; +import org.openmetadata.service.mapper.EntityMapper; + +public class DashboardServiceMapper + implements EntityMapper { + @Override + public DashboardService createToEntity(CreateDashboardService create, String user) { + return copy(new DashboardService(), create, user) + .withServiceType(create.getServiceType()) + .withConnection(create.getConnection()); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/dashboard/DashboardServiceResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/dashboard/DashboardServiceResource.java index fa72220f3ede..d000ac35775d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/dashboard/DashboardServiceResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/dashboard/DashboardServiceResource.java @@ -72,6 +72,7 @@ public class DashboardServiceResource extends ServiceEntityResource< DashboardService, DashboardServiceRepository, DashboardConnection> { + private final DashboardServiceMapper mapper = new DashboardServiceMapper(); public static final String COLLECTION_PATH = "v1/services/dashboardServices"; static final String FIELDS = "owners,domain"; @@ -333,7 +334,8 @@ public Response create( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateDashboardService create) { - DashboardService service = getService(create, securityContext.getUserPrincipal().getName()); + DashboardService service = + mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); Response response = create(uriInfo, securityContext, service); decryptOrNullify(securityContext, (DashboardService) response.getEntity()); return response; @@ -358,7 +360,8 @@ public Response createOrUpdate( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateDashboardService update) { - DashboardService service = getService(update, securityContext.getUserPrincipal().getName()); + DashboardService service = + mapper.createToEntity(update, securityContext.getUserPrincipal().getName()); Response response = createOrUpdate(uriInfo, securityContext, unmask(service)); decryptOrNullify(securityContext, (DashboardService) response.getEntity()); return response; @@ -507,13 +510,6 @@ public Response restoreDashboardService( return restoreEntity(uriInfo, securityContext, restore.getId()); } - private DashboardService getService(CreateDashboardService create, String user) { - return repository - .copy(new DashboardService(), create, user) - .withServiceType(create.getServiceType()) - .withConnection(create.getConnection()); - } - @Override protected DashboardService nullifyConnection(DashboardService service) { return service.withConnection(null); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/database/DatabaseServiceMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/database/DatabaseServiceMapper.java new file mode 100644 index 000000000000..db8ac8818afd --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/database/DatabaseServiceMapper.java @@ -0,0 +1,14 @@ +package org.openmetadata.service.resources.services.database; + +import org.openmetadata.schema.api.services.CreateDatabaseService; +import org.openmetadata.schema.entity.services.DatabaseService; +import org.openmetadata.service.mapper.EntityMapper; + +public class DatabaseServiceMapper implements EntityMapper { + @Override + public DatabaseService createToEntity(CreateDatabaseService create, String user) { + return copy(new DatabaseService(), create, user) + .withServiceType(create.getServiceType()) + .withConnection(create.getConnection()); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/database/DatabaseServiceResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/database/DatabaseServiceResource.java index adf3fd7b8d59..0faf6fd73139 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/database/DatabaseServiceResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/database/DatabaseServiceResource.java @@ -80,6 +80,7 @@ @Collection(name = "databaseServices") public class DatabaseServiceResource extends ServiceEntityResource { + private final DatabaseServiceMapper mapper = new DatabaseServiceMapper(); public static final String COLLECTION_PATH = "v1/services/databaseServices/"; static final String FIELDS = "pipelines,owners,tags,domain"; @@ -352,7 +353,8 @@ public Response create( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateDatabaseService create) { - DatabaseService service = getService(create, securityContext.getUserPrincipal().getName()); + DatabaseService service = + mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); Response response = create(uriInfo, securityContext, service); decryptOrNullify(securityContext, (DatabaseService) response.getEntity()); return response; @@ -377,7 +379,8 @@ public Response createOrUpdate( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateDatabaseService update) { - DatabaseService service = getService(update, securityContext.getUserPrincipal().getName()); + DatabaseService service = + mapper.createToEntity(update, securityContext.getUserPrincipal().getName()); Response response = createOrUpdate(uriInfo, securityContext, unmask(service)); decryptOrNullify(securityContext, (DatabaseService) response.getEntity()); return response; @@ -642,13 +645,6 @@ public Response restoreDatabaseService( return restoreEntity(uriInfo, securityContext, restore.getId()); } - private DatabaseService getService(CreateDatabaseService create, String user) { - return repository - .copy(new DatabaseService(), create, user) - .withServiceType(create.getServiceType()) - .withConnection(create.getConnection()); - } - @Override protected DatabaseService nullifyConnection(DatabaseService service) { return service.withConnection(null); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/ingestionpipelines/IngestionPipelineMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/ingestionpipelines/IngestionPipelineMapper.java new file mode 100644 index 000000000000..e2134d9b11f7 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/ingestionpipelines/IngestionPipelineMapper.java @@ -0,0 +1,31 @@ +package org.openmetadata.service.resources.services.ingestionpipelines; + +import org.openmetadata.schema.api.services.ingestionPipelines.CreateIngestionPipeline; +import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline; +import org.openmetadata.schema.services.connections.metadata.OpenMetadataConnection; +import org.openmetadata.service.OpenMetadataApplicationConfig; +import org.openmetadata.service.mapper.EntityMapper; +import org.openmetadata.service.util.OpenMetadataConnectionBuilder; + +public class IngestionPipelineMapper + implements EntityMapper { + private final OpenMetadataApplicationConfig openMetadataApplicationConfig; + + public IngestionPipelineMapper(OpenMetadataApplicationConfig openMetadataApplicationConfig) { + this.openMetadataApplicationConfig = openMetadataApplicationConfig; + } + + @Override + public IngestionPipeline createToEntity(CreateIngestionPipeline create, String user) { + OpenMetadataConnection openMetadataServerConnection = + new OpenMetadataConnectionBuilder(openMetadataApplicationConfig).build(); + + return copy(new IngestionPipeline(), create, user) + .withPipelineType(create.getPipelineType()) + .withAirflowConfig(create.getAirflowConfig()) + .withOpenMetadataServerConnection(openMetadataServerConnection) + .withSourceConfig(create.getSourceConfig()) + .withLoggerLevel(create.getLoggerLevel()) + .withService(create.getService()); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/ingestionpipelines/IngestionPipelineResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/ingestionpipelines/IngestionPipelineResource.java index 51757d6c9f6d..3a64d212caee 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/ingestionpipelines/IngestionPipelineResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/ingestionpipelines/IngestionPipelineResource.java @@ -97,6 +97,7 @@ @Collection(name = "IngestionPipelines") public class IngestionPipelineResource extends EntityResource { + private IngestionPipelineMapper mapper; public static final String COLLECTION_PATH = "v1/services/ingestionPipelines/"; private PipelineServiceClientInterface pipelineServiceClient; private OpenMetadataApplicationConfig openMetadataApplicationConfig; @@ -116,7 +117,7 @@ public IngestionPipelineResource(Authorizer authorizer, Limits limits) { @Override public void initialize(OpenMetadataApplicationConfig config) { this.openMetadataApplicationConfig = config; - + this.mapper = new IngestionPipelineMapper(config); this.pipelineServiceClient = PipelineServiceClientFactory.createPipelineServiceClient( config.getPipelineServiceClientConfiguration()); @@ -426,7 +427,7 @@ public Response create( @Context SecurityContext securityContext, @Valid CreateIngestionPipeline create) { IngestionPipeline ingestionPipeline = - getIngestionPipeline(create, securityContext.getUserPrincipal().getName()); + mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); Response response = create(uriInfo, securityContext, ingestionPipeline); validateProfileSample(ingestionPipeline); decryptOrNullify(securityContext, (IngestionPipeline) response.getEntity(), false); @@ -516,7 +517,7 @@ public Response createOrUpdate( @Context SecurityContext securityContext, @Valid CreateIngestionPipeline update) { IngestionPipeline ingestionPipeline = - getIngestionPipeline(update, securityContext.getUserPrincipal().getName()); + mapper.createToEntity(update, securityContext.getUserPrincipal().getName()); unmask(ingestionPipeline); Response response = createOrUpdate(uriInfo, securityContext, ingestionPipeline); validateProfileSample(ingestionPipeline); @@ -936,20 +937,6 @@ public IngestionPipeline deletePipelineStatus( return addHref(uriInfo, ingestionPipeline); } - private IngestionPipeline getIngestionPipeline(CreateIngestionPipeline create, String user) { - OpenMetadataConnection openMetadataServerConnection = - new OpenMetadataConnectionBuilder(openMetadataApplicationConfig).build(); - - return repository - .copy(new IngestionPipeline(), create, user) - .withPipelineType(create.getPipelineType()) - .withAirflowConfig(create.getAirflowConfig()) - .withOpenMetadataServerConnection(openMetadataServerConnection) - .withSourceConfig(create.getSourceConfig()) - .withLoggerLevel(create.getLoggerLevel()) - .withService(create.getService()); - } - private void unmask(IngestionPipeline ingestionPipeline) { repository.setFullyQualifiedName(ingestionPipeline); IngestionPipeline originalIngestionPipeline = diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/messaging/MessagingServiceMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/messaging/MessagingServiceMapper.java new file mode 100644 index 000000000000..666ba51cccb5 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/messaging/MessagingServiceMapper.java @@ -0,0 +1,15 @@ +package org.openmetadata.service.resources.services.messaging; + +import org.openmetadata.schema.api.services.CreateMessagingService; +import org.openmetadata.schema.entity.services.MessagingService; +import org.openmetadata.service.mapper.EntityMapper; + +public class MessagingServiceMapper + implements EntityMapper { + @Override + public MessagingService createToEntity(CreateMessagingService create, String user) { + return copy(new MessagingService(), create, user) + .withConnection(create.getConnection()) + .withServiceType(create.getServiceType()); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/messaging/MessagingServiceResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/messaging/MessagingServiceResource.java index 139861ecf301..beb48d97dda8 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/messaging/MessagingServiceResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/messaging/MessagingServiceResource.java @@ -72,6 +72,7 @@ public class MessagingServiceResource extends ServiceEntityResource< MessagingService, MessagingServiceRepository, MessagingConnection> { + private final MessagingServiceMapper mapper = new MessagingServiceMapper(); public static final String COLLECTION_PATH = "v1/services/messagingServices/"; public static final String FIELDS = "owners,domain"; @@ -339,7 +340,8 @@ public Response create( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateMessagingService create) { - MessagingService service = getService(create, securityContext.getUserPrincipal().getName()); + MessagingService service = + mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); Response response = create(uriInfo, securityContext, service); decryptOrNullify(securityContext, (MessagingService) response.getEntity()); return response; @@ -365,7 +367,8 @@ public Response createOrUpdate( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateMessagingService update) { - MessagingService service = getService(update, securityContext.getUserPrincipal().getName()); + MessagingService service = + mapper.createToEntity(update, securityContext.getUserPrincipal().getName()); Response response = createOrUpdate(uriInfo, securityContext, unmask(service)); decryptOrNullify(securityContext, (MessagingService) response.getEntity()); return response; @@ -513,13 +516,6 @@ public Response restoreMessagingService( return restoreEntity(uriInfo, securityContext, restore.getId()); } - private MessagingService getService(CreateMessagingService create, String user) { - return repository - .copy(new MessagingService(), create, user) - .withConnection(create.getConnection()) - .withServiceType(create.getServiceType()); - } - @Override protected MessagingService nullifyConnection(MessagingService service) { return service.withConnection(null); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/metadata/MetadataServiceMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/metadata/MetadataServiceMapper.java new file mode 100644 index 000000000000..a546eb60140e --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/metadata/MetadataServiceMapper.java @@ -0,0 +1,14 @@ +package org.openmetadata.service.resources.services.metadata; + +import org.openmetadata.schema.api.services.CreateMetadataService; +import org.openmetadata.schema.entity.services.MetadataService; +import org.openmetadata.service.mapper.EntityMapper; + +public class MetadataServiceMapper implements EntityMapper { + @Override + public MetadataService createToEntity(CreateMetadataService create, String user) { + return copy(new MetadataService(), create, user) + .withServiceType(create.getServiceType()) + .withConnection(create.getConnection()); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/metadata/MetadataServiceResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/metadata/MetadataServiceResource.java index dc0ca583ad43..58b77643b4c0 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/metadata/MetadataServiceResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/metadata/MetadataServiceResource.java @@ -16,7 +16,6 @@ import java.util.List; import java.util.Objects; import java.util.UUID; -import java.util.stream.Collectors; import javax.json.JsonPatch; import javax.validation.Valid; import javax.validation.constraints.Max; @@ -75,6 +74,7 @@ @Collection(name = "metadataServices", order = 8) // init before IngestionPipelineService public class MetadataServiceResource extends ServiceEntityResource { + private final MetadataServiceMapper mapper = new MetadataServiceMapper(); public static final String OPENMETADATA_SERVICE = "OpenMetadata"; public static final String COLLECTION_PATH = "v1/services/metadataServices/"; public static final String FIELDS = "pipelines,owners,tags"; @@ -320,7 +320,7 @@ public EntityHistory listVersions( return json; } }) - .collect(Collectors.toList()); + .toList(); entityHistory.setVersions(versions); return entityHistory; } @@ -378,7 +378,7 @@ public Response create( @Context SecurityContext securityContext, @Valid CreateMetadataService create) { MetadataService service = - getMetadataService(create, securityContext.getUserPrincipal().getName()); + mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); Response response = create(uriInfo, securityContext, service); decryptOrNullify(securityContext, (MetadataService) response.getEntity()); return response; @@ -404,7 +404,7 @@ public Response createOrUpdate( @Context SecurityContext securityContext, @Valid CreateMetadataService update) { MetadataService service = - getMetadataService(update, securityContext.getUserPrincipal().getName()); + mapper.createToEntity(update, securityContext.getUserPrincipal().getName()); Response response = createOrUpdate(uriInfo, securityContext, unmask(service)); decryptOrNullify(securityContext, (MetadataService) response.getEntity()); return response; @@ -552,13 +552,6 @@ public Response restoreMetadataService( return restoreEntity(uriInfo, securityContext, restore.getId()); } - private MetadataService getMetadataService(CreateMetadataService create, String user) { - return repository - .copy(new MetadataService(), create, user) - .withServiceType(create.getServiceType()) - .withConnection(create.getConnection()); - } - @Override protected MetadataService nullifyConnection(MetadataService service) { return service.withConnection(null); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/mlmodel/MlModelServiceMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/mlmodel/MlModelServiceMapper.java new file mode 100644 index 000000000000..fac0635e55fa --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/mlmodel/MlModelServiceMapper.java @@ -0,0 +1,14 @@ +package org.openmetadata.service.resources.services.mlmodel; + +import org.openmetadata.schema.api.services.CreateMlModelService; +import org.openmetadata.schema.entity.services.MlModelService; +import org.openmetadata.service.mapper.EntityMapper; + +public class MlModelServiceMapper implements EntityMapper { + @Override + public MlModelService createToEntity(CreateMlModelService create, String user) { + return copy(new MlModelService(), create, user) + .withServiceType(create.getServiceType()) + .withConnection(create.getConnection()); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/mlmodel/MlModelServiceResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/mlmodel/MlModelServiceResource.java index 5b70603806ed..bf1c389d5d28 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/mlmodel/MlModelServiceResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/mlmodel/MlModelServiceResource.java @@ -73,6 +73,7 @@ @Collection(name = "mlmodelServices") public class MlModelServiceResource extends ServiceEntityResource { + private final MlModelServiceMapper mapper = new MlModelServiceMapper(); public static final String COLLECTION_PATH = "v1/services/mlmodelServices/"; public static final String FIELDS = "pipelines,owners,tags,domain"; @@ -351,7 +352,8 @@ public Response create( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateMlModelService create) { - MlModelService service = getService(create, securityContext.getUserPrincipal().getName()); + MlModelService service = + mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); Response response = create(uriInfo, securityContext, service); decryptOrNullify(securityContext, (MlModelService) response.getEntity()); return response; @@ -377,7 +379,8 @@ public Response createOrUpdate( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateMlModelService update) { - MlModelService service = getService(update, securityContext.getUserPrincipal().getName()); + MlModelService service = + mapper.createToEntity(update, securityContext.getUserPrincipal().getName()); Response response = createOrUpdate(uriInfo, securityContext, unmask(service)); decryptOrNullify(securityContext, (MlModelService) response.getEntity()); return response; @@ -526,13 +529,6 @@ public Response restoreTable( return restoreEntity(uriInfo, securityContext, restore.getId()); } - private MlModelService getService(CreateMlModelService create, String user) { - return repository - .copy(new MlModelService(), create, user) - .withServiceType(create.getServiceType()) - .withConnection(create.getConnection()); - } - @Override protected MlModelService nullifyConnection(MlModelService service) { return service.withConnection(null); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/pipeline/PipelineServiceMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/pipeline/PipelineServiceMapper.java new file mode 100644 index 000000000000..a6eec765aa4a --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/pipeline/PipelineServiceMapper.java @@ -0,0 +1,14 @@ +package org.openmetadata.service.resources.services.pipeline; + +import org.openmetadata.schema.api.services.CreatePipelineService; +import org.openmetadata.schema.entity.services.PipelineService; +import org.openmetadata.service.mapper.EntityMapper; + +public class PipelineServiceMapper implements EntityMapper { + @Override + public PipelineService createToEntity(CreatePipelineService create, String user) { + return copy(new PipelineService(), create, user) + .withServiceType(create.getServiceType()) + .withConnection(create.getConnection()); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/pipeline/PipelineServiceResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/pipeline/PipelineServiceResource.java index b6b5406ce008..1727413224f0 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/pipeline/PipelineServiceResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/pipeline/PipelineServiceResource.java @@ -71,6 +71,7 @@ @Collection(name = "pipelineServices") public class PipelineServiceResource extends ServiceEntityResource { + private final PipelineServiceMapper mapper = new PipelineServiceMapper(); public static final String COLLECTION_PATH = "v1/services/pipelineServices/"; static final String FIELDS = "pipelines,owners,domain"; @@ -352,7 +353,8 @@ public Response create( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreatePipelineService create) { - PipelineService service = getService(create, securityContext.getUserPrincipal().getName()); + PipelineService service = + mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); Response response = create(uriInfo, securityContext, service); decryptOrNullify(securityContext, (PipelineService) response.getEntity()); return response; @@ -378,7 +380,8 @@ public Response createOrUpdate( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreatePipelineService update) { - PipelineService service = getService(update, securityContext.getUserPrincipal().getName()); + PipelineService service = + mapper.createToEntity(update, securityContext.getUserPrincipal().getName()); Response response = createOrUpdate(uriInfo, securityContext, unmask(service)); decryptOrNullify(securityContext, (PipelineService) response.getEntity()); return response; @@ -529,13 +532,6 @@ public Response restorePipelineService( return restoreEntity(uriInfo, securityContext, restore.getId()); } - private PipelineService getService(CreatePipelineService create, String user) { - return repository - .copy(new PipelineService(), create, user) - .withServiceType(create.getServiceType()) - .withConnection(create.getConnection()); - } - @Override protected PipelineService nullifyConnection(PipelineService service) { return service.withConnection(null); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/searchIndexes/SearchServiceMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/searchIndexes/SearchServiceMapper.java new file mode 100644 index 000000000000..d4a220908eb1 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/searchIndexes/SearchServiceMapper.java @@ -0,0 +1,14 @@ +package org.openmetadata.service.resources.services.searchIndexes; + +import org.openmetadata.schema.api.services.CreateSearchService; +import org.openmetadata.schema.entity.services.SearchService; +import org.openmetadata.service.mapper.EntityMapper; + +public class SearchServiceMapper implements EntityMapper { + @Override + public SearchService createToEntity(CreateSearchService create, String user) { + return copy(new SearchService(), create, user) + .withServiceType(create.getServiceType()) + .withConnection(create.getConnection()); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/searchIndexes/SearchServiceResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/searchIndexes/SearchServiceResource.java index 59584f45fbf1..a1f7c592f4aa 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/searchIndexes/SearchServiceResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/searchIndexes/SearchServiceResource.java @@ -63,6 +63,7 @@ @Collection(name = "searchServices") public class SearchServiceResource extends ServiceEntityResource { + private final SearchServiceMapper mapper = new SearchServiceMapper(); public static final String COLLECTION_PATH = "v1/services/searchServices/"; static final String FIELDS = "pipelines,owners,tags,domain"; @@ -333,7 +334,8 @@ public Response create( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateSearchService create) { - SearchService service = getService(create, securityContext.getUserPrincipal().getName()); + SearchService service = + mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); Response response = create(uriInfo, securityContext, service); decryptOrNullify(securityContext, (SearchService) response.getEntity()); return response; @@ -358,7 +360,8 @@ public Response createOrUpdate( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateSearchService update) { - SearchService service = getService(update, securityContext.getUserPrincipal().getName()); + SearchService service = + mapper.createToEntity(update, securityContext.getUserPrincipal().getName()); Response response = createOrUpdate(uriInfo, securityContext, unmask(service)); decryptOrNullify(securityContext, (SearchService) response.getEntity()); return response; @@ -502,13 +505,6 @@ public Response restoreSearchService( return restoreEntity(uriInfo, securityContext, restore.getId()); } - private SearchService getService(CreateSearchService create, String user) { - return repository - .copy(new SearchService(), create, user) - .withServiceType(create.getServiceType()) - .withConnection(create.getConnection()); - } - @Override protected SearchService nullifyConnection(SearchService service) { return service.withConnection(null); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/storage/StorageServiceMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/storage/StorageServiceMapper.java new file mode 100644 index 000000000000..4fb0c4aa02c3 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/storage/StorageServiceMapper.java @@ -0,0 +1,14 @@ +package org.openmetadata.service.resources.services.storage; + +import org.openmetadata.schema.api.services.CreateStorageService; +import org.openmetadata.schema.entity.services.StorageService; +import org.openmetadata.service.mapper.EntityMapper; + +public class StorageServiceMapper implements EntityMapper { + @Override + public StorageService createToEntity(CreateStorageService create, String user) { + return copy(new StorageService(), create, user) + .withServiceType(create.getServiceType()) + .withConnection(create.getConnection()); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/storage/StorageServiceResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/storage/StorageServiceResource.java index cfbe55a5ca62..ed5e489099a2 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/storage/StorageServiceResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/storage/StorageServiceResource.java @@ -63,6 +63,7 @@ @Collection(name = "storageServices") public class StorageServiceResource extends ServiceEntityResource { + private final StorageServiceMapper mapper = new StorageServiceMapper(); public static final String COLLECTION_PATH = "v1/services/storageServices/"; static final String FIELDS = "pipelines,owners,tags,domain"; @@ -332,7 +333,8 @@ public Response create( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateStorageService create) { - StorageService service = getService(create, securityContext.getUserPrincipal().getName()); + StorageService service = + mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); Response response = create(uriInfo, securityContext, service); decryptOrNullify(securityContext, (StorageService) response.getEntity()); return response; @@ -357,7 +359,8 @@ public Response createOrUpdate( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateStorageService update) { - StorageService service = getService(update, securityContext.getUserPrincipal().getName()); + StorageService service = + mapper.createToEntity(update, securityContext.getUserPrincipal().getName()); Response response = createOrUpdate(uriInfo, securityContext, unmask(service)); decryptOrNullify(securityContext, (StorageService) response.getEntity()); return response; @@ -473,13 +476,6 @@ public Response restoreStorageService( return restoreEntity(uriInfo, securityContext, restore.getId()); } - private StorageService getService(CreateStorageService create, String user) { - return repository - .copy(new StorageService(), create, user) - .withServiceType(create.getServiceType()) - .withConnection(create.getConnection()); - } - @Override protected StorageService nullifyConnection(StorageService service) { return service.withConnection(null); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/settings/SettingsCache.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/settings/SettingsCache.java index a6785ddb4616..dff08f8ccc16 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/settings/SettingsCache.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/settings/SettingsCache.java @@ -19,6 +19,7 @@ import static org.openmetadata.schema.settings.SettingsType.LINEAGE_SETTINGS; import static org.openmetadata.schema.settings.SettingsType.LOGIN_CONFIGURATION; import static org.openmetadata.schema.settings.SettingsType.SEARCH_SETTINGS; +import static org.openmetadata.schema.settings.SettingsType.WORKFLOW_SETTINGS; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; @@ -27,6 +28,7 @@ import javax.annotation.CheckForNull; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang.StringUtils; import org.openmetadata.api.configuration.LogoConfiguration; import org.openmetadata.api.configuration.ThemeConfiguration; import org.openmetadata.api.configuration.UiThemePreference; @@ -35,13 +37,15 @@ import org.openmetadata.schema.api.lineage.LineageSettings; import org.openmetadata.schema.api.search.SearchSettings; import org.openmetadata.schema.configuration.AssetCertificationSettings; +import org.openmetadata.schema.configuration.ExecutorConfiguration; +import org.openmetadata.schema.configuration.HistoryCleanUpConfiguration; +import org.openmetadata.schema.configuration.WorkflowSettings; import org.openmetadata.schema.email.SmtpSettings; import org.openmetadata.schema.settings.Settings; import org.openmetadata.schema.settings.SettingsType; import org.openmetadata.service.Entity; import org.openmetadata.service.OpenMetadataApplicationConfig; import org.openmetadata.service.exception.EntityNotFoundException; -import org.openmetadata.service.jdbi3.SystemRepository; import org.openmetadata.service.util.JsonUtils; @Slf4j @@ -52,7 +56,6 @@ public class SettingsCache { .maximumSize(1000) .expireAfterWrite(3, TimeUnit.MINUTES) .build(new SettingsLoader()); - protected static SystemRepository systemRepository; private SettingsCache() { // Private constructor for singleton @@ -61,7 +64,6 @@ private SettingsCache() { // Expected to be called only once from the DefaultAuthorizer public static void initialize(OpenMetadataApplicationConfig config) { if (!initialized) { - systemRepository = Entity.getSystemRepository(); initialized = true; createDefaultConfiguration(config); } @@ -69,18 +71,19 @@ public static void initialize(OpenMetadataApplicationConfig config) { private static void createDefaultConfiguration(OpenMetadataApplicationConfig applicationConfig) { // Initialise Email Setting - Settings storedSettings = systemRepository.getConfigWithKey(EMAIL_CONFIGURATION.toString()); + Settings storedSettings = + Entity.getSystemRepository().getConfigWithKey(EMAIL_CONFIGURATION.toString()); if (storedSettings == null) { // Only in case a config doesn't exist in DB we insert it SmtpSettings emailConfig = applicationConfig.getSmtpSettings(); Settings setting = new Settings().withConfigType(EMAIL_CONFIGURATION).withConfigValue(emailConfig); - systemRepository.createNewSetting(setting); + Entity.getSystemRepository().createNewSetting(setting); } // Initialise Theme Setting Settings storedCustomUiThemeConf = - systemRepository.getConfigWithKey(CUSTOM_UI_THEME_PREFERENCE.toString()); + Entity.getSystemRepository().getConfigWithKey(CUSTOM_UI_THEME_PREFERENCE.toString()); if (storedCustomUiThemeConf == null) { // Only in case a config doesn't exist in DB we insert it Settings setting = @@ -100,12 +103,13 @@ private static void createDefaultConfiguration(OpenMetadataApplicationConfig app .withErrorColor("") .withWarningColor("") .withInfoColor(""))); - systemRepository.createNewSetting(setting); + Entity.getSystemRepository().createNewSetting(setting); } // Initialise Login Configuration // Initialise Logo Setting - Settings storedLoginConf = systemRepository.getConfigWithKey(LOGIN_CONFIGURATION.toString()); + Settings storedLoginConf = + Entity.getSystemRepository().getConfigWithKey(LOGIN_CONFIGURATION.toString()); if (storedLoginConf == null) { // Only in case a config doesn't exist in DB we insert it Settings setting = @@ -114,25 +118,26 @@ private static void createDefaultConfiguration(OpenMetadataApplicationConfig app .withConfigValue( new LoginConfiguration() .withMaxLoginFailAttempts(3) - .withAccessBlockTime(600) + .withAccessBlockTime(30) .withJwtTokenExpiryTime(3600)); - systemRepository.createNewSetting(setting); + Entity.getSystemRepository().createNewSetting(setting); } // Initialise Rbac Settings - Settings storedRbacSettings = systemRepository.getConfigWithKey(SEARCH_SETTINGS.toString()); + Settings storedRbacSettings = + Entity.getSystemRepository().getConfigWithKey(SEARCH_SETTINGS.toString()); if (storedRbacSettings == null) { // Only in case a config doesn't exist in DB we insert it Settings setting = new Settings() .withConfigType(SEARCH_SETTINGS) .withConfigValue(new SearchSettings().withEnableAccessControl(false)); - systemRepository.createNewSetting(setting); + Entity.getSystemRepository().createNewSetting(setting); } // Initialise Certification Settings Settings certificationSettings = - systemRepository.getConfigWithKey(ASSET_CERTIFICATION_SETTINGS.toString()); + Entity.getSystemRepository().getConfigWithKey(ASSET_CERTIFICATION_SETTINGS.toString()); if (certificationSettings == null) { Settings setting = new Settings() @@ -141,10 +146,25 @@ private static void createDefaultConfiguration(OpenMetadataApplicationConfig app new AssetCertificationSettings() .withAllowedClassification("Certification") .withValidityPeriod("P30D")); - systemRepository.createNewSetting(setting); + Entity.getSystemRepository().createNewSetting(setting); } - Settings lineageSettings = systemRepository.getConfigWithKey(LINEAGE_SETTINGS.toString()); + // Initialise Workflow Settings + Settings workflowSettings = + Entity.getSystemRepository().getConfigWithKey(WORKFLOW_SETTINGS.toString()); + if (workflowSettings == null) { + Settings setting = + new Settings() + .withConfigType(WORKFLOW_SETTINGS) + .withConfigValue( + new WorkflowSettings() + .withExecutorConfiguration(new ExecutorConfiguration()) + .withHistoryCleanUpConfiguration(new HistoryCleanUpConfiguration())); + Entity.getSystemRepository().createNewSetting(setting); + } + + Settings lineageSettings = + Entity.getSystemRepository().getConfigWithKey(LINEAGE_SETTINGS.toString()); if (lineageSettings == null) { // Only in case a config doesn't exist in DB we insert it Settings setting = @@ -155,7 +175,7 @@ private static void createDefaultConfiguration(OpenMetadataApplicationConfig app .withDownstreamDepth(2) .withUpstreamDepth(2) .withLineageLayer(LineageLayer.ENTITY_LINEAGE)); - systemRepository.createNewSetting(setting); + Entity.getSystemRepository().createNewSetting(setting); } } @@ -182,37 +202,52 @@ public static void invalidateSettings(String settingsName) { } } + private static SmtpSettings getDefaultSmtpSettings() { + return new SmtpSettings() + .withPassword(StringUtils.EMPTY) + .withEmailingEntity("OpenMetadata") + .withSupportUrl("https://slack.open-metadata.org") + .withEnableSmtpServer(Boolean.FALSE) + .withTransportationStrategy(SmtpSettings.TransportationStrategy.SMTP_TLS) + .withTemplates(SmtpSettings.Templates.OPENMETADATA); + } + static class SettingsLoader extends CacheLoader { @Override public @NonNull Settings load(@CheckForNull String settingsName) { Settings fetchedSettings; switch (SettingsType.fromValue(settingsName)) { case EMAIL_CONFIGURATION -> { - fetchedSettings = systemRepository.getEmailConfigInternal(); + fetchedSettings = Entity.getSystemRepository().getEmailConfigInternal(); + if (fetchedSettings == null) { + return new Settings() + .withConfigType(EMAIL_CONFIGURATION) + .withConfigValue(getDefaultSmtpSettings()); + } LOG.info("Loaded Email Setting"); } case SLACK_APP_CONFIGURATION -> { // Only if available - fetchedSettings = systemRepository.getSlackApplicationConfigInternal(); + fetchedSettings = Entity.getSystemRepository().getSlackApplicationConfigInternal(); LOG.info("Loaded Slack Application Configuration"); } case SLACK_BOT -> { // Only if available - fetchedSettings = systemRepository.getSlackbotConfigInternal(); + fetchedSettings = Entity.getSystemRepository().getSlackbotConfigInternal(); LOG.info("Loaded Slack Bot Configuration"); } case SLACK_INSTALLER -> { // Only if available - fetchedSettings = systemRepository.getSlackInstallerConfigInternal(); + fetchedSettings = Entity.getSystemRepository().getSlackInstallerConfigInternal(); LOG.info("Loaded Slack Installer Configuration"); } case SLACK_STATE -> { // Only if available - fetchedSettings = systemRepository.getSlackStateConfigInternal(); + fetchedSettings = Entity.getSystemRepository().getSlackStateConfigInternal(); LOG.info("Loaded Slack state Configuration"); } default -> { - fetchedSettings = systemRepository.getConfigWithKey(settingsName); + fetchedSettings = Entity.getSystemRepository().getConfigWithKey(settingsName); LOG.info("Loaded Setting {}", fetchedSettings.getConfigType()); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/storages/ContainerMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/storages/ContainerMapper.java new file mode 100644 index 000000000000..e16ea5865565 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/storages/ContainerMapper.java @@ -0,0 +1,25 @@ +package org.openmetadata.service.resources.storages; + +import static org.openmetadata.service.util.EntityUtil.getEntityReference; + +import org.openmetadata.schema.api.data.CreateContainer; +import org.openmetadata.schema.entity.data.Container; +import org.openmetadata.service.Entity; +import org.openmetadata.service.mapper.EntityMapper; + +public class ContainerMapper implements EntityMapper { + @Override + public Container createToEntity(CreateContainer create, String user) { + return copy(new Container(), create, user) + .withService(getEntityReference(Entity.STORAGE_SERVICE, create.getService())) + .withParent(create.getParent()) + .withDataModel(create.getDataModel()) + .withPrefix(create.getPrefix()) + .withNumberOfObjects(create.getNumberOfObjects()) + .withSize(create.getSize()) + .withFullPath(create.getFullPath()) + .withFileFormats(create.getFileFormats()) + .withSourceUrl(create.getSourceUrl()) + .withSourceHash(create.getSourceHash()); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/storages/ContainerResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/storages/ContainerResource.java index ae0f3a4edeac..806b18a893aa 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/storages/ContainerResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/storages/ContainerResource.java @@ -59,6 +59,7 @@ @Consumes(MediaType.APPLICATION_JSON) @Collection(name = "containers") public class ContainerResource extends EntityResource { + private final ContainerMapper mapper = new ContainerMapper(); public static final String COLLECTION_PATH = "v1/containers/"; static final String FIELDS = "parent,children,dataModel,owners,tags,followers,extension,domain,sourceHash"; @@ -239,7 +240,8 @@ public Response create( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateContainer create) { - Container container = getContainer(create, securityContext.getUserPrincipal().getName()); + Container container = + mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return create(uriInfo, securityContext, container); } @@ -320,7 +322,8 @@ public Response createOrUpdate( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateContainer create) { - Container container = getContainer(create, securityContext.getUserPrincipal().getName()); + Container container = + mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return createOrUpdate(uriInfo, securityContext, container); } @@ -543,19 +546,4 @@ public Response restoreContainer( @Valid RestoreEntity restore) { return restoreEntity(uriInfo, securityContext, restore.getId()); } - - private Container getContainer(CreateContainer create, String user) { - return repository - .copy(new Container(), create, user) - .withService(getEntityReference(Entity.STORAGE_SERVICE, create.getService())) - .withParent(create.getParent()) - .withDataModel(create.getDataModel()) - .withPrefix(create.getPrefix()) - .withNumberOfObjects(create.getNumberOfObjects()) - .withSize(create.getSize()) - .withFullPath(create.getFullPath()) - .withFileFormats(create.getFileFormats()) - .withSourceUrl(create.getSourceUrl()) - .withSourceHash(create.getSourceHash()); - } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/ClassificationMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/ClassificationMapper.java new file mode 100644 index 000000000000..b75fe17280d1 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/ClassificationMapper.java @@ -0,0 +1,15 @@ +package org.openmetadata.service.resources.tags; + +import org.openmetadata.schema.api.classification.CreateClassification; +import org.openmetadata.schema.entity.classification.Classification; +import org.openmetadata.service.mapper.EntityMapper; + +public class ClassificationMapper implements EntityMapper { + @Override + public Classification createToEntity(CreateClassification create, String user) { + return copy(new Classification(), create, user) + .withFullyQualifiedName(create.getName()) + .withProvider(create.getProvider()) + .withMutuallyExclusive(create.getMutuallyExclusive()); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/ClassificationResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/ClassificationResource.java index 18a60fa1a986..e25160ca878e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/ClassificationResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/ClassificationResource.java @@ -77,6 +77,7 @@ order = 4) // Initialize before TagResource, Glossary, and GlossaryTerms public class ClassificationResource extends EntityResource { + private final ClassificationMapper mapper = new ClassificationMapper(); public static final String TAG_COLLECTION_PATH = "/v1/classifications/"; static final String FIELDS = "usageCount,termCount"; @@ -302,7 +303,8 @@ public Response create( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateClassification create) { - Classification category = getClassification(create, securityContext); + Classification category = + mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return create(uriInfo, securityContext, category); } @@ -315,7 +317,8 @@ public Response createOrUpdate( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateClassification create) { - Classification category = getClassification(create, securityContext); + Classification category = + mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return createOrUpdate(uriInfo, securityContext, category); } @@ -447,18 +450,4 @@ public Response restore( @Valid RestoreEntity restore) { return restoreEntity(uriInfo, securityContext, restore.getId()); } - - private Classification getClassification( - CreateClassification create, SecurityContext securityContext) { - return getClassification(repository, create, securityContext.getUserPrincipal().getName()); - } - - public static Classification getClassification( - ClassificationRepository repository, CreateClassification create, String updatedBy) { - return repository - .copy(new Classification(), create, updatedBy) - .withFullyQualifiedName(create.getName()) - .withProvider(create.getProvider()) - .withMutuallyExclusive(create.getMutuallyExclusive()); - } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagMapper.java new file mode 100644 index 000000000000..e45961dc8b17 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagMapper.java @@ -0,0 +1,21 @@ +package org.openmetadata.service.resources.tags; + +import static org.openmetadata.service.Entity.CLASSIFICATION; +import static org.openmetadata.service.Entity.TAG; +import static org.openmetadata.service.util.EntityUtil.getEntityReference; + +import org.openmetadata.schema.api.classification.CreateTag; +import org.openmetadata.schema.entity.classification.Tag; +import org.openmetadata.service.mapper.EntityMapper; + +public class TagMapper implements EntityMapper { + @Override + public Tag createToEntity(CreateTag create, String user) { + return copy(new Tag(), create, user) + .withStyle(create.getStyle()) + .withParent(getEntityReference(TAG, create.getParent())) + .withClassification(getEntityReference(CLASSIFICATION, create.getClassification())) + .withProvider(create.getProvider()) + .withMutuallyExclusive(create.getMutuallyExclusive()); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagResource.java index 1ce89ee5054b..0a0fe3d62c43 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagResource.java @@ -15,7 +15,6 @@ import static org.openmetadata.service.Entity.ADMIN_USER_NAME; import static org.openmetadata.service.Entity.CLASSIFICATION; -import static org.openmetadata.service.Entity.TAG; import io.swagger.v3.oas.annotations.ExternalDocumentation; import io.swagger.v3.oas.annotations.Operation; @@ -91,6 +90,8 @@ name = "tags", order = 5) // initialize after Classification, and before Glossary and GlossaryTerm public class TagResource extends EntityResource { + private final ClassificationMapper classificationMapper = new ClassificationMapper(); + private final TagMapper mapper = new TagMapper(); public static final String TAG_COLLECTION_PATH = "/v1/tags/"; static final String FIELDS = "children,usageCount"; @@ -119,15 +120,14 @@ public void initialize(OpenMetadataApplicationConfig config) throws IOException CLASSIFICATION, ".*json/data/tags/.*\\.json$", LoadTags.class); for (LoadTags loadTags : loadTagsList) { Classification classification = - ClassificationResource.getClassification( - classificationRepository, loadTags.getCreateClassification(), ADMIN_USER_NAME); + classificationMapper.createToEntity(loadTags.getCreateClassification(), ADMIN_USER_NAME); classificationRepository.initializeEntity(classification); List tagsToCreate = new ArrayList<>(); for (CreateTag createTag : loadTags.getCreateTags()) { createTag.withClassification(classification.getName()); createTag.withProvider(classification.getProvider()); - Tag tag = getTag(createTag, ADMIN_USER_NAME); + Tag tag = mapper.createToEntity(createTag, ADMIN_USER_NAME); repository.setFullyQualifiedName(tag); // FQN required for ordering tags based on hierarchy tagsToCreate.add(tag); } @@ -352,7 +352,7 @@ public Tag getVersion( }) public Response create( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateTag create) { - Tag tag = getTag(securityContext, create); + Tag tag = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return create(uriInfo, securityContext, tag); } @@ -430,7 +430,7 @@ public Response patch( }) public Response createOrUpdate( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateTag create) { - Tag tag = getTag(create, securityContext.getUserPrincipal().getName()); + Tag tag = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return createOrUpdate(uriInfo, securityContext, tag); } @@ -563,18 +563,4 @@ public Tag addHref(UriInfo uriInfo, Tag tag) { Entity.withHref(uriInfo, tag.getParent()); return tag; } - - private Tag getTag(SecurityContext securityContext, CreateTag create) { - return getTag(create, securityContext.getUserPrincipal().getName()); - } - - private Tag getTag(CreateTag create, String updateBy) { - return repository - .copy(new Tag(), create, updateBy) - .withStyle(create.getStyle()) - .withParent(getEntityReference(TAG, create.getParent())) - .withClassification(getEntityReference(CLASSIFICATION, create.getClassification())) - .withProvider(create.getProvider()) - .withMutuallyExclusive(create.getMutuallyExclusive()); - } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/PersonaMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/PersonaMapper.java new file mode 100644 index 000000000000..e33d6f7b441e --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/PersonaMapper.java @@ -0,0 +1,15 @@ +package org.openmetadata.service.resources.teams; + +import org.openmetadata.schema.api.teams.CreatePersona; +import org.openmetadata.schema.entity.teams.Persona; +import org.openmetadata.service.Entity; +import org.openmetadata.service.mapper.EntityMapper; +import org.openmetadata.service.util.EntityUtil; + +public class PersonaMapper implements EntityMapper { + @Override + public Persona createToEntity(CreatePersona create, String user) { + return copy(new Persona(), create, user) + .withUsers(EntityUtil.toEntityReferences(create.getUsers(), Entity.USER)); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/PersonaResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/PersonaResource.java index 7f9db2ee7beb..e084f9131ea5 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/PersonaResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/PersonaResource.java @@ -46,7 +46,6 @@ import org.openmetadata.service.resources.Collection; import org.openmetadata.service.resources.EntityResource; import org.openmetadata.service.security.Authorizer; -import org.openmetadata.service.util.EntityUtil; import org.openmetadata.service.util.ResultList; @Slf4j @@ -60,6 +59,7 @@ @Consumes(MediaType.APPLICATION_JSON) @Collection(name = "personas", order = 2) public class PersonaResource extends EntityResource { + private final PersonaMapper mapper = new PersonaMapper(); public static final String COLLECTION_PATH = "/v1/personas"; static final String FIELDS = "users"; @@ -278,7 +278,7 @@ public Persona getVersion( }) public Response create( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreatePersona cp) { - Persona persona = getPersona(cp, securityContext.getUserPrincipal().getName()); + Persona persona = mapper.createToEntity(cp, securityContext.getUserPrincipal().getName()); return create(uriInfo, securityContext, persona); } @@ -299,7 +299,7 @@ public Response create( }) public Response createOrUpdate( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreatePersona cp) { - Persona persona = getPersona(cp, securityContext.getUserPrincipal().getName()); + Persona persona = mapper.createToEntity(cp, securityContext.getUserPrincipal().getName()); return createOrUpdate(uriInfo, securityContext, persona); } @@ -398,10 +398,4 @@ public Response delete( String name) { return deleteByName(uriInfo, securityContext, name, false, true); } - - private Persona getPersona(CreatePersona cp, String user) { - return repository - .copy(new Persona(), cp, user) - .withUsers(EntityUtil.toEntityReferences(cp.getUsers(), Entity.USER)); - } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/RoleMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/RoleMapper.java new file mode 100644 index 000000000000..3042da9e5226 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/RoleMapper.java @@ -0,0 +1,20 @@ +package org.openmetadata.service.resources.teams; + +import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; +import static org.openmetadata.service.util.EntityUtil.getEntityReferences; + +import org.openmetadata.schema.api.teams.CreateRole; +import org.openmetadata.schema.entity.teams.Role; +import org.openmetadata.service.Entity; +import org.openmetadata.service.mapper.EntityMapper; + +public class RoleMapper implements EntityMapper { + @Override + public Role createToEntity(CreateRole create, String user) { + if (nullOrEmpty(create.getPolicies())) { + throw new IllegalArgumentException("At least one policy is required to create a role"); + } + return copy(new Role(), create, user) + .withPolicies(getEntityReferences(Entity.POLICY, create.getPolicies())); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/RoleResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/RoleResource.java index 9b5631c01707..aa3f2f519af6 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/RoleResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/RoleResource.java @@ -13,8 +13,6 @@ package org.openmetadata.service.resources.teams; -import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; - import io.dropwizard.jersey.PATCH; import io.swagger.v3.oas.annotations.ExternalDocumentation; import io.swagger.v3.oas.annotations.Operation; @@ -81,6 +79,7 @@ requiredForOps = true) // Load roles after PolicyResource are loaded at Order 0 @Slf4j public class RoleResource extends EntityResource { + private final RoleMapper mapper = new RoleMapper(); public static final String COLLECTION_PATH = "/v1/roles/"; public static final String FIELDS = "policies,teams,users"; @@ -335,7 +334,7 @@ public Response create( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateRole createRole) { - Role role = getRole(createRole, securityContext.getUserPrincipal().getName()); + Role role = mapper.createToEntity(createRole, securityContext.getUserPrincipal().getName()); return create(uriInfo, securityContext, role); } @@ -358,7 +357,7 @@ public Response createOrUpdateRole( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateRole createRole) { - Role role = getRole(createRole, securityContext.getUserPrincipal().getName()); + Role role = mapper.createToEntity(createRole, securityContext.getUserPrincipal().getName()); return createOrUpdate(uriInfo, securityContext, role); } @@ -487,18 +486,4 @@ public Response restoreRole( @Valid RestoreEntity restore) { return restoreEntity(uriInfo, securityContext, restore.getId()); } - - private Role getRole(CreateRole create, String user) { - if (nullOrEmpty(create.getPolicies())) { - throw new IllegalArgumentException("At least one policy is required to create a role"); - } - return repository - .copy(new Role(), create, user) - .withPolicies(getEntityReferences(Entity.POLICY, create.getPolicies())); - } - - public static EntityReference getRole(String roleName) { - RoleRepository roleRepository = (RoleRepository) Entity.getEntityRepository(Entity.ROLE); - return roleRepository.getReferenceByName(roleName, Include.NON_DELETED); - } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/TeamMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/TeamMapper.java new file mode 100644 index 000000000000..6118c5ade5d7 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/TeamMapper.java @@ -0,0 +1,33 @@ +package org.openmetadata.service.resources.teams; + +import static org.openmetadata.service.exception.CatalogExceptionMessage.CREATE_GROUP; +import static org.openmetadata.service.exception.CatalogExceptionMessage.CREATE_ORGANIZATION; + +import org.openmetadata.schema.api.teams.CreateTeam; +import org.openmetadata.schema.entity.teams.Team; +import org.openmetadata.service.Entity; +import org.openmetadata.service.mapper.EntityMapper; +import org.openmetadata.service.util.EntityUtil; + +public class TeamMapper implements EntityMapper { + @Override + public Team createToEntity(CreateTeam create, String user) { + if (create.getTeamType().equals(CreateTeam.TeamType.ORGANIZATION)) { + throw new IllegalArgumentException(CREATE_ORGANIZATION); + } + if (create.getTeamType().equals(CreateTeam.TeamType.GROUP) && create.getChildren() != null) { + throw new IllegalArgumentException(CREATE_GROUP); + } + return copy(new Team(), create, user) + .withProfile(create.getProfile()) + .withIsJoinable(create.getIsJoinable()) + .withUsers(EntityUtil.toEntityReferences(create.getUsers(), Entity.USER)) + .withDefaultRoles(EntityUtil.toEntityReferences(create.getDefaultRoles(), Entity.ROLE)) + .withTeamType(create.getTeamType()) + .withParents(EntityUtil.toEntityReferences(create.getParents(), Entity.TEAM)) + .withChildren(EntityUtil.toEntityReferences(create.getChildren(), Entity.TEAM)) + .withPolicies(EntityUtil.toEntityReferences(create.getPolicies(), Entity.POLICY)) + .withEmail(create.getEmail()) + .withDomains(EntityUtil.getEntityReferences(Entity.DOMAIN, create.getDomains())); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/TeamResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/TeamResource.java index c30c3178bb25..e3374cbca785 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/TeamResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/TeamResource.java @@ -14,8 +14,6 @@ package org.openmetadata.service.resources.teams; import static org.openmetadata.common.utils.CommonUtil.listOf; -import static org.openmetadata.service.exception.CatalogExceptionMessage.CREATE_GROUP; -import static org.openmetadata.service.exception.CatalogExceptionMessage.CREATE_ORGANIZATION; import io.dropwizard.jersey.PATCH; import io.swagger.v3.oas.annotations.ExternalDocumentation; @@ -52,7 +50,6 @@ import lombok.extern.slf4j.Slf4j; import org.openmetadata.schema.api.data.RestoreEntity; import org.openmetadata.schema.api.teams.CreateTeam; -import org.openmetadata.schema.api.teams.CreateTeam.TeamType; import org.openmetadata.schema.entity.teams.Team; import org.openmetadata.schema.entity.teams.TeamHierarchy; import org.openmetadata.schema.type.ChangeEvent; @@ -72,9 +69,7 @@ import org.openmetadata.service.resources.Collection; import org.openmetadata.service.resources.EntityResource; import org.openmetadata.service.security.Authorizer; -import org.openmetadata.service.security.policyevaluator.OperationContext; import org.openmetadata.service.util.CSVExportResponse; -import org.openmetadata.service.util.EntityUtil; import org.openmetadata.service.util.JsonUtils; import org.openmetadata.service.util.ResultList; @@ -93,6 +88,7 @@ requiredForOps = true) // Load after roles, and policy resources public class TeamResource extends EntityResource { public static final String COLLECTION_PATH = "/v1/teams/"; + private final TeamMapper mapper = new TeamMapper(); static final String FIELDS = "owners,profile,users,owns,defaultRoles,parents,children,policies,userCount,childrenCount,domains"; @@ -379,7 +375,7 @@ public Team getVersion( }) public Response create( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateTeam ct) { - Team team = getTeam(ct, securityContext.getUserPrincipal().getName()); + Team team = mapper.createToEntity(ct, securityContext.getUserPrincipal().getName()); return create(uriInfo, securityContext, team); } @@ -400,7 +396,7 @@ public Response create( }) public Response createOrUpdate( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateTeam ct) { - Team team = getTeam(ct, securityContext.getUserPrincipal().getName()); + Team team = mapper.createToEntity(ct, securityContext.getUserPrincipal().getName()); return createOrUpdate(uriInfo, securityContext, team); } @@ -687,10 +683,6 @@ public Response updateTeamUsers( @Context SecurityContext securityContext, @PathParam("teamId") UUID teamId, List users) { - - OperationContext operationContext = - new OperationContext(entityType, MetadataOperation.EDIT_ALL); - authorizer.authorize(securityContext, operationContext, getResourceContextById(teamId)); return repository .updateTeamUsers(securityContext.getUserPrincipal().getName(), teamId, users) .toResponse(); @@ -721,10 +713,6 @@ public Response deleteTeamUser( @Parameter(description = "Id of the user being removed", schema = @Schema(type = "string")) @PathParam("userId") String userId) { - - OperationContext operationContext = - new OperationContext(entityType, MetadataOperation.EDIT_ALL); - authorizer.authorize(securityContext, operationContext, getResourceContextById(teamId)); return repository .deleteTeamUser( securityContext.getUserPrincipal().getName(), teamId, UUID.fromString(userId)) @@ -761,25 +749,4 @@ public Response importCsvAsync( String csv) { return importCsvInternalAsync(securityContext, name, csv, dryRun); } - - private Team getTeam(CreateTeam ct, String user) { - if (ct.getTeamType().equals(TeamType.ORGANIZATION)) { - throw new IllegalArgumentException(CREATE_ORGANIZATION); - } - if (ct.getTeamType().equals(TeamType.GROUP) && ct.getChildren() != null) { - throw new IllegalArgumentException(CREATE_GROUP); - } - return repository - .copy(new Team(), ct, user) - .withProfile(ct.getProfile()) - .withIsJoinable(ct.getIsJoinable()) - .withUsers(EntityUtil.toEntityReferences(ct.getUsers(), Entity.USER)) - .withDefaultRoles(EntityUtil.toEntityReferences(ct.getDefaultRoles(), Entity.ROLE)) - .withTeamType(ct.getTeamType()) - .withParents(EntityUtil.toEntityReferences(ct.getParents(), Entity.TEAM)) - .withChildren(EntityUtil.toEntityReferences(ct.getChildren(), Entity.TEAM)) - .withPolicies(EntityUtil.toEntityReferences(ct.getPolicies(), Entity.POLICY)) - .withEmail(ct.getEmail()) - .withDomains(EntityUtil.getEntityReferences(Entity.DOMAIN, ct.getDomains())); - } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/UserMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/UserMapper.java new file mode 100644 index 000000000000..10ce880b7dc9 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/UserMapper.java @@ -0,0 +1,33 @@ +package org.openmetadata.service.resources.teams; + +import java.util.UUID; +import org.openmetadata.schema.api.teams.CreateUser; +import org.openmetadata.schema.entity.teams.User; +import org.openmetadata.schema.utils.EntityInterfaceUtil; +import org.openmetadata.service.Entity; +import org.openmetadata.service.mapper.EntityMapper; +import org.openmetadata.service.util.EntityUtil; + +public class UserMapper implements EntityMapper { + @Override + public User createToEntity(CreateUser create, String user) { + return new User() + .withId(UUID.randomUUID()) + .withName(create.getName().toLowerCase()) + .withFullyQualifiedName(EntityInterfaceUtil.quoteName(create.getName().toLowerCase())) + .withEmail(create.getEmail().toLowerCase()) + .withDescription(create.getDescription()) + .withDisplayName(create.getDisplayName()) + .withIsBot(create.getIsBot()) + .withIsAdmin(create.getIsAdmin()) + .withProfile(create.getProfile()) + .withPersonas(create.getPersonas()) + .withDefaultPersona(create.getDefaultPersona()) + .withTimezone(create.getTimezone()) + .withUpdatedBy(user.toLowerCase()) + .withUpdatedAt(System.currentTimeMillis()) + .withTeams(EntityUtil.toEntityReferences(create.getTeams(), Entity.TEAM)) + .withRoles(EntityUtil.toEntityReferences(create.getRoles(), Entity.ROLE)) + .withDomains(EntityUtil.getEntityReferences(Entity.DOMAIN, create.getDomains())); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/topics/TopicMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/topics/TopicMapper.java new file mode 100644 index 000000000000..470bfdb82840 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/topics/TopicMapper.java @@ -0,0 +1,27 @@ +package org.openmetadata.service.resources.topics; + +import static org.openmetadata.service.util.EntityUtil.getEntityReference; + +import org.openmetadata.schema.api.data.CreateTopic; +import org.openmetadata.schema.entity.data.Topic; +import org.openmetadata.service.Entity; +import org.openmetadata.service.mapper.EntityMapper; + +public class TopicMapper implements EntityMapper { + @Override + public Topic createToEntity(CreateTopic create, String user) { + return copy(new Topic(), create, user) + .withService(getEntityReference(Entity.MESSAGING_SERVICE, create.getService())) + .withPartitions(create.getPartitions()) + .withMessageSchema(create.getMessageSchema()) + .withCleanupPolicies(create.getCleanupPolicies()) + .withMaximumMessageSize(create.getMaximumMessageSize()) + .withMinimumInSyncReplicas(create.getMinimumInSyncReplicas()) + .withRetentionSize(create.getRetentionSize()) + .withRetentionTime(create.getRetentionTime()) + .withReplicationFactor(create.getReplicationFactor()) + .withTopicConfig(create.getTopicConfig()) + .withSourceUrl(create.getSourceUrl()) + .withSourceHash(create.getSourceHash()); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/topics/TopicResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/topics/TopicResource.java index a5a2916a34bf..8a7f5b46ae35 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/topics/TopicResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/topics/TopicResource.java @@ -77,6 +77,7 @@ @Collection(name = "topics") public class TopicResource extends EntityResource { public static final String COLLECTION_PATH = "v1/topics/"; + private final TopicMapper mapper = new TopicMapper(); static final String FIELDS = "owners,followers,tags,extension,domain,dataProducts,sourceHash"; @Override @@ -302,7 +303,7 @@ public Response create( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateTopic create) { - Topic topic = getTopic(create, securityContext.getUserPrincipal().getName()); + Topic topic = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return create(uriInfo, securityContext, topic); } @@ -381,7 +382,7 @@ public Response createOrUpdate( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateTopic create) { - Topic topic = getTopic(create, securityContext.getUserPrincipal().getName()); + Topic topic = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return createOrUpdate(uriInfo, securityContext, topic); } @@ -597,21 +598,4 @@ public Response restoreTopic( @Valid RestoreEntity restore) { return restoreEntity(uriInfo, securityContext, restore.getId()); } - - private Topic getTopic(CreateTopic create, String user) { - return repository - .copy(new Topic(), create, user) - .withService(getEntityReference(Entity.MESSAGING_SERVICE, create.getService())) - .withPartitions(create.getPartitions()) - .withMessageSchema(create.getMessageSchema()) - .withCleanupPolicies(create.getCleanupPolicies()) - .withMaximumMessageSize(create.getMaximumMessageSize()) - .withMinimumInSyncReplicas(create.getMinimumInSyncReplicas()) - .withRetentionSize(create.getRetentionSize()) - .withRetentionTime(create.getRetentionTime()) - .withReplicationFactor(create.getReplicationFactor()) - .withTopicConfig(create.getTopicConfig()) - .withSourceUrl(create.getSourceUrl()) - .withSourceHash(create.getSourceHash()); - } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/types/TypeMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/types/TypeMapper.java new file mode 100644 index 000000000000..e8b21b942f5c --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/types/TypeMapper.java @@ -0,0 +1,15 @@ +package org.openmetadata.service.resources.types; + +import org.openmetadata.schema.api.CreateType; +import org.openmetadata.schema.entity.Type; +import org.openmetadata.service.mapper.EntityMapper; + +public class TypeMapper implements EntityMapper { + @Override + public Type createToEntity(CreateType create, String user) { + return copy(new Type(), create, user) + .withFullyQualifiedName(create.getName()) + .withCategory(create.getCategory()) + .withSchema(create.getSchema()); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/types/TypeResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/types/TypeResource.java index 65c9b2a34364..99e8330e6016 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/types/TypeResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/types/TypeResource.java @@ -84,6 +84,7 @@ @Slf4j public class TypeResource extends EntityResource { public static final String COLLECTION_PATH = "v1/metadata/types/"; + private final TypeMapper mapper = new TypeMapper(); public SchemaFieldExtractor extractor; @Override @@ -117,7 +118,9 @@ public void initialize(OpenMetadataApplicationConfig config) { type.setCustomProperties(storedType.getCustomProperties()); } } catch (Exception e) { - LOG.debug("Creating entity that does not exist ", e); + LOG.debug( + "Type '{}' not found. Proceeding to add new type entity in database.", + type.getName()); } this.repository.createOrUpdate(null, type); this.repository.addToRegistry(type); @@ -324,7 +327,7 @@ public Response create( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateType create) { - Type type = getType(create, securityContext.getUserPrincipal().getName()); + Type type = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return create(uriInfo, securityContext, type); } @@ -403,7 +406,7 @@ public Response createOrUpdate( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateType create) { - Type type = getType(create, securityContext.getUserPrincipal().getName()); + Type type = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); return createOrUpdate(uriInfo, securityContext, type); } @@ -520,12 +523,4 @@ public Response getAllCustomPropertiesByEntityType( .build(); } } - - private Type getType(CreateType create, String user) { - return repository - .copy(new Type(), create, user) - .withFullyQualifiedName(create.getName()) - .withCategory(create.getCategory()) - .withSchema(create.getSchema()); - } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java index a02aaad5168e..88fb80571a38 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java @@ -42,6 +42,7 @@ public interface SearchClient { String DELETE = "delete"; String GLOBAL_SEARCH_ALIAS = "all"; + String DATA_ASSET_SEARCH_ALIAS = "dataAsset"; String GLOSSARY_TERM_SEARCH_INDEX = "glossary_term_search_index"; String TAG_SEARCH_INDEX = "tag_search_index"; String DEFAULT_UPDATE_SCRIPT = "for (k in params.keySet()) { ctx._source.put(k, params.get(k)) }"; @@ -74,6 +75,8 @@ public interface SearchClient { String REMOVE_TAGS_CHILDREN_SCRIPT = "for (int i = 0; i < ctx._source.tags.length; i++) { if (ctx._source.tags[i].tagFQN == params.fqn) { ctx._source.tags.remove(i) }}"; + String REMOVE_DATA_PRODUCTS_CHILDREN_SCRIPT = + "for (int i = 0; i < ctx._source.dataProducts.length; i++) { if (ctx._source.dataProducts[i].fullyQualifiedName == params.fqn) { ctx._source.dataProducts.remove(i) }}"; String UPDATE_CERTIFICATION_SCRIPT = "if (ctx._source.certification != null && ctx._source.certification.tagLabel != null) {ctx._source.certification.tagLabel.style = params.style; ctx._source.certification.tagLabel.description = params.description; ctx._source.certification.tagLabel.tagFQN = params.tagFQN; ctx._source.certification.tagLabel.name = params.name; }"; @@ -214,6 +217,15 @@ default ResultList listPageHierarchy(String parent, String pageType, int offset, Response.Status.NOT_IMPLEMENTED, NOT_IMPLEMENTED_ERROR_TYPE, NOT_IMPLEMENTED_METHOD); } + /* + Used for listing knowledge page hierarchy for a given active Page and page type, used in Elastic/Open SearchClientExtension + */ + default ResultList listPageHierarchyForActivePage( + String activeFqn, String pageType, int offset, int limit) { + throw new CustomExceptionMessage( + Response.Status.NOT_IMPLEMENTED, NOT_IMPLEMENTED_ERROR_TYPE, NOT_IMPLEMENTED_METHOD); + } + @SuppressWarnings("unused") default ResultList searchPageHierarchy(String query, String pageType, int offset, int limit) { throw new CustomExceptionMessage( @@ -378,4 +390,6 @@ static boolean shouldApplyRbacConditions( && !subjectContext.isBot() && rbacConditionEvaluator != null; } + + SearchHealthStatus getSearchHealthStatus() throws IOException; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchHealthStatus.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchHealthStatus.java new file mode 100644 index 000000000000..f3d2b5cf0430 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchHealthStatus.java @@ -0,0 +1,12 @@ +package org.openmetadata.service.search; + +import lombok.Getter; + +@Getter +public class SearchHealthStatus { + public SearchHealthStatus(String status) { + this.status = status; + } + + String status; +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchListFilter.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchListFilter.java index bf7580aaf878..be8b447c1c72 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchListFilter.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchListFilter.java @@ -257,8 +257,8 @@ private String getTestSuiteCondition() { boolean includeEmptyTestSuites = Boolean.parseBoolean(getQueryParam("includeEmptyTestSuites")); if (testSuiteType != null) { - boolean executable = !testSuiteType.equals("logical"); - conditions.add(String.format("{\"term\": {\"executable\": \"%s\"}}", executable)); + boolean basic = !testSuiteType.equals("logical"); + conditions.add(String.format("{\"term\": {\"basic\": \"%s\"}}", basic)); } if (!includeEmptyTestSuites) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java index 98f4477afdf1..015aa7639fa5 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java @@ -16,6 +16,7 @@ import static org.openmetadata.service.search.SearchClient.PROPAGATE_ENTITY_REFERENCE_FIELD_SCRIPT; import static org.openmetadata.service.search.SearchClient.PROPAGATE_FIELD_SCRIPT; import static org.openmetadata.service.search.SearchClient.PROPAGATE_TEST_SUITES_SCRIPT; +import static org.openmetadata.service.search.SearchClient.REMOVE_DATA_PRODUCTS_CHILDREN_SCRIPT; import static org.openmetadata.service.search.SearchClient.REMOVE_DOMAINS_CHILDREN_SCRIPT; import static org.openmetadata.service.search.SearchClient.REMOVE_OWNERS_SCRIPT; import static org.openmetadata.service.search.SearchClient.REMOVE_PROPAGATED_ENTITY_REFERENCE_FIELD_SCRIPT; @@ -38,6 +39,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; @@ -726,6 +728,13 @@ public void deleteOrUpdateChildren(EntityInterface entity, IndexMapping indexMap indexMapping.getChildAliases(clusterAlias), List.of(new ImmutablePair<>(entityType + ".id", docId))); } + case Entity.DATA_PRODUCT -> searchClient.updateChildren( + GLOBAL_SEARCH_ALIAS, + new ImmutablePair<>("dataProducts.id", docId), + new ImmutablePair<>( + REMOVE_DATA_PRODUCTS_CHILDREN_SCRIPT, + Collections.singletonMap("fqn", entity.getFullyQualifiedName()))); + case Entity.TAG, Entity.GLOSSARY_TERM -> searchClient.updateChildren( GLOBAL_SEARCH_ALIAS, new ImmutablePair<>("tags.tagFQN", entity.getFullyQualifiedName()), @@ -744,7 +753,7 @@ public void deleteOrUpdateChildren(EntityInterface entity, IndexMapping indexMap } case Entity.TEST_SUITE -> { TestSuite testSuite = (TestSuite) entity; - if (Boolean.TRUE.equals(testSuite.getExecutable())) { + if (Boolean.TRUE.equals(testSuite.getBasic())) { searchClient.deleteEntityByFields( indexMapping.getChildAliases(clusterAlias), List.of(new ImmutablePair<>("testSuite.id", docId))); @@ -1057,4 +1066,8 @@ public List getEntitiesContainingFQNFromES( } return new ArrayList<>(); } + + public Set getSearchEntities() { + return new HashSet<>(entityIndexMap.keySet()); + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java index fec119ce0a46..91cfe5022452 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java @@ -13,6 +13,8 @@ import static org.openmetadata.service.Entity.QUERY; import static org.openmetadata.service.Entity.RAW_COST_ANALYSIS_REPORT_DATA; import static org.openmetadata.service.Entity.TABLE; +import static org.openmetadata.service.events.scheduled.ServicesStatusJobHandler.HEALTHY_STATUS; +import static org.openmetadata.service.events.scheduled.ServicesStatusJobHandler.UNHEALTHY_STATUS; import static org.openmetadata.service.exception.CatalogGenericExceptionMapper.getResponse; import static org.openmetadata.service.search.EntityBuilderConstant.API_RESPONSE_SCHEMA_FIELD; import static org.openmetadata.service.search.EntityBuilderConstant.API_RESPONSE_SCHEMA_FIELD_KEYWORD; @@ -38,6 +40,8 @@ import com.fasterxml.jackson.databind.JsonNode; import es.org.elasticsearch.ElasticsearchStatusException; +import es.org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; +import es.org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import es.org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import es.org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; import es.org.elasticsearch.action.bulk.BulkRequest; @@ -60,6 +64,7 @@ import es.org.elasticsearch.client.indices.GetMappingsRequest; import es.org.elasticsearch.client.indices.GetMappingsResponse; import es.org.elasticsearch.client.indices.PutMappingRequest; +import es.org.elasticsearch.cluster.health.ClusterHealthStatus; import es.org.elasticsearch.cluster.metadata.MappingMetadata; import es.org.elasticsearch.common.lucene.search.function.CombineFunction; import es.org.elasticsearch.common.lucene.search.function.FieldValueFactorFunction; @@ -127,6 +132,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import java.util.stream.Stream; import javax.json.JsonObject; import javax.net.ssl.SSLContext; @@ -148,7 +154,7 @@ import org.openmetadata.schema.dataInsight.custom.DataInsightCustomChart; import org.openmetadata.schema.dataInsight.custom.DataInsightCustomChartResultList; import org.openmetadata.schema.dataInsight.custom.FormulaHolder; -import org.openmetadata.schema.entity.data.EntityHierarchy__1; +import org.openmetadata.schema.entity.data.EntityHierarchy; import org.openmetadata.schema.entity.data.Table; import org.openmetadata.schema.service.configuration.elasticsearch.ElasticSearchConfiguration; import org.openmetadata.schema.tests.DataQualityReport; @@ -165,6 +171,7 @@ import org.openmetadata.service.jdbi3.TestCaseResultRepository; import org.openmetadata.service.search.SearchAggregation; import org.openmetadata.service.search.SearchClient; +import org.openmetadata.service.search.SearchHealthStatus; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.SearchRequest; import org.openmetadata.service.search.SearchSortFilter; @@ -468,50 +475,10 @@ public Response search(SearchRequest request, SubjectContext subjectContext) thr .getIndexMapping(GLOSSARY_TERM) .getIndexName(clusterAlias))) { searchSourceBuilder.query(QueryBuilders.boolQuery().must(searchSourceBuilder.query())); - - if (request.isGetHierarchy()) { - QueryBuilder baseQuery = - QueryBuilders.boolQuery() - .should(searchSourceBuilder.query()) - .should(QueryBuilders.matchPhraseQuery("fullyQualifiedName", request.getQuery())) - .should(QueryBuilders.matchPhraseQuery("name", request.getQuery())) - .should(QueryBuilders.matchPhraseQuery("displayName", request.getQuery())) - .should( - QueryBuilders.matchPhraseQuery( - "glossary.fullyQualifiedName", request.getQuery())) - .should(QueryBuilders.matchPhraseQuery("glossary.displayName", request.getQuery())) - .must(QueryBuilders.matchQuery("status", "Approved")) - .minimumShouldMatch(1); - searchSourceBuilder.query(baseQuery); - - SearchResponse searchResponse = - client.search( - new es.org.elasticsearch.action.search.SearchRequest(request.getIndex()) - .source(searchSourceBuilder), - RequestOptions.DEFAULT); - - // Extract parent terms from aggregation - BoolQueryBuilder parentTermQueryBuilder = QueryBuilders.boolQuery(); - Terms parentTerms = searchResponse.getAggregations().get("fqnParts_agg"); - - // Build es query to get parent terms for the user input query , to build correct hierarchy - if (!parentTerms.getBuckets().isEmpty() && !request.getQuery().equals("*")) { - parentTerms.getBuckets().stream() - .map(Terms.Bucket::getKeyAsString) - .forEach( - parentTerm -> - parentTermQueryBuilder.should( - QueryBuilders.matchQuery("fullyQualifiedName", parentTerm))); - - searchSourceBuilder.query( - parentTermQueryBuilder - .minimumShouldMatch(1) - .must(QueryBuilders.matchQuery("status", "Approved"))); - } - searchSourceBuilder.sort(SortBuilders.fieldSort("fullyQualifiedName").order(SortOrder.ASC)); - } } + buildHierarchyQuery(request, searchSourceBuilder, client); + /* for performance reasons ElasticSearch doesn't provide accurate hits if we enable trackTotalHits parameter it will try to match every result, count and return hits however in most cases for search results an approximate value is good enough. @@ -579,34 +546,108 @@ public Response getDocByID(String indexName, String entityId) throws IOException return getResponse(NOT_FOUND, "Document not found."); } + private void buildHierarchyQuery( + SearchRequest request, SearchSourceBuilder searchSourceBuilder, RestHighLevelClient client) + throws IOException { + + if (!request.isGetHierarchy()) { + return; + } + + String indexName = request.getIndex(); + String glossaryTermIndex = + Entity.getSearchRepository().getIndexMapping(GLOSSARY_TERM).getIndexName(clusterAlias); + String domainIndex = + Entity.getSearchRepository().getIndexMapping(DOMAIN).getIndexName(clusterAlias); + + BoolQueryBuilder baseQuery = + QueryBuilders.boolQuery() + .should(searchSourceBuilder.query()) + .should(QueryBuilders.matchPhraseQuery("fullyQualifiedName", request.getQuery())) + .should(QueryBuilders.matchPhraseQuery("name", request.getQuery())) + .should(QueryBuilders.matchPhraseQuery("displayName", request.getQuery())); + + if (indexName.equalsIgnoreCase(glossaryTermIndex)) { + baseQuery + .should(QueryBuilders.matchPhraseQuery("glossary.fullyQualifiedName", request.getQuery())) + .should(QueryBuilders.matchPhraseQuery("glossary.displayName", request.getQuery())) + .must(QueryBuilders.matchQuery("status", "Approved")); + } else if (indexName.equalsIgnoreCase(domainIndex)) { + baseQuery + .should(QueryBuilders.matchPhraseQuery("parent.fullyQualifiedName", request.getQuery())) + .should(QueryBuilders.matchPhraseQuery("parent.displayName", request.getQuery())); + } + + baseQuery.minimumShouldMatch(1); + searchSourceBuilder.query(baseQuery); + + SearchResponse searchResponse = + client.search( + new es.org.elasticsearch.action.search.SearchRequest(request.getIndex()) + .source(searchSourceBuilder), + RequestOptions.DEFAULT); + + Terms parentTerms = searchResponse.getAggregations().get("fqnParts_agg"); + + // Build es query to get parent terms for the user input query , to build correct hierarchy + // In case of default search , no need to get parent terms they are already present in the + // response + if (parentTerms != null + && !parentTerms.getBuckets().isEmpty() + && !request.getQuery().equals("*")) { + BoolQueryBuilder parentTermQueryBuilder = QueryBuilders.boolQuery(); + + parentTerms.getBuckets().stream() + .map(Terms.Bucket::getKeyAsString) + .forEach( + parentTerm -> + parentTermQueryBuilder.should( + QueryBuilders.matchQuery("fullyQualifiedName", parentTerm))); + if (indexName.equalsIgnoreCase(glossaryTermIndex)) { + parentTermQueryBuilder + .minimumShouldMatch(1) + .must(QueryBuilders.matchQuery("status", "Approved")); + } else { + parentTermQueryBuilder.minimumShouldMatch(1); + } + searchSourceBuilder.query(parentTermQueryBuilder); + } + + searchSourceBuilder.sort(SortBuilders.fieldSort("fullyQualifiedName").order(SortOrder.ASC)); + } + public List buildSearchHierarchy(SearchRequest request, SearchResponse searchResponse) { List response = new ArrayList<>(); - if (request - .getIndex() - .equalsIgnoreCase( - Entity.getSearchRepository() - .getIndexMapping(GLOSSARY_TERM) - .getIndexName(clusterAlias))) { + + String indexName = request.getIndex(); + String glossaryTermIndex = + Entity.getSearchRepository().getIndexMapping(GLOSSARY_TERM).getIndexName(clusterAlias); + String domainIndex = + Entity.getSearchRepository().getIndexMapping(DOMAIN).getIndexName(clusterAlias); + + if (indexName.equalsIgnoreCase(glossaryTermIndex)) { response = buildGlossaryTermSearchHierarchy(searchResponse); + } else if (indexName.equalsIgnoreCase(domainIndex)) { + response = buildDomainSearchHierarchy(searchResponse); } return response; } - public List buildGlossaryTermSearchHierarchy(SearchResponse searchResponse) { - Map termMap = + public List buildGlossaryTermSearchHierarchy(SearchResponse searchResponse) { + Map termMap = new LinkedHashMap<>(); // termMap represent glossary terms - Map rootTerms = + Map rootTerms = new LinkedHashMap<>(); // rootTerms represent glossaries for (var hit : searchResponse.getHits().getHits()) { String jsonSource = hit.getSourceAsString(); - EntityHierarchy__1 term = JsonUtils.readValue(jsonSource, EntityHierarchy__1.class); - EntityHierarchy__1 glossaryInfo = + EntityHierarchy term = JsonUtils.readValue(jsonSource, EntityHierarchy.class); + EntityHierarchy glossaryInfo = JsonUtils.readTree(jsonSource).path("glossary").isMissingNode() ? null : JsonUtils.convertValue( - JsonUtils.readTree(jsonSource).path("glossary"), EntityHierarchy__1.class); + JsonUtils.readTree(jsonSource).path("glossary"), EntityHierarchy.class); if (glossaryInfo != null) { rootTerms.putIfAbsent(glossaryInfo.getFullyQualifiedName(), glossaryInfo); @@ -626,15 +667,15 @@ public List buildGlossaryTermSearchHierarchy(SearchResponse String termFQN = term.getFullyQualifiedName(); if (parentFQN != null && termMap.containsKey(parentFQN)) { - EntityHierarchy__1 parentTerm = termMap.get(parentFQN); - List children = parentTerm.getChildren(); + EntityHierarchy parentTerm = termMap.get(parentFQN); + List children = parentTerm.getChildren(); children.removeIf( child -> child.getFullyQualifiedName().equals(term.getFullyQualifiedName())); children.add(term); parentTerm.setChildren(children); } else { if (rootTerms.containsKey(termFQN)) { - EntityHierarchy__1 rootTerm = rootTerms.get(termFQN); + EntityHierarchy rootTerm = rootTerms.get(termFQN); rootTerm.setChildren(term.getChildren()); } } @@ -643,6 +684,38 @@ public List buildGlossaryTermSearchHierarchy(SearchResponse return new ArrayList<>(rootTerms.values()); } + public List buildDomainSearchHierarchy(SearchResponse searchResponse) { + Map entityHierarchyMap = + Arrays.stream(searchResponse.getHits().getHits()) + .map(hit -> JsonUtils.readValue(hit.getSourceAsString(), EntityHierarchy.class)) + .collect( + Collectors.toMap( + EntityHierarchy::getFullyQualifiedName, + entity -> { + entity.setChildren(new ArrayList<>()); + return entity; + }, + (existing, replacement) -> existing, + LinkedHashMap::new)); + + List rootDomains = new ArrayList<>(); + + entityHierarchyMap + .values() + .forEach( + entity -> { + String parentFqn = getParentFQN(entity.getFullyQualifiedName()); + EntityHierarchy parentEntity = entityHierarchyMap.get(parentFqn); + if (parentEntity != null) { + parentEntity.getChildren().add(entity); + } else { + rootDomains.add(entity); + } + }); + + return rootDomains; + } + @Override public SearchResultListMapper listWithOffset( String filter, @@ -1050,7 +1123,7 @@ private void getLineage( searchSourceBuilder.query( QueryBuilders.boolQuery() .must(QueryBuilders.termQuery(direction, FullyQualifiedName.buildHash(fqn)))); - if (CommonUtil.nullOrEmpty(deleted)) { + if (!CommonUtil.nullOrEmpty(deleted)) { searchSourceBuilder.query( QueryBuilders.boolQuery() .must(QueryBuilders.termQuery(direction, FullyQualifiedName.buildHash(fqn))) @@ -1231,59 +1304,72 @@ private Map searchPipelineLineage( throws IOException { Set> edges = new HashSet<>(); Set> nodes = new HashSet<>(); - es.org.elasticsearch.action.search.SearchRequest searchRequest = - new es.org.elasticsearch.action.search.SearchRequest( - Entity.getSearchRepository().getIndexOrAliasName(GLOBAL_SEARCH_ALIAS)); - es.org.elasticsearch.index.query.BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); - boolQueryBuilder.should( - QueryBuilders.boolQuery() - .must(QueryBuilders.termQuery("lineage.pipeline.fullyQualifiedName.keyword", fqn))); - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchSourceBuilder.fetchSource(null, SOURCE_FIELDS_TO_EXCLUDE.toArray(String[]::new)); - searchSourceBuilder.query(boolQueryBuilder); - if (CommonUtil.nullOrEmpty(deleted)) { - searchSourceBuilder.query( + Object[] searchAfter = null; + long processedRecords = 0; + long totalRecords = -1; + // Process pipeline as edge + while (totalRecords != processedRecords) { + es.org.elasticsearch.action.search.SearchRequest searchRequest = + new es.org.elasticsearch.action.search.SearchRequest( + Entity.getSearchRepository().getIndexOrAliasName(GLOBAL_SEARCH_ALIAS)); + es.org.elasticsearch.index.query.BoolQueryBuilder boolQueryBuilder = + QueryBuilders.boolQuery(); + boolQueryBuilder.should( QueryBuilders.boolQuery() - .must(boolQueryBuilder) - .must(QueryBuilders.termQuery("deleted", deleted))); - } - buildSearchSourceFilter(queryFilter, searchSourceBuilder); - searchRequest.source(searchSourceBuilder); - SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); - for (var hit : searchResponse.getHits().getHits()) { - List> lineage = - (List>) hit.getSourceAsMap().get("lineage"); - HashMap tempMap = new HashMap<>(JsonUtils.getMap(hit.getSourceAsMap())); - nodes.add(tempMap); - for (Map lin : lineage) { - HashMap fromEntity = (HashMap) lin.get("fromEntity"); - HashMap toEntity = (HashMap) lin.get("toEntity"); - HashMap pipeline = (HashMap) lin.get("pipeline"); - if (pipeline != null && pipeline.get("fullyQualifiedName").equalsIgnoreCase(fqn)) { - edges.add(lin); - getLineage( - fromEntity.get("fqn"), - upstreamDepth, - edges, - nodes, - queryFilter, - "lineage.toEntity.fqn.keyword", - deleted); - getLineage( - toEntity.get("fqn"), - downstreamDepth, - edges, - nodes, - queryFilter, - "lineage.fromEntity.fqn.keyword", - deleted); + .must(QueryBuilders.termQuery("lineage.pipeline.fullyQualifiedName.keyword", fqn))); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.fetchSource(null, SOURCE_FIELDS_TO_EXCLUDE.toArray(String[]::new)); + searchSourceBuilder.size(1000); + FieldSortBuilder sortBuilder = SortBuilders.fieldSort("fullyQualifiedName"); + searchSourceBuilder.sort(sortBuilder); + searchSourceBuilder.query(boolQueryBuilder); + if (searchAfter != null) { + searchSourceBuilder.searchAfter(searchAfter); + } + if (!CommonUtil.nullOrEmpty(deleted)) { + searchSourceBuilder.query( + QueryBuilders.boolQuery() + .must(boolQueryBuilder) + .must(QueryBuilders.termQuery("deleted", deleted))); + } + buildSearchSourceFilter(queryFilter, searchSourceBuilder); + searchRequest.source(searchSourceBuilder); + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + + for (var hit : searchResponse.getHits().getHits()) { + List> lineage = + (List>) hit.getSourceAsMap().get("lineage"); + HashMap tempMap = new HashMap<>(JsonUtils.getMap(hit.getSourceAsMap())); + nodes.add(tempMap); + for (Map lin : lineage) { + HashMap pipeline = (HashMap) lin.get("pipeline"); + if (pipeline != null && pipeline.get("fullyQualifiedName").equalsIgnoreCase(fqn)) { + edges.add(lin); + } } } + totalRecords = searchResponse.getHits().getTotalHits().value; + int currentHits = searchResponse.getHits().getHits().length; + processedRecords += currentHits; + if (currentHits > 0) { + searchAfter = searchResponse.getHits().getHits()[currentHits - 1].getSortValues(); + } else { + // when current records are 0 break the loop + break; + } } + + // Process pipeline as node getLineage( - fqn, downstreamDepth, edges, nodes, queryFilter, "lineage.fromEntity.fqn.keyword", deleted); + fqn, + downstreamDepth, + edges, + nodes, + queryFilter, + "lineage.fromEntity.fqnHash.keyword", + deleted); getLineage( - fqn, upstreamDepth, edges, nodes, queryFilter, "lineage.toEntity.fqn.keyword", deleted); + fqn, upstreamDepth, edges, nodes, queryFilter, "lineage.toEntity.fqnHash.keyword", deleted); // TODO: Fix this , this is hack if (edges.isEmpty()) { @@ -1797,7 +1883,10 @@ private static SearchSourceBuilder buildDomainsSearch(String query, int from, in buildSearchQueryBuilder(query, DomainIndex.getFields()); FunctionScoreQueryBuilder queryBuilder = boostScore(queryStringBuilder); HighlightBuilder hb = buildHighlights(new ArrayList<>()); - return searchBuilder(queryBuilder, hb, from, size); + SearchSourceBuilder searchSourceBuilder = searchBuilder(queryBuilder, hb, from, size); + searchSourceBuilder.aggregation( + AggregationBuilders.terms("fqnParts_agg").field("fqnParts").size(1000)); + return addAggregation(searchSourceBuilder); } private static SearchSourceBuilder buildCostAnalysisReportDataSearch( @@ -2212,7 +2301,7 @@ public void updateElasticSearch(UpdateRequest updateRequest) { private void updateElasticSearchByQuery(UpdateByQueryRequest updateByQueryRequest) { if (updateByQueryRequest != null && isClientAvailable) { updateByQueryRequest.setRefresh(true); - LOG.debug(SENDING_REQUEST_TO_ELASTIC_SEARCH, updateByQueryRequest); + LOG.info(SENDING_REQUEST_TO_ELASTIC_SEARCH, updateByQueryRequest); client.updateByQuery(updateByQueryRequest, RequestOptions.DEFAULT); } } @@ -2750,6 +2839,18 @@ public Object getLowLevelClient() { return client.getLowLevelClient(); } + @Override + public SearchHealthStatus getSearchHealthStatus() throws IOException { + ClusterHealthRequest request = new ClusterHealthRequest(); + ClusterHealthResponse response = client.cluster().health(request, RequestOptions.DEFAULT); + if (response.getStatus().equals(ClusterHealthStatus.GREEN) + || response.getStatus().equals(ClusterHealthStatus.YELLOW)) { + return new SearchHealthStatus(HEALTHY_STATUS); + } else { + return new SearchHealthStatus(UNHEALTHY_STATUS); + } + } + private void buildSearchRBACQuery( SubjectContext subjectContext, SearchSourceBuilder searchSourceBuilder) { if (SearchClient.shouldApplyRbacConditions(subjectContext, rbacConditionEvaluator)) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/APIEndpointIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/APIEndpointIndex.java index 9fbfa1555395..b55e07b897a8 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/APIEndpointIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/APIEndpointIndex.java @@ -75,7 +75,7 @@ public Map buildSearchIndexDocInternal(Map doc) && apiEndpoint.getRequestSchema().getSchemaFields() != null && !apiEndpoint.getRequestSchema().getSchemaFields().isEmpty()) { List flattenFields = new ArrayList<>(); - parseSchemaFields(apiEndpoint.getResponseSchema().getSchemaFields(), flattenFields, null); + parseSchemaFields(apiEndpoint.getRequestSchema().getSchemaFields(), flattenFields, null); for (FlattenSchemaField field : flattenFields) { fieldSuggest.add(SearchSuggest.builder().input(field.getName()).weight(5).build()); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ColumnIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ColumnIndex.java index c712f5c74922..fd40d921ce03 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ColumnIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ColumnIndex.java @@ -50,6 +50,6 @@ default String getColumnDescriptionStatus(EntityInterface entity) { } } } - return CommonUtil.nullOrEmpty(entity.getDescription()) ? "INCOMPLETE" : "COMPLETE"; + return "COMPLETE"; } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java index e75cc7262b61..902366cea84e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java @@ -1,7 +1,8 @@ package org.openmetadata.service.search.indexes; +import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; -import static org.openmetadata.schema.type.Include.NON_DELETED; +import static org.openmetadata.schema.type.Include.ALL; import static org.openmetadata.service.Entity.FIELD_DESCRIPTION; import static org.openmetadata.service.Entity.FIELD_DISPLAY_NAME; import static org.openmetadata.service.Entity.getEntityByName; @@ -182,59 +183,57 @@ private static void processConstraints( Table relatedEntity, List> constraints, Boolean updateForeignTableIndex) { - if (!nullOrEmpty(entity.getTableConstraints())) { - for (TableConstraint tableConstraint : entity.getTableConstraints()) { - if (!tableConstraint - .getConstraintType() - .value() - .equalsIgnoreCase(TableConstraint.ConstraintType.FOREIGN_KEY.value())) { - continue; - } - int columnIndex = 0; - for (String referredColumn : tableConstraint.getReferredColumns()) { - String relatedEntityFQN = getParentFQN(referredColumn); - String destinationIndexName = null; - try { - if (updateForeignTableIndex) { - relatedEntity = getEntityByName(Entity.TABLE, relatedEntityFQN, "*", NON_DELETED); - IndexMapping destinationIndexMapping = - Entity.getSearchRepository() - .getIndexMapping(relatedEntity.getEntityReference().getType()); - destinationIndexName = - destinationIndexMapping.getIndexName( - Entity.getSearchRepository().getClusterAlias()); - } - Map relationshipsMap = buildRelationshipsMap(entity, relatedEntity); - int relatedEntityIndex = - checkRelatedEntity( - entity.getFullyQualifiedName(), - relatedEntity.getFullyQualifiedName(), - constraints); - if (relatedEntityIndex >= 0) { - updateExistingConstraint( - entity, - tableConstraint, - constraints.get(relatedEntityIndex), - destinationIndexName, - relatedEntity, - referredColumn, - columnIndex, - updateForeignTableIndex); - } else { - addNewConstraint( - entity, - tableConstraint, - constraints, - relationshipsMap, - destinationIndexName, - relatedEntity, - referredColumn, - columnIndex, - updateForeignTableIndex); - } - columnIndex++; - } catch (EntityNotFoundException ex) { + for (TableConstraint tableConstraint : listOrEmpty(entity.getTableConstraints())) { + if (!tableConstraint + .getConstraintType() + .value() + .equalsIgnoreCase(TableConstraint.ConstraintType.FOREIGN_KEY.value())) { + continue; + } + int columnIndex = 0; + for (String referredColumn : listOrEmpty(tableConstraint.getReferredColumns())) { + String relatedEntityFQN = getParentFQN(referredColumn); + String destinationIndexName = null; + try { + if (updateForeignTableIndex) { + relatedEntity = getEntityByName(Entity.TABLE, relatedEntityFQN, "*", ALL); + IndexMapping destinationIndexMapping = + Entity.getSearchRepository() + .getIndexMapping(relatedEntity.getEntityReference().getType()); + destinationIndexName = + destinationIndexMapping.getIndexName( + Entity.getSearchRepository().getClusterAlias()); + } + Map relationshipsMap = buildRelationshipsMap(entity, relatedEntity); + int relatedEntityIndex = + checkRelatedEntity( + entity.getFullyQualifiedName(), + relatedEntity.getFullyQualifiedName(), + constraints); + if (relatedEntityIndex >= 0) { + updateExistingConstraint( + entity, + tableConstraint, + constraints.get(relatedEntityIndex), + destinationIndexName, + relatedEntity, + referredColumn, + columnIndex, + updateForeignTableIndex); + } else { + addNewConstraint( + entity, + tableConstraint, + constraints, + relationshipsMap, + destinationIndexName, + relatedEntity, + referredColumn, + columnIndex, + updateForeignTableIndex); } + columnIndex++; + } catch (EntityNotFoundException ex) { } } } @@ -253,8 +252,7 @@ static List> populateEntityRelationshipData(Table entity) { .findFrom(entity.getId(), Entity.TABLE, Relationship.RELATED_TO.ordinal()); for (CollectionDAO.EntityRelationshipRecord table : relatedTables) { - Table foreignTable = - Entity.getEntity(Entity.TABLE, table.getId(), "tableConstraints", NON_DELETED); + Table foreignTable = Entity.getEntity(Entity.TABLE, table.getId(), "tableConstraints", ALL); processConstraints(foreignTable, entity, constraints, false); } return constraints; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseIndex.java index 92e82f5f4269..b08894a05741 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseIndex.java @@ -69,8 +69,10 @@ private void setParentRelationships(Map doc, TestCase testCase) return; } TestSuite testSuite = Entity.getEntityOrNull(testSuiteEntityReference, "", Include.ALL); - EntityReference entityReference = testSuite.getExecutableEntityReference(); - TestSuiteIndex.addTestSuiteParentEntityRelations(entityReference, doc); + EntityReference entityReference = testSuite.getBasicEntityReference(); + if (entityReference != null) { + TestSuiteIndex.addTestSuiteParentEntityRelations(entityReference, doc); + } } public static Map getFields() { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseResolutionStatusIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseResolutionStatusIndex.java index 4f9a512bd0c3..56de9830049e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseResolutionStatusIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseResolutionStatusIndex.java @@ -65,7 +65,9 @@ private void setParentRelationships(Map doc) { TestSuite testSuite = Entity.getEntityOrNull(testCase.getTestSuite(), "", Include.ALL); if (testSuite == null) return; doc.put("testSuite", testSuite.getEntityReference()); - TestSuiteIndex.addTestSuiteParentEntityRelations(testSuite.getExecutableEntityReference(), doc); + if (testSuite.getBasicEntityReference() != null) { + TestSuiteIndex.addTestSuiteParentEntityRelations(testSuite.getBasicEntityReference(), doc); + } } public static Map getFields() { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestSuiteIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestSuiteIndex.java index 62c52fc3c2b3..cffe87bef2d6 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestSuiteIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestSuiteIndex.java @@ -39,7 +39,7 @@ public Map buildSearchIndexDocInternal(Map doc) private void setParentRelationships(Map doc, TestSuite testSuite) { // denormalize the parent relationships for search - EntityReference entityReference = testSuite.getExecutableEntityReference(); + EntityReference entityReference = testSuite.getBasicEntityReference(); if (entityReference == null) return; addTestSuiteParentEntityRelations(entityReference, doc); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java index 2ba538f37cdc..2b9cdc5a62ba 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java @@ -12,6 +12,8 @@ import static org.openmetadata.service.Entity.QUERY; import static org.openmetadata.service.Entity.RAW_COST_ANALYSIS_REPORT_DATA; import static org.openmetadata.service.Entity.TABLE; +import static org.openmetadata.service.events.scheduled.ServicesStatusJobHandler.HEALTHY_STATUS; +import static org.openmetadata.service.events.scheduled.ServicesStatusJobHandler.UNHEALTHY_STATUS; import static org.openmetadata.service.exception.CatalogGenericExceptionMapper.getResponse; import static org.openmetadata.service.search.EntityBuilderConstant.API_RESPONSE_SCHEMA_FIELD; import static org.openmetadata.service.search.EntityBuilderConstant.API_RESPONSE_SCHEMA_FIELD_KEYWORD; @@ -47,6 +49,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import java.util.stream.Stream; import javax.json.JsonObject; import javax.net.ssl.SSLContext; @@ -68,7 +71,7 @@ import org.openmetadata.schema.dataInsight.custom.DataInsightCustomChart; import org.openmetadata.schema.dataInsight.custom.DataInsightCustomChartResultList; import org.openmetadata.schema.dataInsight.custom.FormulaHolder; -import org.openmetadata.schema.entity.data.EntityHierarchy__1; +import org.openmetadata.schema.entity.data.EntityHierarchy; import org.openmetadata.schema.entity.data.Table; import org.openmetadata.schema.service.configuration.elasticsearch.ElasticSearchConfiguration; import org.openmetadata.schema.tests.DataQualityReport; @@ -85,6 +88,7 @@ import org.openmetadata.service.jdbi3.TestCaseResultRepository; import org.openmetadata.service.search.SearchAggregation; import org.openmetadata.service.search.SearchClient; +import org.openmetadata.service.search.SearchHealthStatus; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.SearchRequest; import org.openmetadata.service.search.SearchSortFilter; @@ -134,6 +138,8 @@ import org.openmetadata.service.workflows.searchIndex.ReindexingUtil; import os.org.opensearch.OpenSearchException; import os.org.opensearch.OpenSearchStatusException; +import os.org.opensearch.action.admin.cluster.health.ClusterHealthRequest; +import os.org.opensearch.action.admin.cluster.health.ClusterHealthResponse; import os.org.opensearch.action.admin.indices.alias.IndicesAliasesRequest; import os.org.opensearch.action.admin.indices.delete.DeleteIndexRequest; import os.org.opensearch.action.bulk.BulkRequest; @@ -155,6 +161,7 @@ import os.org.opensearch.client.indices.GetMappingsRequest; import os.org.opensearch.client.indices.GetMappingsResponse; import os.org.opensearch.client.indices.PutMappingRequest; +import os.org.opensearch.cluster.health.ClusterHealthStatus; import os.org.opensearch.cluster.metadata.MappingMetadata; import os.org.opensearch.common.lucene.search.function.CombineFunction; import os.org.opensearch.common.lucene.search.function.FieldValueFactorFunction; @@ -462,53 +469,10 @@ public Response search(SearchRequest request, SubjectContext subjectContext) thr .getIndexMapping(GLOSSARY_TERM) .getIndexName(clusterAlias))) { searchSourceBuilder.query(QueryBuilders.boolQuery().must(searchSourceBuilder.query())); - - if (request.isGetHierarchy()) { - /* - Search for user input terms in name, fullyQualifiedName, displayName and glossary.fullyQualifiedName, glossary.displayName - */ - QueryBuilder baseQuery = - QueryBuilders.boolQuery() - .should(searchSourceBuilder.query()) - .should(QueryBuilders.matchPhraseQuery("fullyQualifiedName", request.getQuery())) - .should(QueryBuilders.matchPhraseQuery("name", request.getQuery())) - .should(QueryBuilders.matchPhraseQuery("displayName", request.getQuery())) - .should( - QueryBuilders.matchPhraseQuery( - "glossary.fullyQualifiedName", request.getQuery())) - .should(QueryBuilders.matchPhraseQuery("glossary.displayName", request.getQuery())) - .must(QueryBuilders.matchQuery("status", "Approved")) - .minimumShouldMatch(1); - searchSourceBuilder.query(baseQuery); - - SearchResponse searchResponse = - client.search( - new os.org.opensearch.action.search.SearchRequest(request.getIndex()) - .source(searchSourceBuilder), - RequestOptions.DEFAULT); - - // Extract parent terms from aggregation - BoolQueryBuilder parentTermQueryBuilder = QueryBuilders.boolQuery(); - Terms parentTerms = searchResponse.getAggregations().get("fqnParts_agg"); - - // Build es query to get parent terms for the user input query , to build correct hierarchy - if (!parentTerms.getBuckets().isEmpty() && !request.getQuery().equals("*")) { - parentTerms.getBuckets().stream() - .map(Terms.Bucket::getKeyAsString) - .forEach( - parentTerm -> - parentTermQueryBuilder.should( - QueryBuilders.matchQuery("fullyQualifiedName", parentTerm))); - - searchSourceBuilder.query( - parentTermQueryBuilder - .minimumShouldMatch(1) - .must(QueryBuilders.matchQuery("status", "Approved"))); - } - searchSourceBuilder.sort(SortBuilders.fieldSort("fullyQualifiedName").order(SortOrder.ASC)); - } } + buildHierarchyQuery(request, searchSourceBuilder, client); + /* for performance reasons OpenSearch doesn't provide accurate hits if we enable trackTotalHits parameter it will try to match every result, count and return hits however in most cases for search results an approximate value is good enough. @@ -571,34 +535,107 @@ public Response getDocByID(String indexName, String entityId) throws IOException return getResponse(NOT_FOUND, "Document not found."); } + private void buildHierarchyQuery( + SearchRequest request, SearchSourceBuilder searchSourceBuilder, RestHighLevelClient client) + throws IOException { + + if (!request.isGetHierarchy()) { + return; + } + + String indexName = request.getIndex(); + String glossaryTermIndex = + Entity.getSearchRepository().getIndexMapping(GLOSSARY_TERM).getIndexName(clusterAlias); + String domainIndex = + Entity.getSearchRepository().getIndexMapping(DOMAIN).getIndexName(clusterAlias); + + BoolQueryBuilder baseQuery = + QueryBuilders.boolQuery() + .should(searchSourceBuilder.query()) + .should(QueryBuilders.matchPhraseQuery("fullyQualifiedName", request.getQuery())) + .should(QueryBuilders.matchPhraseQuery("name", request.getQuery())) + .should(QueryBuilders.matchPhraseQuery("displayName", request.getQuery())); + + if (indexName.equalsIgnoreCase(glossaryTermIndex)) { + baseQuery + .should(QueryBuilders.matchPhraseQuery("glossary.fullyQualifiedName", request.getQuery())) + .should(QueryBuilders.matchPhraseQuery("glossary.displayName", request.getQuery())) + .must(QueryBuilders.matchQuery("status", "Approved")); + } else if (indexName.equalsIgnoreCase(domainIndex)) { + baseQuery + .should(QueryBuilders.matchPhraseQuery("parent.fullyQualifiedName", request.getQuery())) + .should(QueryBuilders.matchPhraseQuery("parent.displayName", request.getQuery())); + } + + baseQuery.minimumShouldMatch(1); + searchSourceBuilder.query(baseQuery); + + SearchResponse searchResponse = + client.search( + new os.org.opensearch.action.search.SearchRequest(request.getIndex()) + .source(searchSourceBuilder), + RequestOptions.DEFAULT); + + Terms parentTerms = searchResponse.getAggregations().get("fqnParts_agg"); + + // Build es query to get parent terms for the user input query , to build correct hierarchy + // In case of default search , no need to get parent terms they are already present in the + // response + if (parentTerms != null + && !parentTerms.getBuckets().isEmpty() + && !request.getQuery().equals("*")) { + BoolQueryBuilder parentTermQueryBuilder = QueryBuilders.boolQuery(); + + parentTerms.getBuckets().stream() + .map(Terms.Bucket::getKeyAsString) + .forEach( + parentTerm -> + parentTermQueryBuilder.should( + QueryBuilders.matchQuery("fullyQualifiedName", parentTerm))); + if (indexName.equalsIgnoreCase(glossaryTermIndex)) { + parentTermQueryBuilder + .minimumShouldMatch(1) + .must(QueryBuilders.matchQuery("status", "Approved")); + } else { + parentTermQueryBuilder.minimumShouldMatch(1); + } + searchSourceBuilder.query(parentTermQueryBuilder); + } + + searchSourceBuilder.sort(SortBuilders.fieldSort("fullyQualifiedName").order(SortOrder.ASC)); + } + public List buildSearchHierarchy(SearchRequest request, SearchResponse searchResponse) { List response = new ArrayList<>(); - if (request - .getIndex() - .equalsIgnoreCase( - Entity.getSearchRepository() - .getIndexMapping(GLOSSARY_TERM) - .getIndexName(clusterAlias))) { + String indexName = request.getIndex(); + String glossaryTermIndex = + Entity.getSearchRepository().getIndexMapping(GLOSSARY_TERM).getIndexName(clusterAlias); + String domainIndex = + Entity.getSearchRepository().getIndexMapping(DOMAIN).getIndexName(clusterAlias); + + if (indexName.equalsIgnoreCase(glossaryTermIndex)) { response = buildGlossaryTermSearchHierarchy(searchResponse); + } else if (indexName.equalsIgnoreCase(domainIndex)) { + response = buildDomainSearchHierarchy(searchResponse); } return response; } - public List buildGlossaryTermSearchHierarchy(SearchResponse searchResponse) { - Map termMap = + public List buildGlossaryTermSearchHierarchy(SearchResponse searchResponse) { + Map termMap = new LinkedHashMap<>(); // termMap represent glossary terms - Map rootTerms = + Map rootTerms = new LinkedHashMap<>(); // rootTerms represent glossaries for (var hit : searchResponse.getHits().getHits()) { String jsonSource = hit.getSourceAsString(); - EntityHierarchy__1 term = JsonUtils.readValue(jsonSource, EntityHierarchy__1.class); - EntityHierarchy__1 glossaryInfo = + EntityHierarchy term = JsonUtils.readValue(jsonSource, EntityHierarchy.class); + EntityHierarchy glossaryInfo = JsonUtils.readTree(jsonSource).path("glossary").isMissingNode() ? null : JsonUtils.convertValue( - JsonUtils.readTree(jsonSource).path("glossary"), EntityHierarchy__1.class); + JsonUtils.readTree(jsonSource).path("glossary"), EntityHierarchy.class); if (glossaryInfo != null) { rootTerms.putIfAbsent(glossaryInfo.getFullyQualifiedName(), glossaryInfo); @@ -618,15 +655,15 @@ public List buildGlossaryTermSearchHierarchy(SearchResponse String termFQN = term.getFullyQualifiedName(); if (parentFQN != null && termMap.containsKey(parentFQN)) { - EntityHierarchy__1 parentTerm = termMap.get(parentFQN); - List children = parentTerm.getChildren(); + EntityHierarchy parentTerm = termMap.get(parentFQN); + List children = parentTerm.getChildren(); children.removeIf( child -> child.getFullyQualifiedName().equals(term.getFullyQualifiedName())); children.add(term); parentTerm.setChildren(children); } else { if (rootTerms.containsKey(termFQN)) { - EntityHierarchy__1 rootTerm = rootTerms.get(termFQN); + EntityHierarchy rootTerm = rootTerms.get(termFQN); rootTerm.setChildren(term.getChildren()); } } @@ -635,6 +672,38 @@ public List buildGlossaryTermSearchHierarchy(SearchResponse return new ArrayList<>(rootTerms.values()); } + public List buildDomainSearchHierarchy(SearchResponse searchResponse) { + Map entityHierarchyMap = + Arrays.stream(searchResponse.getHits().getHits()) + .map(hit -> JsonUtils.readValue(hit.getSourceAsString(), EntityHierarchy.class)) + .collect( + Collectors.toMap( + EntityHierarchy::getFullyQualifiedName, + entity -> { + entity.setChildren(new ArrayList<>()); + return entity; + }, + (existing, replacement) -> existing, + LinkedHashMap::new)); + + List rootDomains = new ArrayList<>(); + + entityHierarchyMap + .values() + .forEach( + entity -> { + String parentFqn = getParentFQN(entity.getFullyQualifiedName()); + EntityHierarchy parentEntity = entityHierarchyMap.get(parentFqn); + if (parentEntity != null) { + parentEntity.getChildren().add(entity); + } else { + rootDomains.add(entity); + } + }); + + return rootDomains; + } + @Override public SearchResultListMapper listWithOffset( String filter, @@ -1050,7 +1119,7 @@ private void getLineage( searchSourceBuilder.query( QueryBuilders.boolQuery() .must(QueryBuilders.termQuery(direction, FullyQualifiedName.buildHash(fqn)))); - if (CommonUtil.nullOrEmpty(deleted)) { + if (!CommonUtil.nullOrEmpty(deleted)) { searchSourceBuilder.query( QueryBuilders.boolQuery() .must(QueryBuilders.termQuery(direction, FullyQualifiedName.buildHash(fqn))) @@ -1230,60 +1299,71 @@ private Map searchPipelineLineage( Set> edges = new HashSet<>(); Set> nodes = new HashSet<>(); responseMap.put("entity", null); - os.org.opensearch.action.search.SearchRequest searchRequest = - new os.org.opensearch.action.search.SearchRequest( - Entity.getSearchRepository().getIndexOrAliasName(GLOBAL_SEARCH_ALIAS)); - BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); - boolQueryBuilder.should( - QueryBuilders.boolQuery() - .must(QueryBuilders.termQuery("lineage.pipeline.fullyQualifiedName.keyword", fqn))); - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchSourceBuilder.fetchSource(null, SOURCE_FIELDS_TO_EXCLUDE.toArray(String[]::new)); - searchSourceBuilder.query(boolQueryBuilder); - if (CommonUtil.nullOrEmpty(deleted)) { - searchSourceBuilder.query( + Object[] searchAfter = null; + long processedRecords = 0; + long totalRecords = -1; + // Process pipeline as edge + while (totalRecords != processedRecords) { + os.org.opensearch.action.search.SearchRequest searchRequest = + new os.org.opensearch.action.search.SearchRequest( + Entity.getSearchRepository().getIndexOrAliasName(GLOBAL_SEARCH_ALIAS)); + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + boolQueryBuilder.should( QueryBuilders.boolQuery() - .must(boolQueryBuilder) - .must(QueryBuilders.termQuery("deleted", deleted))); - } - buildSearchSourceFilter(queryFilter, searchSourceBuilder); + .must(QueryBuilders.termQuery("lineage.pipeline.fullyQualifiedName.keyword", fqn))); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.fetchSource(null, SOURCE_FIELDS_TO_EXCLUDE.toArray(String[]::new)); + FieldSortBuilder sortBuilder = SortBuilders.fieldSort("fullyQualifiedName"); + searchSourceBuilder.sort(sortBuilder); + searchSourceBuilder.size(1000); + searchSourceBuilder.query(boolQueryBuilder); + if (searchAfter != null) { + searchSourceBuilder.searchAfter(searchAfter); + } + if (!CommonUtil.nullOrEmpty(deleted)) { + searchSourceBuilder.query( + QueryBuilders.boolQuery() + .must(boolQueryBuilder) + .must(QueryBuilders.termQuery("deleted", deleted))); + } + buildSearchSourceFilter(queryFilter, searchSourceBuilder); - searchRequest.source(searchSourceBuilder); - SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); - for (var hit : searchResponse.getHits().getHits()) { - List> lineage = - (List>) hit.getSourceAsMap().get("lineage"); - HashMap tempMap = new HashMap<>(JsonUtils.getMap(hit.getSourceAsMap())); - nodes.add(tempMap); - for (Map lin : lineage) { - HashMap fromEntity = (HashMap) lin.get("fromEntity"); - HashMap toEntity = (HashMap) lin.get("toEntity"); - HashMap pipeline = (HashMap) lin.get("pipeline"); - if (pipeline != null && pipeline.get("fullyQualifiedName").equalsIgnoreCase(fqn)) { - edges.add(lin); - getLineage( - fromEntity.get("fqn"), - upstreamDepth, - edges, - nodes, - queryFilter, - "lineage.toEntity.fqn.keyword", - deleted); - getLineage( - toEntity.get("fqn"), - downstreamDepth, - edges, - nodes, - queryFilter, - "lineage.fromEntity.fqn.keyword", - deleted); + searchRequest.source(searchSourceBuilder); + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + for (var hit : searchResponse.getHits().getHits()) { + List> lineage = + (List>) hit.getSourceAsMap().get("lineage"); + HashMap tempMap = new HashMap<>(JsonUtils.getMap(hit.getSourceAsMap())); + nodes.add(tempMap); + for (Map lin : lineage) { + HashMap pipeline = (HashMap) lin.get("pipeline"); + if (pipeline != null && pipeline.get("fullyQualifiedName").equalsIgnoreCase(fqn)) { + edges.add(lin); + } } } + totalRecords = searchResponse.getHits().getTotalHits().value; + int currentHits = searchResponse.getHits().getHits().length; + processedRecords += currentHits; + if (currentHits > 0) { + searchAfter = searchResponse.getHits().getHits()[currentHits - 1].getSortValues(); + } else { + // when current records are 0 break the loop + break; + } } + + // Process pipeline as node getLineage( - fqn, downstreamDepth, edges, nodes, queryFilter, "lineage.fromEntity.fqn.keyword", deleted); + fqn, + downstreamDepth, + edges, + nodes, + queryFilter, + "lineage.fromEntity.fqnHash.keyword", + deleted); getLineage( - fqn, upstreamDepth, edges, nodes, queryFilter, "lineage.toEntity.fqn.keyword", deleted); + fqn, upstreamDepth, edges, nodes, queryFilter, "lineage.toEntity.fqnHash.keyword", deleted); if (edges.isEmpty()) { os.org.opensearch.action.search.SearchRequest searchRequestForEntity = @@ -1780,7 +1860,10 @@ private static SearchSourceBuilder buildDomainsSearch(String query, int from, in buildSearchQueryBuilder(query, DomainIndex.getFields()); FunctionScoreQueryBuilder queryBuilder = boostScore(queryStringBuilder); HighlightBuilder hb = buildHighlights(new ArrayList<>()); - return searchBuilder(queryBuilder, hb, from, size); + SearchSourceBuilder searchSourceBuilder = searchBuilder(queryBuilder, hb, from, size); + searchSourceBuilder.aggregation( + AggregationBuilders.terms("fqnParts_agg").field("fqnParts").size(1000)); + return addAggregation(searchSourceBuilder); } private static SearchSourceBuilder buildSearchEntitySearch(String query, int from, int size) { @@ -2754,4 +2837,16 @@ private static void buildSearchSourceFilter( } } } + + @Override + public SearchHealthStatus getSearchHealthStatus() throws IOException { + ClusterHealthRequest request = new ClusterHealthRequest(); + ClusterHealthResponse response = client.cluster().health(request, RequestOptions.DEFAULT); + if (response.getStatus().equals(ClusterHealthStatus.GREEN) + || response.getStatus().equals(ClusterHealthStatus.YELLOW)) { + return new SearchHealthStatus(HEALTHY_STATUS); + } else { + return new SearchHealthStatus(UNHEALTHY_STATUS); + } + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/security/RBACConditionEvaluator.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/security/RBACConditionEvaluator.java index f2f66a008e9f..8f8cc8a18165 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/security/RBACConditionEvaluator.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/security/RBACConditionEvaluator.java @@ -200,6 +200,10 @@ private void handleMethodReference(SpelNode node, ConditionCollector collector) String methodName = methodRef.getName(); switch (methodName) { + case "matchAnyCertification" -> { + List certificationLabels = extractMethodArguments(methodRef); + matchAnyCertification(certificationLabels, collector); + } case "matchAnyTag" -> { List tags = extractMethodArguments(methodRef); matchAnyTag(tags, collector); @@ -259,6 +263,22 @@ public void matchAllTags(List tags, ConditionCollector collector) { } } + public void matchAnyCertification( + List certificationLabels, ConditionCollector collector) { + List certificationQueries = new ArrayList<>(); + for (String certificationLabel : certificationLabels) { + certificationQueries.add( + queryBuilderFactory.termQuery("certification.tagLabel.tagFQN", certificationLabel)); + } + OMQueryBuilder certificationQueriesCombined; + if (certificationQueries.size() == 1) { + certificationQueriesCombined = certificationQueries.get(0); + } else { + certificationQueriesCombined = queryBuilderFactory.boolQuery().should(certificationQueries); + } + collector.addMust(certificationQueriesCombined); + } + public void isOwner(User user, ConditionCollector collector) { List ownerQueries = new ArrayList<>(); ownerQueries.add(queryBuilderFactory.termQuery("owners.id", user.getId().toString())); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ClassConverterFactory.java b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ClassConverterFactory.java index f292215cfa39..8cb55e26b89c 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ClassConverterFactory.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ClassConverterFactory.java @@ -28,7 +28,10 @@ import org.openmetadata.schema.services.connections.database.datalake.GCSConfig; import org.openmetadata.schema.services.connections.database.deltalake.StorageConfig; import org.openmetadata.schema.services.connections.database.iceberg.IcebergFileSystem; +import org.openmetadata.schema.services.connections.mlmodel.VertexAIConnection; import org.openmetadata.schema.services.connections.pipeline.AirflowConnection; +import org.openmetadata.schema.services.connections.pipeline.MatillionConnection; +import org.openmetadata.schema.services.connections.pipeline.NifiConnection; import org.openmetadata.schema.services.connections.search.ElasticSearchConnection; import org.openmetadata.schema.services.connections.storage.GCSConnection; @@ -72,7 +75,10 @@ private ClassConverterFactory() { TestServiceConnectionRequest.class, new TestServiceConnectionRequestClassConverter()), Map.entry(TrinoConnection.class, new TrinoConnectionClassConverter()), - Map.entry(Workflow.class, new WorkflowClassConverter())); + Map.entry(Workflow.class, new WorkflowClassConverter()), + Map.entry(NifiConnection.class, new NifiConnectionClassConverter()), + Map.entry(MatillionConnection.class, new MatillionConnectionClassConverter()), + Map.entry(VertexAIConnection.class, new VertexAIConnectionClassConverter())); } public static ClassConverter getConverter(Class clazz) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/MatillionConnectionClassConverter.java b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/MatillionConnectionClassConverter.java new file mode 100644 index 000000000000..90b4a572fc30 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/MatillionConnectionClassConverter.java @@ -0,0 +1,38 @@ +/* + * Copyright 2025 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.secrets.converter; + +import java.util.List; +import org.openmetadata.schema.services.connections.pipeline.MatillionConnection; +import org.openmetadata.schema.services.connections.pipeline.matillion.MatillionETLAuth; +import org.openmetadata.service.util.JsonUtils; + +/** Converter class to get an `MatillionConnection` object. */ +public class MatillionConnectionClassConverter extends ClassConverter { + + public MatillionConnectionClassConverter() { + super(MatillionConnection.class); + } + + @Override + public Object convert(Object object) { + MatillionConnection matillionConnection = + (MatillionConnection) JsonUtils.convertValue(object, this.clazz); + + tryToConvertOrFail(matillionConnection.getConnection(), List.of(MatillionETLAuth.class)) + .ifPresent(matillionConnection::setConnection); + + return matillionConnection; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/NifiConnectionClassConverter.java b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/NifiConnectionClassConverter.java new file mode 100644 index 000000000000..234456b39632 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/NifiConnectionClassConverter.java @@ -0,0 +1,39 @@ +/* + * Copyright 2025 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.secrets.converter; + +import java.util.List; +import org.openmetadata.schema.services.connections.pipeline.NifiConnection; +import org.openmetadata.schema.services.connections.pipeline.nifi.BasicAuth; +import org.openmetadata.schema.services.connections.pipeline.nifi.ClientCertificateAuth; +import org.openmetadata.service.util.JsonUtils; + +/** Converter class to get an `NifiConnection` object. */ +public class NifiConnectionClassConverter extends ClassConverter { + + public NifiConnectionClassConverter() { + super(NifiConnection.class); + } + + @Override + public Object convert(Object object) { + NifiConnection nifiConnection = (NifiConnection) JsonUtils.convertValue(object, this.clazz); + + tryToConvertOrFail( + nifiConnection.getNifiConfig(), List.of(BasicAuth.class, ClientCertificateAuth.class)) + .ifPresent(nifiConnection::setNifiConfig); + + return nifiConnection; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/VertexAIConnectionClassConverter.java b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/VertexAIConnectionClassConverter.java new file mode 100644 index 000000000000..ecdd571f9121 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/VertexAIConnectionClassConverter.java @@ -0,0 +1,37 @@ +/* + * Copyright 2025 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.secrets.converter; + +import java.util.List; +import org.openmetadata.schema.security.credentials.GCPCredentials; +import org.openmetadata.schema.services.connections.mlmodel.VertexAIConnection; +import org.openmetadata.service.util.JsonUtils; + +/** Converter class to get an `VertexAIConnection` object. */ +public class VertexAIConnectionClassConverter extends ClassConverter { + + public VertexAIConnectionClassConverter() { + super(VertexAIConnection.class); + } + + @Override + public Object convert(Object object) { + VertexAIConnection vertexAIConnection = + (VertexAIConnection) JsonUtils.convertValue(object, this.clazz); + + tryToConvertOrFail(vertexAIConnection.getCredentials(), List.of(GCPCredentials.class)) + .ifPresent(obj -> vertexAIConnection.setCredentials((GCPCredentials) obj)); + + return vertexAIConnection; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/security/AuthenticationCodeFlowHandler.java b/openmetadata-service/src/main/java/org/openmetadata/service/security/AuthenticationCodeFlowHandler.java index de95bcce7765..b1d38d09933d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/security/AuthenticationCodeFlowHandler.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/security/AuthenticationCodeFlowHandler.java @@ -642,6 +642,7 @@ private ClientAuthenticationMethod firstSupportedMethod( @SneakyThrows public static void getErrorMessage(HttpServletResponse resp, Exception e) { + resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); resp.setContentType("text/html; charset=UTF-8"); LOG.error("[Auth Callback Servlet] Failed in Auth Login : {}", e.getMessage()); resp.getOutputStream() diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/security/Authorizer.java b/openmetadata-service/src/main/java/org/openmetadata/service/security/Authorizer.java index 26dbe0babfde..f94193c0d817 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/security/Authorizer.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/security/Authorizer.java @@ -43,6 +43,8 @@ void authorize( void authorizeAdmin(SecurityContext securityContext); + void authorizeAdmin(String adminName); + void authorizeAdminOrBot(SecurityContext securityContext); boolean shouldMaskPasswords(SecurityContext securityContext); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/security/DefaultAuthorizer.java b/openmetadata-service/src/main/java/org/openmetadata/service/security/DefaultAuthorizer.java index 51b0e32ca80e..a8636b53ad44 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/security/DefaultAuthorizer.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/security/DefaultAuthorizer.java @@ -101,6 +101,15 @@ public void authorizeAdmin(SecurityContext securityContext) { throw new AuthorizationException(notAdmin(securityContext.getUserPrincipal().getName())); } + @Override + public void authorizeAdmin(String adminName) { + SubjectContext subjectContext = SubjectContext.getSubjectContext(adminName); + if (subjectContext.isAdmin()) { + return; + } + throw new AuthorizationException(notAdmin(adminName)); + } + @Override public void authorizeAdminOrBot(SecurityContext securityContext) { SubjectContext subjectContext = getSubjectContext(securityContext); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/security/JwtFilter.java b/openmetadata-service/src/main/java/org/openmetadata/service/security/JwtFilter.java index 0e57365b069e..60f1bb688daf 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/security/JwtFilter.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/security/JwtFilter.java @@ -22,6 +22,7 @@ import static org.openmetadata.service.security.SecurityUtil.validatePrincipalClaimsMapping; import static org.openmetadata.service.security.jwt.JWTTokenGenerator.ROLES_CLAIM; import static org.openmetadata.service.security.jwt.JWTTokenGenerator.TOKEN_TYPE; +import static org.openmetadata.service.security.jwt.JWTTokenGenerator.getAlgorithm; import com.auth0.jwk.Jwk; import com.auth0.jwk.JwkProvider; @@ -71,6 +72,7 @@ public class JwtFilter implements ContainerRequestFilter { private boolean enforcePrincipalDomain; private AuthProvider providerType; private boolean useRolesFromProvider = false; + private AuthenticationConfiguration.TokenValidationAlgorithm tokenValidationAlgorithm; private static final List DEFAULT_PUBLIC_KEY_URLS = Arrays.asList( @@ -123,6 +125,7 @@ public JwtFilter( this.principalDomain = authorizerConfiguration.getPrincipalDomain(); this.enforcePrincipalDomain = authorizerConfiguration.getEnforcePrincipalDomain(); this.useRolesFromProvider = authorizerConfiguration.getUseRolesFromProvider(); + this.tokenValidationAlgorithm = authenticationConfiguration.getTokenValidationAlgorithm(); } @VisibleForTesting @@ -224,7 +227,8 @@ public Map validateJwtAndGetClaims(String token) { // Validate JWT with public key Jwk jwk = jwkProvider.get(jwt.getKeyId()); - Algorithm algorithm = Algorithm.RSA256((RSAPublicKey) jwk.getPublicKey(), null); + Algorithm algorithm = + getAlgorithm(tokenValidationAlgorithm, (RSAPublicKey) jwk.getPublicKey(), null); try { algorithm.verify(jwt); } catch (RuntimeException runtimeException) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/security/NoopAuthorizer.java b/openmetadata-service/src/main/java/org/openmetadata/service/security/NoopAuthorizer.java index 7b9ed7ee6bd7..355bfe809905 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/security/NoopAuthorizer.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/security/NoopAuthorizer.java @@ -96,6 +96,11 @@ public void authorizeAdmin(SecurityContext securityContext) { /* Always authorize */ } + @Override + public void authorizeAdmin(String adminName) { + /* Always authorize */ + } + @Override public void authorizeAdminOrBot(SecurityContext securityContext) { /* Always authorize */ diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/security/auth/BasicAuthenticator.java b/openmetadata-service/src/main/java/org/openmetadata/service/security/auth/BasicAuthenticator.java index a99a7eeab87a..a46809417aa3 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/security/auth/BasicAuthenticator.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/security/auth/BasicAuthenticator.java @@ -102,7 +102,6 @@ public class BasicAuthenticator implements AuthenticatorHandler { private static final int HASHING_COST = 12; private UserRepository userRepository; private TokenRepository tokenRepository; - private LoginAttemptCache loginAttemptCache; private AuthorizerConfiguration authorizerConfiguration; private boolean isSelfSignUpAvailable; @@ -111,7 +110,6 @@ public void init(OpenMetadataApplicationConfig config) { this.userRepository = (UserRepository) Entity.getEntityRepository(Entity.USER); this.tokenRepository = Entity.getTokenRepository(); this.authorizerConfiguration = config.getAuthorizerConfiguration(); - this.loginAttemptCache = new LoginAttemptCache(); this.isSelfSignUpAvailable = config.getAuthenticationConfiguration().getEnableSelfSignup(); } @@ -186,7 +184,7 @@ public void sendEmailVerification(UriInfo uriInfo, User user) throws IOException String.format( "%s/users/registrationConfirmation?user=%s&token=%s", getSmtpSettings().getOpenMetadataUrl(), - user.getFullyQualifiedName(), + URLEncoder.encode(user.getFullyQualifiedName(), StandardCharsets.UTF_8), mailVerificationToken); try { EmailUtil.sendEmailVerification(emailVerificationLink, user); @@ -267,7 +265,7 @@ public void resetUserPasswordWithToken(UriInfo uriInfo, PasswordResetRequest req LOG.error("Error in sending Password Change Mail to User. Reason : " + ex.getMessage(), ex); throw new CustomExceptionMessage(424, FAILED_SEND_EMAIL, EMAIL_SENDING_ISSUE); } - loginAttemptCache.recordSuccessfulLogin(request.getUsername()); + LoginAttemptCache.getInstance().recordSuccessfulLogin(request.getUsername()); } @Override @@ -312,7 +310,7 @@ public void changeUserPwdWithOldPwd( storedUser.getAuthenticationMechanism().setConfig(storedBasicAuthMechanism); PutResponse response = userRepository.createOrUpdate(uriInfo, storedUser); // remove login/details from cache - loginAttemptCache.recordSuccessfulLogin(userName); + LoginAttemptCache.getInstance().recordSuccessfulLogin(userName); // in case admin updates , send email to user if (request.getRequestType() == USER && getSmtpSettings().getEnableSmtpServer()) { @@ -476,7 +474,7 @@ public JwtResponse loginUser(LoginRequest loginRequest) throws IOException, Temp @Override public void checkIfLoginBlocked(String email) { - if (loginAttemptCache.isLoginBlocked(email)) { + if (LoginAttemptCache.getInstance().isLoginBlocked(email)) { throw new AuthenticationException(MAX_FAILED_LOGIN_ATTEMPT); } } @@ -484,15 +482,15 @@ public void checkIfLoginBlocked(String email) { @Override public void recordFailedLoginAttempt(String email, String userName) throws TemplateException, IOException { - loginAttemptCache.recordFailedLogin(email); - int failedLoginAttempt = loginAttemptCache.getUserFailedLoginCount(email); + LoginAttemptCache.getInstance().recordFailedLogin(email); + int failedLoginAttempt = LoginAttemptCache.getInstance().getUserFailedLoginCount(email); if (failedLoginAttempt == SecurityUtil.getLoginConfiguration().getMaxLoginFailAttempts()) { sendAccountStatus( userName, email, "Multiple Failed Login Attempts.", String.format( - "Someone is trying to access your account. Login is Blocked for %s minutes. Please change your password.", + "Someone is trying to access your account. Login is Blocked for %s seconds. Please change your password.", SecurityUtil.getLoginConfiguration().getAccessBlockTime())); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/security/auth/LdapAuthenticator.java b/openmetadata-service/src/main/java/org/openmetadata/service/security/auth/LdapAuthenticator.java index 9531d76c1f97..d608434d7b2c 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/security/auth/LdapAuthenticator.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/security/auth/LdapAuthenticator.java @@ -84,7 +84,6 @@ public class LdapAuthenticator implements AuthenticatorHandler { private RoleRepository roleRepository; private UserRepository userRepository; private TokenRepository tokenRepository; - private LoginAttemptCache loginAttemptCache; private LdapConfiguration ldapConfiguration; private LDAPConnectionPool ldapLookupConnectionPool; private boolean isSelfSignUpEnabled; @@ -102,7 +101,6 @@ public void init(OpenMetadataApplicationConfig config) { this.roleRepository = (RoleRepository) Entity.getEntityRepository(Entity.ROLE); this.tokenRepository = Entity.getTokenRepository(); this.ldapConfiguration = config.getAuthenticationConfiguration().getLdapConfiguration(); - this.loginAttemptCache = new LoginAttemptCache(); this.isSelfSignUpEnabled = config.getAuthenticationConfiguration().getEnableSelfSignup(); } @@ -176,7 +174,7 @@ private User checkAndCreateUser(String userDn, String email, String userName) th @Override public void checkIfLoginBlocked(String email) { - if (loginAttemptCache.isLoginBlocked(email)) { + if (LoginAttemptCache.getInstance().isLoginBlocked(email)) { throw new AuthenticationException(MAX_FAILED_LOGIN_ATTEMPT); } } @@ -184,8 +182,8 @@ public void checkIfLoginBlocked(String email) { @Override public void recordFailedLoginAttempt(String email, String userName) throws TemplateException, IOException { - loginAttemptCache.recordFailedLogin(email); - int failedLoginAttempt = loginAttemptCache.getUserFailedLoginCount(email); + LoginAttemptCache.getInstance().recordFailedLogin(email); + int failedLoginAttempt = LoginAttemptCache.getInstance().getUserFailedLoginCount(email); if (failedLoginAttempt == SecurityUtil.getLoginConfiguration().getMaxLoginFailAttempts()) { EmailUtil.sendAccountStatus( userName, diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/security/auth/LoginAttemptCache.java b/openmetadata-service/src/main/java/org/openmetadata/service/security/auth/LoginAttemptCache.java index 0ceb672d098c..336b97e473d1 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/security/auth/LoginAttemptCache.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/security/auth/LoginAttemptCache.java @@ -3,6 +3,7 @@ import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; +import io.dropwizard.logback.shaded.guava.annotations.VisibleForTesting; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import lombok.NonNull; @@ -12,10 +13,11 @@ import org.openmetadata.service.resources.settings.SettingsCache; public class LoginAttemptCache { + private static LoginAttemptCache INSTANCE; private int maxAttempt = 3; private final LoadingCache attemptsCache; - public LoginAttemptCache() { + private LoginAttemptCache() { LoginConfiguration loginConfiguration = SettingsCache.getSetting(SettingsType.LOGIN_CONFIGURATION, LoginConfiguration.class); long accessBlockTime = 600; @@ -35,6 +37,18 @@ public LoginAttemptCache() { }); } + public static LoginAttemptCache getInstance() { + if (INSTANCE == null) { + INSTANCE = new LoginAttemptCache(); + } + return INSTANCE; + } + + public static void updateLoginConfiguration() { + INSTANCE = new LoginAttemptCache(); + } + + @VisibleForTesting public LoginAttemptCache(int maxAttempt, int blockTimeInSec) { this.maxAttempt = maxAttempt; attemptsCache = diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/security/jwt/JWTTokenGenerator.java b/openmetadata-service/src/main/java/org/openmetadata/service/security/jwt/JWTTokenGenerator.java index 868175326469..21aaeeeef9b2 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/security/jwt/JWTTokenGenerator.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/security/jwt/JWTTokenGenerator.java @@ -37,6 +37,7 @@ import java.util.Set; import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.openmetadata.schema.api.security.AuthenticationConfiguration; import org.openmetadata.schema.api.security.jwt.JWTTokenConfiguration; import org.openmetadata.schema.auth.JWTAuthMechanism; import org.openmetadata.schema.auth.JWTTokenExpiry; @@ -56,6 +57,7 @@ public class JWTTokenGenerator { @Getter private RSAPublicKey publicKey; private String issuer; private String kid; + private AuthenticationConfiguration.TokenValidationAlgorithm tokenValidationAlgorithm; private JWTTokenGenerator() { /* Private constructor for singleton */ @@ -66,7 +68,9 @@ public static JWTTokenGenerator getInstance() { } /** Expected to be initialized only once during application start */ - public void init(JWTTokenConfiguration jwtTokenConfiguration) { + public void init( + AuthenticationConfiguration.TokenValidationAlgorithm algorithm, + JWTTokenConfiguration jwtTokenConfiguration) { try { if (jwtTokenConfiguration.getRsaprivateKeyFilePath() != null && !jwtTokenConfiguration.getRsaprivateKeyFilePath().isEmpty() @@ -84,6 +88,7 @@ public void init(JWTTokenConfiguration jwtTokenConfiguration) { publicKey = (RSAPublicKey) kf.generatePublic(spec); issuer = jwtTokenConfiguration.getJwtissuer(); kid = jwtTokenConfiguration.getKeyId(); + tokenValidationAlgorithm = algorithm; } } catch (Exception ex) { LOG.error("Failed to initialize JWTTokenGenerator ", ex); @@ -141,7 +146,7 @@ public JWTAuthMechanism getJwtAuthMechanism( } } JWTAuthMechanism jwtAuthMechanism = new JWTAuthMechanism().withJWTTokenExpiry(expiry); - Algorithm algorithm = Algorithm.RSA256(null, privateKey); + Algorithm algorithm = getAlgorithm(tokenValidationAlgorithm, null, privateKey); String token = JWT.create() .withIssuer(issuer) @@ -214,4 +219,15 @@ public Date getTokenExpiryFromJWT(String token) { return jwt.getExpiresAt(); } + + public static Algorithm getAlgorithm( + AuthenticationConfiguration.TokenValidationAlgorithm algorithm, + RSAPublicKey publicKey, + RSAPrivateKey privateKey) { + return switch (algorithm) { + case RS_256 -> Algorithm.RSA256(publicKey, privateKey); + case RS_384 -> Algorithm.RSA384(publicKey, privateKey); + case RS_512 -> Algorithm.RSA512(publicKey, privateKey); + }; + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/security/policyevaluator/CompiledRule.java b/openmetadata-service/src/main/java/org/openmetadata/service/security/policyevaluator/CompiledRule.java index 659cb9fb23dd..2a2c25c2c0a6 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/security/policyevaluator/CompiledRule.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/security/policyevaluator/CompiledRule.java @@ -197,16 +197,17 @@ protected boolean matchResource(String resource) { } private boolean matchOperation(MetadataOperation operation) { - if (getOperations().contains(MetadataOperation.ALL)) { + List operations = getOperations(); + if (operations.contains(MetadataOperation.ALL)) { LOG.debug("matched all operations"); return true; // Match all operations } - if (getOperations().contains(MetadataOperation.EDIT_ALL) + if (operations.contains(MetadataOperation.EDIT_ALL) && OperationContext.isEditOperation(operation)) { LOG.debug("matched editAll operations"); return true; } - if (getOperations().contains(MetadataOperation.VIEW_ALL) + if (operations.contains(MetadataOperation.VIEW_ALL) && OperationContext.isViewOperation(operation)) { LOG.debug("matched viewAll operations"); return true; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/security/policyevaluator/RuleEvaluator.java b/openmetadata-service/src/main/java/org/openmetadata/service/security/policyevaluator/RuleEvaluator.java index 0327ed8a6280..4c5bff12f9c5 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/security/policyevaluator/RuleEvaluator.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/security/policyevaluator/RuleEvaluator.java @@ -5,8 +5,10 @@ import java.util.Arrays; import java.util.List; +import java.util.Optional; import lombok.extern.slf4j.Slf4j; import org.openmetadata.schema.Function; +import org.openmetadata.schema.type.AssetCertification; import org.openmetadata.schema.type.TagLabel; import org.openmetadata.service.Entity; import org.openmetadata.service.security.policyevaluator.SubjectContext.PolicyContext; @@ -142,20 +144,59 @@ public boolean matchAnyTag(String... tagFQNs) { return false; } List tags = resourceContext.getTags(); - LOG.debug( - "matchAnyTag {} resourceTags {}", - Arrays.toString(tagFQNs), - Arrays.toString(tags.toArray())); - for (String tagFQN : tagFQNs) { - TagLabel found = - tags.stream().filter(t -> t.getTagFQN().equals(tagFQN)).findAny().orElse(null); - if (found != null) { - return true; + if (!nullOrEmpty(tags)) { + LOG.debug( + "matchAnyTag {} resourceTags {}", + Arrays.toString(tagFQNs), + Arrays.toString(tags.toArray())); + for (String tagFQN : tagFQNs) { + TagLabel found = + tags.stream().filter(t -> t.getTagFQN().equals(tagFQN)).findAny().orElse(null); + if (found != null) { + return true; + } } } return false; } + @Function( + name = "matchAnyCertification", + input = "List of comma separated Certification fully qualified names", + description = + "Returns true if the entity being accessed has any of the Certification given as input", + examples = {"matchAnyCertification('Certification.Silver', 'Certification.Gold')"}) + @SuppressWarnings("unused") // Used in SpelExpressions + public boolean matchAnyCertification(String... tagFQNs) { + if (expressionValidation) { + for (String tagFqn : tagFQNs) { + Entity.getEntityReferenceByName(Entity.TAG, tagFqn, NON_DELETED); // Validate tag exists + } + return false; + } + if (resourceContext == null) { + return false; + } + + Optional oCertification = + Optional.ofNullable(resourceContext.getEntity().getCertification()); + + if (oCertification.isEmpty()) { + LOG.debug( + "matchAnyCertification {} resourceCertification is null.", Arrays.toString(tagFQNs)); + return false; + } else { + AssetCertification certification = oCertification.get(); + + LOG.debug( + "matchAnyCertification {} resourceCertification {}", + Arrays.toString(tagFQNs), + certification.getTagLabel().getTagFQN()); + return Arrays.stream(tagFQNs) + .anyMatch(tagFQN -> tagFQN.equals(certification.getTagLabel().getTagFQN())); + } + } + @Function( name = "matchTeam", input = "None", diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/security/saml/SamlAssertionConsumerServlet.java b/openmetadata-service/src/main/java/org/openmetadata/service/security/saml/SamlAssertionConsumerServlet.java index 2bc6a9cbd2bb..516ba6d54e83 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/security/saml/SamlAssertionConsumerServlet.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/security/saml/SamlAssertionConsumerServlet.java @@ -130,17 +130,22 @@ private void handleResponse(HttpServletRequest req, HttpServletResponse resp) th // Redirect with JWT Token String redirectUri = (String) req.getSession().getAttribute(SESSION_REDIRECT_URI); String url = - redirectUri - + "?id_token=" - + jwtAuthMechanism.getJWTToken() - + "&email=" - + nameId - + "&name=" - + username; + String.format( + "%s?id_token=%s&email=%s&name=%s", + (nullOrEmpty(redirectUri) ? buildBaseRequestUrl(req) : redirectUri), + jwtAuthMechanism.getJWTToken(), + nameId, + username); resp.sendRedirect(url); } } + private String buildBaseRequestUrl(HttpServletRequest req) { + // In case of IDP initiated one it needs to be built on fly, since the session might not exist + return String.format( + "%s://%s:%s/saml/callback", req.getScheme(), req.getServerName(), req.getServerPort()); + } + private JwtResponse getJwtResponseWithRefresh( User storedUser, JWTAuthMechanism jwtAuthMechanism) { RefreshToken newRefreshToken = TokenUtil.getRefreshToken(storedUser.getId(), UUID.randomUUID()); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/socket/WebSocketManager.java b/openmetadata-service/src/main/java/org/openmetadata/service/socket/WebSocketManager.java index d4f6ff9ebf01..e794166e29eb 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/socket/WebSocketManager.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/socket/WebSocketManager.java @@ -26,6 +26,7 @@ public class WebSocketManager { public static final String TASK_BROADCAST_CHANNEL = "taskChannel"; public static final String SEARCH_INDEX_JOB_BROADCAST_CHANNEL = "searchIndexJobStatus"; public static final String DATA_INSIGHTS_JOB_BROADCAST_CHANNEL = "dataInsightsJobStatus"; + public static final String BACKGROUND_JOB_CHANNEL = "backgroundJobStatus"; public static final String MENTION_CHANNEL = "mentionChannel"; public static final String ANNOUNCEMENT_CHANNEL = "announcementChannel"; public static final String CSV_EXPORT_CHANNEL = "csvExportChannel"; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/AppMarketPlaceUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/AppMarketPlaceUtil.java new file mode 100644 index 000000000000..28a5c5657649 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/AppMarketPlaceUtil.java @@ -0,0 +1,59 @@ +package org.openmetadata.service.util; + +import static org.openmetadata.service.Entity.APPLICATION; +import static org.openmetadata.service.jdbi3.AppRepository.APP_BOT_ROLE; +import static org.openmetadata.service.jdbi3.EntityRepository.getEntitiesFromSeedData; + +import java.io.IOException; +import java.util.List; +import org.openmetadata.schema.entity.app.AppMarketPlaceDefinition; +import org.openmetadata.schema.entity.app.CreateAppMarketPlaceDefinitionReq; +import org.openmetadata.schema.entity.teams.Role; +import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.type.Include; +import org.openmetadata.service.Entity; +import org.openmetadata.service.exception.EntityNotFoundException; +import org.openmetadata.service.jdbi3.AppMarketPlaceRepository; +import org.openmetadata.service.jdbi3.PolicyRepository; +import org.openmetadata.service.jdbi3.RoleRepository; +import org.openmetadata.service.jdbi3.TeamRepository; +import org.openmetadata.service.resources.apps.AppMarketPlaceMapper; + +public class AppMarketPlaceUtil { + public static void createAppMarketPlaceDefinitions( + AppMarketPlaceRepository appMarketRepository, AppMarketPlaceMapper mapper) + throws IOException { + PolicyRepository policyRepository = Entity.getPolicyRepository(); + RoleRepository roleRepository = Entity.getRoleRepository(); + + try { + roleRepository.findByName(APP_BOT_ROLE, Include.NON_DELETED); + } catch (EntityNotFoundException e) { + policyRepository.initSeedDataFromResources(); + List roles = roleRepository.getEntitiesFromSeedData(); + for (Role role : roles) { + role.setFullyQualifiedName(role.getName()); + List policies = role.getPolicies(); + for (EntityReference policy : policies) { + EntityReference ref = + Entity.getEntityReferenceByName(Entity.POLICY, policy.getName(), Include.NON_DELETED); + policy.setId(ref.getId()); + } + roleRepository.initializeEntity(role); + } + TeamRepository teamRepository = (TeamRepository) Entity.getEntityRepository(Entity.TEAM); + teamRepository.initOrganization(); + } + + List createAppMarketPlaceDefinitionReqs = + getEntitiesFromSeedData( + APPLICATION, + String.format(".*json/data/%s/.*\\.json$", Entity.APP_MARKET_PLACE_DEF), + CreateAppMarketPlaceDefinitionReq.class); + for (CreateAppMarketPlaceDefinitionReq definitionReq : createAppMarketPlaceDefinitionReqs) { + AppMarketPlaceDefinition definition = mapper.createToEntity(definitionReq, "admin"); + appMarketRepository.setFullyQualifiedName(definition); + appMarketRepository.createOrUpdate(null, definition); + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/EntityHierarchyList.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/EntityHierarchyList.java new file mode 100644 index 000000000000..fe58dc390de3 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/EntityHierarchyList.java @@ -0,0 +1,13 @@ +package org.openmetadata.service.util; + +import java.util.List; +import org.openmetadata.schema.entity.data.EntityHierarchy; + +public class EntityHierarchyList extends ResultList { + @SuppressWarnings("unused") + public EntityHierarchyList() {} + + public EntityHierarchyList(List data) { + super(data, null, null, data.size()); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/JsonUtils.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/JsonUtils.java index f4b1ab79d842..6cedb7bc1019 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/util/JsonUtils.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/JsonUtils.java @@ -17,6 +17,7 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.StreamReadConstraints; import com.fasterxml.jackson.core.StreamReadFeature; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; @@ -39,6 +40,7 @@ import java.nio.file.Paths; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -86,6 +88,10 @@ public final class JsonUtils { static { OBJECT_MAPPER = new ObjectMapper(); + OBJECT_MAPPER + .getFactory() + .setStreamReadConstraints( + StreamReadConstraints.builder().maxStringLength(Integer.MAX_VALUE).build()); // Ensure the date-time fields are serialized in ISO-8601 format OBJECT_MAPPER.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); OBJECT_MAPPER.setDateFormat(DATE_TIME_FORMAT); @@ -640,6 +646,24 @@ public static JsonNode pojoToJsonNode(Object obj) { } } + @SuppressWarnings("unused") + public static Map getMapFromJson(String json) { + return (Map) (JsonUtils.readValue(json, Map.class)); + } + + @SuppressWarnings("unused") + public static T convertObjectWithFilteredFields( + Object input, Set fields, Class clazz) { + Map inputMap = JsonUtils.getMap(input); + Map result = new HashMap<>(); + for (String field : fields) { + if (inputMap.containsKey(field)) { + result.put(field, inputMap.get(field)); + } + } + return JsonUtils.convertValue(result, clazz); + } + public static JsonPatch convertFgeToJavax(com.github.fge.jsonpatch.JsonPatch fgeJsonPatch) { String jsonString = fgeJsonPatch.toString(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java index 02be91fc84ea..1a183c7ff404 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java @@ -2,9 +2,13 @@ import static org.flywaydb.core.internal.info.MigrationInfoDumper.dumpToAsciiTable; import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; +import static org.openmetadata.service.Entity.ADMIN_USER_NAME; import static org.openmetadata.service.Entity.FIELD_OWNERS; +import static org.openmetadata.service.apps.bundles.insights.utils.TimestampUtils.timestampToString; import static org.openmetadata.service.formatter.decorators.MessageDecorator.getDateStringEpochMilli; +import static org.openmetadata.service.jdbi3.UserRepository.AUTH_MECHANISM_FIELD; import static org.openmetadata.service.util.AsciiTable.printOpenMetadataText; +import static org.openmetadata.service.util.UserUtil.updateUserWithHashedPwd; import ch.qos.logback.classic.Level; import ch.qos.logback.classic.Logger; @@ -23,9 +27,11 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Scanner; import java.util.Set; import java.util.concurrent.Callable; @@ -39,8 +45,17 @@ import org.openmetadata.schema.EntityInterface; import org.openmetadata.schema.ServiceEntityInterface; import org.openmetadata.schema.entity.app.App; +import org.openmetadata.schema.entity.app.AppConfiguration; +import org.openmetadata.schema.entity.app.AppMarketPlaceDefinition; import org.openmetadata.schema.entity.app.AppRunRecord; +import org.openmetadata.schema.entity.app.AppSchedule; +import org.openmetadata.schema.entity.app.CreateApp; +import org.openmetadata.schema.entity.app.ScheduleTimeline; +import org.openmetadata.schema.entity.applications.configuration.internal.BackfillConfiguration; +import org.openmetadata.schema.entity.applications.configuration.internal.DataInsightsAppConfig; import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline; +import org.openmetadata.schema.entity.teams.User; +import org.openmetadata.schema.services.connections.metadata.AuthProvider; import org.openmetadata.schema.settings.Settings; import org.openmetadata.schema.settings.SettingsType; import org.openmetadata.schema.system.EventPublisherJob; @@ -48,26 +63,34 @@ import org.openmetadata.sdk.PipelineServiceClientInterface; import org.openmetadata.service.Entity; import org.openmetadata.service.OpenMetadataApplicationConfig; +import org.openmetadata.service.TypeRegistry; import org.openmetadata.service.apps.ApplicationHandler; import org.openmetadata.service.apps.scheduler.AppScheduler; import org.openmetadata.service.clients.pipeline.PipelineServiceClientFactory; import org.openmetadata.service.exception.EntityNotFoundException; import org.openmetadata.service.fernet.Fernet; +import org.openmetadata.service.jdbi3.AppMarketPlaceRepository; import org.openmetadata.service.jdbi3.AppRepository; import org.openmetadata.service.jdbi3.CollectionDAO; import org.openmetadata.service.jdbi3.EntityRepository; +import org.openmetadata.service.jdbi3.EventSubscriptionRepository; import org.openmetadata.service.jdbi3.IngestionPipelineRepository; import org.openmetadata.service.jdbi3.ListFilter; import org.openmetadata.service.jdbi3.MigrationDAO; import org.openmetadata.service.jdbi3.SystemRepository; +import org.openmetadata.service.jdbi3.TypeRepository; +import org.openmetadata.service.jdbi3.UserRepository; import org.openmetadata.service.jdbi3.locator.ConnectionType; import org.openmetadata.service.migration.api.MigrationWorkflow; import org.openmetadata.service.resources.CollectionRegistry; +import org.openmetadata.service.resources.apps.AppMapper; +import org.openmetadata.service.resources.apps.AppMarketPlaceMapper; import org.openmetadata.service.resources.databases.DatasourceConfig; import org.openmetadata.service.search.SearchRepository; import org.openmetadata.service.secrets.SecretsManager; import org.openmetadata.service.secrets.SecretsManagerFactory; import org.openmetadata.service.secrets.SecretsManagerUpdateService; +import org.openmetadata.service.security.jwt.JWTTokenGenerator; import org.openmetadata.service.util.jdbi.DatabaseAuthenticationProviderFactory; import org.openmetadata.service.util.jdbi.JdbiUtils; import org.slf4j.LoggerFactory; @@ -166,13 +189,12 @@ public Integer repair() { public Integer syncEmailFromEnv() { try { parseConfig(); - Entity.setCollectionDAO(jdbi.onDemand(CollectionDAO.class)); - SystemRepository systemRepository = new SystemRepository(); Settings updatedSettings = new Settings() .withConfigType(SettingsType.EMAIL_CONFIGURATION) .withConfigValue(config.getSmtpSettings()); - systemRepository.createOrUpdate(updatedSettings); + Entity.getSystemRepository().createOrUpdate(updatedSettings); + LOG.info("Synced Email Configuration from Environment."); return 0; } catch (Exception e) { LOG.error("Email Sync failed due to ", e); @@ -180,6 +202,112 @@ public Integer syncEmailFromEnv() { } } + @Command(name = "install-app", description = "Install the application from App MarketPlace.") + public Integer installApp( + @Option( + names = {"-n", "--name"}, + description = "The name of the application to install.", + required = true) + String appName, + @Option( + names = {"--force"}, + description = "Forces migrations to be run again, even if they have ran previously", + defaultValue = "false") + boolean force) { + try { + parseConfig(); + AppRepository appRepository = (AppRepository) Entity.getEntityRepository(Entity.APPLICATION); + + if (!force && isAppInstalled(appRepository, appName)) { + LOG.info("App already installed."); + return 0; + } + + if (force && deleteApplication(appRepository, appName)) { + LOG.info("App deleted."); + } + + LOG.info("App not installed. Installing..."); + installApplication(appName, appRepository); + LOG.info("App Installed."); + return 0; + } catch (Exception e) { + LOG.error("Install Application Failed", e); + return 1; + } + } + + @Command(name = "delete-app", description = "Delete the installed application.") + public Integer deleteApp( + @Option( + names = {"-n", "--name"}, + description = "The name of the application to install.", + required = true) + String appName) { + try { + parseConfig(); + AppRepository appRepository = (AppRepository) Entity.getEntityRepository(Entity.APPLICATION); + if (deleteApplication(appRepository, appName)) { + LOG.info("App deleted."); + } + return 0; + } catch (Exception e) { + LOG.error("Delete Application Failed", e); + return 1; + } + } + + private boolean isAppInstalled(AppRepository appRepository, String appName) { + try { + appRepository.findByName(appName, Include.NON_DELETED); + return true; + } catch (EntityNotFoundException e) { + return false; + } + } + + private boolean deleteApplication(AppRepository appRepository, String appName) { + try { + appRepository.deleteByName(ADMIN_USER_NAME, appName, true, true); + return true; + } catch (EntityNotFoundException e) { + return false; + } + } + + private void installApplication(String appName, AppRepository appRepository) throws Exception { + PipelineServiceClientInterface pipelineServiceClient = + PipelineServiceClientFactory.createPipelineServiceClient( + config.getPipelineServiceClientConfiguration()); + + JWTTokenGenerator.getInstance() + .init( + config.getAuthenticationConfiguration().getTokenValidationAlgorithm(), + config.getJwtTokenConfiguration()); + + AppMarketPlaceMapper mapper = new AppMarketPlaceMapper(pipelineServiceClient); + AppMarketPlaceRepository appMarketRepository = + (AppMarketPlaceRepository) Entity.getEntityRepository(Entity.APP_MARKET_PLACE_DEF); + + AppMarketPlaceUtil.createAppMarketPlaceDefinitions(appMarketRepository, mapper); + + AppMarketPlaceDefinition definition = + appMarketRepository.getByName(null, appName, appMarketRepository.getFields("id")); + + CreateApp createApp = + new CreateApp() + .withName(definition.getName()) + .withDescription(definition.getDescription()) + .withDisplayName(definition.getDisplayName()) + .withAppSchedule(new AppSchedule().withScheduleTimeline(ScheduleTimeline.NONE)) + .withAppConfiguration(new AppConfiguration()); + + AppMapper appMapper = new AppMapper(); + App entity = appMapper.createToEntity(createApp, ADMIN_USER_NAME); + appRepository.prepareInternal(entity, true); + appRepository.createOrUpdate(null, entity); + } + @Command( name = "check-connection", description = @@ -221,6 +349,58 @@ public Integer dropCreate() { } } + @Command(name = "reset-password", description = "Reset the password for a user.") + public Integer resetUserPassword( + @Option( + names = {"-e", "--email"}, + description = "Email for which to reset the password.", + required = true) + String email, + @Option( + names = {"-p", "--password"}, + description = "Enter user password", + arity = "0..1", + interactive = true, + required = true) + char[] password) { + try { + LOG.info("Resetting password for user : {}", email); + if (nullOrEmpty(password)) { + throw new IllegalArgumentException("Password cannot be empty."); + } + parseConfig(); + CollectionRegistry.initialize(); + AuthProvider authProvider = config.getAuthenticationConfiguration().getProvider(); + + // Only Basic Auth provider is supported for password reset + if (!authProvider.equals(AuthProvider.BASIC)) { + LOG.error("Auth Provider is Not Basic. Cannot apply Password"); + return 1; + } + + UserRepository userRepository = (UserRepository) Entity.getEntityRepository(Entity.USER); + Set fieldList = new HashSet<>(userRepository.getPatchFields().getFieldList()); + fieldList.add(AUTH_MECHANISM_FIELD); + User originalUser = userRepository.getByEmail(null, email, new EntityUtil.Fields(fieldList)); + + // Check if the user is a bot user + if (Boolean.TRUE.equals(originalUser.getIsBot())) { + LOG.error("Bot user : {} cannot have password.", originalUser.getName()); + return 1; + } + + User updatedUser = JsonUtils.deepCopy(originalUser, User.class); + String inputPwd = new String(password); + updateUserWithHashedPwd(updatedUser, inputPwd); + UserUtil.addOrUpdateUser(updatedUser); + LOG.info("Password updated successfully."); + return 0; + } catch (Exception e) { + LOG.error("Failed to reset user password.", e); + return 1; + } + } + @Command( name = "migrate", description = "Migrates the OpenMetadata database schema and search index mappings.") @@ -311,10 +491,17 @@ public Integer reIndex( names = {"--retries"}, defaultValue = "3", description = "Maximum number of retries for failed search requests.") - int retries) { + int retries, + @Option( + names = {"--entities"}, + defaultValue = "'all'", + description = + "Entities to reindex. Passing --entities='table,dashboard' will reindex table and dashboard entities. Passing nothing will reindex everything.") + String entityStr) { try { LOG.info( - "Running Reindexing with Batch Size: {}, Payload Size: {}, Recreate-Index: {}, Producer threads: {}, Consumer threads: {}, Queue Size: {}, Back-off: {}, Max Back-off: {}, Max Requests: {}, Retries: {}", + "Running Reindexing with Entities:{} , Batch Size: {}, Payload Size: {}, Recreate-Index: {}, Producer threads: {}, Consumer threads: {}, Queue Size: {}, Back-off: {}, Max Back-off: {}, Max Requests: {}, Retries: {}", + entityStr, batchSize, payloadSize, recreateIndexes, @@ -330,10 +517,15 @@ public Integer reIndex( ApplicationHandler.initialize(config); CollectionRegistry.getInstance().loadSeedData(jdbi, config, null, null, null, true); ApplicationHandler.initialize(config); + TypeRepository typeRepository = (TypeRepository) Entity.getEntityRepository(Entity.TYPE); + TypeRegistry.instance().initialize(typeRepository); AppScheduler.initialize(config, collectionDAO, searchRepository); String appName = "SearchIndexingApplication"; + Set entities = + new HashSet<>(Arrays.asList(entityStr.substring(1, entityStr.length() - 1).split(","))); return executeSearchReindexApp( appName, + entities, batchSize, payloadSize, recreateIndexes, @@ -350,8 +542,29 @@ public Integer reIndex( } } + @Command(name = "syncAlertOffset", description = "Sync the Alert Offset.") + public Integer reIndex( + @Option( + names = {"-n", "--name"}, + description = "Name of the alerts.", + required = true) + String alertName) { + try { + parseConfig(); + CollectionRegistry.initialize(); + EventSubscriptionRepository repository = + (EventSubscriptionRepository) Entity.getEntityRepository(Entity.EVENT_SUBSCRIPTION); + repository.syncEventSubscriptionOffset(alertName); + return 0; + } catch (Exception e) { + LOG.error("Failed to sync alert offset due to ", e); + return 1; + } + } + private int executeSearchReindexApp( String appName, + Set entities, int batchSize, long payloadSize, boolean recreateIndexes, @@ -372,6 +585,7 @@ private int executeSearchReindexApp( EventPublisherJob updatedJob = JsonUtils.deepCopy(storedJob, EventPublisherJob.class); updatedJob + .withEntities(entities) .withBatchSize(batchSize) .withPayLoadSize(payloadSize) .withRecreateIndex(recreateIndexes) @@ -381,8 +595,7 @@ private int executeSearchReindexApp( .withInitialBackoff(backOff) .withMaxBackoff(maxBackOff) .withMaxConcurrentRequests(maxRequests) - .withMaxRetries(retries) - .withEntities(Set.of("all")); + .withMaxRetries(retries); // Update the search index app with the new configurations App updatedSearchIndexApp = JsonUtils.deepCopy(originalSearchIndexApp, App.class); @@ -404,6 +617,100 @@ private int executeSearchReindexApp( return result; } + @Command( + name = "reindexdi", + description = "Re Indexes data insights into search engine from command line.") + public Integer reIndexDI( + @Option( + names = {"-b", "--batch-size"}, + defaultValue = "100", + description = "Number of records to process in each batch.") + int batchSize, + @Option( + names = {"--recreate-indexes"}, + defaultValue = "true", + description = "Flag to determine if indexes should be recreated.") + boolean recreateIndexes, + @Option( + names = {"--start-date"}, + description = "Start Date to backfill from.") + String startDate, + @Option( + names = {"--end-date"}, + description = "End Date to backfill to.") + String endDate) { + try { + LOG.info( + "Running Reindexing with Batch Size: {}, Recreate-Index: {}, Start Date: {}, End Date: {}.", + batchSize, + recreateIndexes, + startDate, + endDate); + parseConfig(); + CollectionRegistry.initialize(); + ApplicationHandler.initialize(config); + CollectionRegistry.getInstance().loadSeedData(jdbi, config, null, null, null, true); + ApplicationHandler.initialize(config); + AppScheduler.initialize(config, collectionDAO, searchRepository); + return executeDataInsightsReindexApp( + batchSize, recreateIndexes, getBackfillConfiguration(startDate, endDate)); + } catch (Exception e) { + LOG.error("Failed to reindex due to ", e); + return 1; + } + } + + private BackfillConfiguration getBackfillConfiguration(String startDate, String endDate) { + BackfillConfiguration backfillConfiguration = new BackfillConfiguration(); + backfillConfiguration.withEnabled(false); + + if (startDate != null) { + backfillConfiguration.withEnabled(true); + backfillConfiguration.withStartDate(startDate); + backfillConfiguration.withEndDate( + Objects.requireNonNullElseGet( + endDate, () -> timestampToString(System.currentTimeMillis(), "yyyy-MM-dd"))); + } + return backfillConfiguration; + } + + private int executeDataInsightsReindexApp( + int batchSize, boolean recreateIndexes, BackfillConfiguration backfillConfiguration) { + AppRepository appRepository = (AppRepository) Entity.getEntityRepository(Entity.APPLICATION); + App originalDataInsightsApp = + appRepository.getByName(null, "DataInsightsApplication", appRepository.getFields("id")); + + DataInsightsAppConfig storedConfig = + JsonUtils.convertValue( + originalDataInsightsApp.getAppConfiguration(), DataInsightsAppConfig.class); + + DataInsightsAppConfig updatedConfig = + JsonUtils.deepCopy(storedConfig, DataInsightsAppConfig.class); + updatedConfig + .withBatchSize(batchSize) + .withRecreateDataAssetsIndex(recreateIndexes) + .withBackfillConfiguration(backfillConfiguration); + + // Update the data insights app with the new configurations + App updatedDataInsightsApp = JsonUtils.deepCopy(originalDataInsightsApp, App.class); + updatedDataInsightsApp.withAppConfiguration(updatedConfig); + JsonPatch patch = JsonUtils.getJsonPatch(originalDataInsightsApp, updatedDataInsightsApp); + + appRepository.patch(null, originalDataInsightsApp.getId(), "admin", patch); + + // Trigger Application + long currentTime = System.currentTimeMillis(); + AppScheduler.getInstance().triggerOnDemandApplication(updatedDataInsightsApp); + + int result = waitAndReturnReindexingAppStatus(updatedDataInsightsApp, currentTime); + + // Re-patch with original configuration + JsonPatch repatch = JsonUtils.getJsonPatch(updatedDataInsightsApp, originalDataInsightsApp); + appRepository.patch(null, originalDataInsightsApp.getId(), "admin", repatch); + + return result; + } + @SneakyThrows private int waitAndReturnReindexingAppStatus(App searchIndexApp, long startTime) { AppRunRecord appRunRecord = null; @@ -652,6 +959,7 @@ private void parseConfig() throws Exception { collectionDAO = jdbi.onDemand(CollectionDAO.class); Entity.setSearchRepository(searchRepository); Entity.setCollectionDAO(collectionDAO); + Entity.setSystemRepository(new SystemRepository()); Entity.initializeRepositories(config, jdbi); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/ResultList.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/ResultList.java index b81690dbf342..ecaadd91c7b9 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/util/ResultList.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/ResultList.java @@ -17,6 +17,8 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonPropertyOrder; import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; import javax.validation.constraints.NotNull; import org.openmetadata.schema.system.EntityError; import org.openmetadata.schema.type.Paging; @@ -95,6 +97,11 @@ public ResultList(List data, Integer offset, int total) { paging = new Paging().withBefore(null).withAfter(null).withTotal(total).withOffset(offset); } + /* Conveniently map the data to another type without the need to create a new ResultList */ + public ResultList map(Function mapper) { + return new ResultList<>(data.stream().map(mapper).collect(Collectors.toList()), paging); + } + public ResultList(List data, Integer offset, Integer limit, Integer total) { this.data = data; paging = @@ -106,6 +113,17 @@ public ResultList(List data, Integer offset, Integer limit, Integer total) { .withLimit(limit); } + public ResultList(List data, Paging other) { + this.data = data; + paging = + new Paging() + .withBefore(null) + .withAfter(null) + .withTotal(other.getTotal()) + .withOffset(other.getOffset()) + .withLimit(other.getLimit()); + } + public ResultList( List data, List errors, String beforeCursor, String afterCursor, int total) { this.data = data; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/SchemaFieldExtractor.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/SchemaFieldExtractor.java index e6d2d446cd56..9186d7721ad2 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/util/SchemaFieldExtractor.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/SchemaFieldExtractor.java @@ -32,7 +32,7 @@ @Slf4j public class SchemaFieldExtractor { - private static final Map> entityFieldsCache = + private static final Map> entityFieldsCache = new ConcurrentHashMap<>(); public SchemaFieldExtractor() { @@ -55,7 +55,7 @@ private static void initializeEntityFieldsCache() { Schema mainSchema = loadMainSchema(schemaPath, entityType, schemaUri, schemaClient); // Extract fields from the schema - Map fieldTypesMap = new LinkedHashMap<>(); + Map fieldTypesMap = new LinkedHashMap<>(); Deque processingStack = new ArrayDeque<>(); Set processedFields = new HashSet<>(); extractFieldsFromSchema(mainSchema, "", fieldTypesMap, processingStack, processedFields); @@ -75,7 +75,7 @@ public List extractFields(Type typeEntity, String entityType) { SchemaClient schemaClient = new CustomSchemaClient(schemaUri); Deque processingStack = new ArrayDeque<>(); Set processedFields = new HashSet<>(); - Map fieldTypesMap = entityFieldsCache.get(entityType); + Map fieldTypesMap = entityFieldsCache.get(entityType); addCustomProperties( typeEntity, schemaUri, schemaClient, fieldTypesMap, processingStack, processedFields); return convertMapToFieldList(fieldTypesMap); @@ -90,7 +90,7 @@ public Map> extractAllCustomProperties( SchemaClient schemaClient = new CustomSchemaClient(schemaUri); EntityUtil.Fields fieldsParam = new EntityUtil.Fields(Set.of("customProperties")); Type typeEntity = repository.getByName(uriInfo, entityType, fieldsParam, Include.ALL, false); - Map fieldTypesMap = new LinkedHashMap<>(); + Map fieldTypesMap = new LinkedHashMap<>(); Set processedFields = new HashSet<>(); Deque processingStack = new ArrayDeque<>(); addCustomProperties( @@ -170,7 +170,7 @@ private static Schema loadMainSchema( private static void extractFieldsFromSchema( Schema schema, String parentPath, - Map fieldTypesMap, + Map fieldTypesMap, Deque processingStack, Set processedFields) { if (processingStack.contains(schema)) { @@ -206,7 +206,8 @@ private static void extractFieldsFromSchema( arraySchema, fullFieldName, fieldTypesMap, processingStack, processedFields); } else { String fieldType = mapSchemaTypeToSimpleType(fieldSchema); - fieldTypesMap.putIfAbsent(fullFieldName, fieldType); + fieldTypesMap.putIfAbsent( + fullFieldName, new FieldDefinition(fullFieldName, fieldType, null)); processedFields.add(fullFieldName); LOG.debug("Added field '{}', Type: '{}'", fullFieldName, fieldType); // Recursively process nested objects or arrays @@ -220,7 +221,7 @@ private static void extractFieldsFromSchema( handleArraySchema(arraySchema, parentPath, fieldTypesMap, processingStack, processedFields); } else { String fieldType = mapSchemaTypeToSimpleType(schema); - fieldTypesMap.putIfAbsent(parentPath, fieldType); + fieldTypesMap.putIfAbsent(parentPath, new FieldDefinition(parentPath, fieldType, null)); LOG.debug("Added field '{}', Type: '{}'", parentPath, fieldType); } } finally { @@ -231,7 +232,7 @@ private static void extractFieldsFromSchema( private static void handleReferenceSchema( ReferenceSchema referenceSchema, String fullFieldName, - Map fieldTypesMap, + Map fieldTypesMap, Deque processingStack, Set processedFields) { @@ -239,7 +240,8 @@ private static void handleReferenceSchema( String referenceType = determineReferenceType(refUri); if (referenceType != null) { - fieldTypesMap.putIfAbsent(fullFieldName, referenceType); + fieldTypesMap.putIfAbsent( + fullFieldName, new FieldDefinition(fullFieldName, referenceType, null)); processedFields.add(fullFieldName); LOG.debug("Added field '{}', Type: '{}'", fullFieldName, referenceType); if (referenceType.startsWith("array<") && referenceType.endsWith(">")) { @@ -255,7 +257,7 @@ private static void handleReferenceSchema( referredSchema, fullFieldName, fieldTypesMap, processingStack, processedFields); } } else { - fieldTypesMap.putIfAbsent(fullFieldName, "object"); + fieldTypesMap.putIfAbsent(fullFieldName, new FieldDefinition(fullFieldName, "object", null)); processedFields.add(fullFieldName); LOG.debug("Added field '{}', Type: 'object'", fullFieldName); extractFieldsFromSchema( @@ -270,7 +272,7 @@ private static void handleReferenceSchema( private static void handleArraySchema( ArraySchema arraySchema, String fullFieldName, - Map fieldTypesMap, + Map fieldTypesMap, Deque processingStack, Set processedFields) { @@ -282,7 +284,8 @@ private static void handleArraySchema( if (itemsReferenceType != null) { String arrayFieldType = "array<" + itemsReferenceType + ">"; - fieldTypesMap.putIfAbsent(fullFieldName, arrayFieldType); + fieldTypesMap.putIfAbsent( + fullFieldName, new FieldDefinition(fullFieldName, arrayFieldType, null)); processedFields.add(fullFieldName); LOG.debug("Added field '{}', Type: '{}'", fullFieldName, arrayFieldType); Schema referredItemsSchema = itemsReferenceSchema.getReferredSchema(); @@ -292,7 +295,8 @@ private static void handleArraySchema( } } String arrayType = mapSchemaTypeToSimpleType(itemsSchema); - fieldTypesMap.putIfAbsent(fullFieldName, "array<" + arrayType + ">"); + fieldTypesMap.putIfAbsent( + fullFieldName, new FieldDefinition(fullFieldName, "array<" + arrayType + ">", null)); processedFields.add(fullFieldName); LOG.debug("Added field '{}', Type: 'array<{}>'", fullFieldName, arrayType); @@ -306,7 +310,7 @@ private void addCustomProperties( Type typeEntity, String schemaUri, SchemaClient schemaClient, - Map fieldTypesMap, + Map fieldTypesMap, Deque processingStack, Set processedFields) { if (typeEntity == null || typeEntity.getCustomProperties() == null) { @@ -320,9 +324,13 @@ private void addCustomProperties( LOG.debug("Processing custom property '{}'", fullFieldName); + Object customPropertyConfigObj = customProperty.getCustomPropertyConfig(); + if (isEntityReferenceList(propertyType)) { String referenceType = "array"; - fieldTypesMap.putIfAbsent(fullFieldName, referenceType); + FieldDefinition referenceFieldDefinition = + new FieldDefinition(fullFieldName, referenceType, customPropertyConfigObj); + fieldTypesMap.putIfAbsent(fullFieldName, referenceFieldDefinition); processedFields.add(fullFieldName); LOG.debug("Added custom property '{}', Type: '{}'", fullFieldName, referenceType); @@ -337,7 +345,9 @@ private void addCustomProperties( } } else if (isEntityReference(propertyType)) { String referenceType = "entityReference"; - fieldTypesMap.putIfAbsent(fullFieldName, referenceType); + FieldDefinition referenceFieldDefinition = + new FieldDefinition(fullFieldName, referenceType, customPropertyConfigObj); + fieldTypesMap.putIfAbsent(fullFieldName, referenceFieldDefinition); processedFields.add(fullFieldName); LOG.debug("Added custom property '{}', Type: '{}'", fullFieldName, referenceType); @@ -351,17 +361,22 @@ private void addCustomProperties( fullFieldName); } } else { - fieldTypesMap.putIfAbsent(fullFieldName, propertyType); + FieldDefinition entityFieldDefinition = + new FieldDefinition(fullFieldName, propertyType, customPropertyConfigObj); + fieldTypesMap.putIfAbsent(fullFieldName, entityFieldDefinition); processedFields.add(fullFieldName); LOG.debug("Added custom property '{}', Type: '{}'", fullFieldName, propertyType); } } } - private List convertMapToFieldList(Map fieldTypesMap) { + private List convertMapToFieldList(Map fieldTypesMap) { List fieldsList = new ArrayList<>(); - for (Map.Entry entry : fieldTypesMap.entrySet()) { - fieldsList.add(new FieldDefinition(entry.getKey(), entry.getValue())); + for (Map.Entry entry : fieldTypesMap.entrySet()) { + FieldDefinition fieldDef = entry.getValue(); + fieldsList.add( + new FieldDefinition( + fieldDef.getName(), fieldDef.getType(), fieldDef.getCustomPropertyConfig())); } return fieldsList; } @@ -581,8 +596,9 @@ private static String getEntitySubdirectory(String entityType) { Map.of( "dashboard", "data", "table", "data", - "pipeline", "services", - "votes", "data"); + "pipeline", "data", + "votes", "data", + "dataProduct", "domains"); return entityTypeToSubdirectory.getOrDefault(entityType, "data"); } @@ -622,10 +638,12 @@ private String mapUrlToResourcePath(String url) { public static class FieldDefinition { private String name; private String type; + private Object customPropertyConfig; - public FieldDefinition(String name, String type) { + public FieldDefinition(String name, String type, Object customPropertyConfig) { this.name = name; this.type = type; + this.customPropertyConfig = customPropertyConfig; } } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/UserUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/UserUtil.java index 01fe23b3a6c4..6e27e01efdd6 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/util/UserUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/UserUtil.java @@ -19,6 +19,7 @@ import static org.openmetadata.schema.type.Include.NON_DELETED; import static org.openmetadata.service.Entity.ADMIN_ROLE; import static org.openmetadata.service.Entity.ADMIN_USER_NAME; +import static org.openmetadata.service.jdbi3.UserRepository.AUTH_MECHANISM_FIELD; import at.favre.lib.crypto.bcrypt.BCrypt; import java.util.ArrayList; @@ -62,8 +63,18 @@ private UserUtil() { public static void addUsers( AuthProvider authProvider, Set adminUsers, String domain, Boolean isAdmin) { try { - for (String username : adminUsers) { - createOrUpdateUser(authProvider, username, domain, isAdmin); + for (String keyValue : adminUsers) { + String userName = ""; + String password = ""; + if (keyValue.contains(":")) { + String[] keyValueArray = keyValue.split(":"); + userName = keyValueArray[0]; + password = keyValueArray[1]; + } else { + userName = keyValue; + password = getPassword(userName); + } + createOrUpdateUser(authProvider, userName, password, domain, isAdmin); } } catch (Exception ex) { LOG.error("[BootstrapUser] Encountered Exception while bootstrapping admin user", ex); @@ -71,27 +82,26 @@ public static void addUsers( } private static void createOrUpdateUser( - AuthProvider authProvider, String username, String domain, Boolean isAdmin) { + AuthProvider authProvider, String username, String password, String domain, Boolean isAdmin) { UserRepository userRepository = (UserRepository) Entity.getEntityRepository(Entity.USER); User updatedUser = null; try { // Create Required Fields List Set fieldList = new HashSet<>(userRepository.getPatchFields().getFieldList()); - fieldList.add("authenticationMechanism"); + fieldList.add(AUTH_MECHANISM_FIELD); // Fetch Original User, is available User originalUser = userRepository.getByName(null, username, new Fields(fieldList)); if (Boolean.FALSE.equals(originalUser.getIsBot()) - && Boolean.FALSE.equals(originalUser.getIsAdmin())) { + && Boolean.TRUE.equals(originalUser.getIsAdmin())) { updatedUser = originalUser; // Update Auth Mechanism if not present, and send mail to the user if (authProvider.equals(AuthProvider.BASIC)) { if (originalUser.getAuthenticationMechanism() == null || originalUser.getAuthenticationMechanism().equals(new AuthenticationMechanism())) { - String randomPwd = getPassword(username); - updateUserWithHashedPwd(updatedUser, randomPwd); - EmailUtil.sendInviteMailToAdmin(updatedUser, randomPwd); + updateUserWithHashedPwd(updatedUser, password); + EmailUtil.sendInviteMailToAdmin(updatedUser, password); } } else { updatedUser.setAuthenticationMechanism(new AuthenticationMechanism()); @@ -114,9 +124,8 @@ private static void createOrUpdateUser( updatedUser = user(username, domain, username).withIsAdmin(isAdmin).withIsEmailVerified(true); // Update Auth Mechanism if not present, and send mail to the user if (authProvider.equals(AuthProvider.BASIC)) { - String randomPwd = getPassword(username); - updateUserWithHashedPwd(updatedUser, randomPwd); - EmailUtil.sendInviteMailToAdmin(updatedUser, randomPwd); + updateUserWithHashedPwd(updatedUser, password); + EmailUtil.sendInviteMailToAdmin(updatedUser, password); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/email/EmailUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/email/EmailUtil.java index ee3ab278f213..1cdfbac8010e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/util/email/EmailUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/email/EmailUtil.java @@ -30,6 +30,7 @@ import static org.openmetadata.service.util.email.TemplateConstants.INVITE_RANDOM_PASSWORD_TEMPLATE; import static org.openmetadata.service.util.email.TemplateConstants.INVITE_SUBJECT; import static org.openmetadata.service.util.email.TemplateConstants.PASSWORD; +import static org.openmetadata.service.util.email.TemplateConstants.PASSWORD_RESET_LINKKEY; import static org.openmetadata.service.util.email.TemplateConstants.PASSWORD_RESET_SUBJECT; import static org.openmetadata.service.util.email.TemplateConstants.REPORT_SUBJECT; import static org.openmetadata.service.util.email.TemplateConstants.SUPPORT_URL; @@ -179,7 +180,7 @@ public static void sendPasswordResetLink( .add(ENTITY, getSmtpSettings().getEmailingEntity()) .add(SUPPORT_URL, getSmtpSettings().getSupportUrl()) .add(USERNAME, user.getName()) - .add(EMAIL_VERIFICATION_LINKKEY, passwordResetLink) + .add(PASSWORD_RESET_LINKKEY, passwordResetLink) .add(EXPIRATION_TIME_KEY, DEFAULT_EXPIRATION_TIME) .build(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/jdbi/BindConcat.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/jdbi/BindConcat.java new file mode 100644 index 000000000000..733aa0515975 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/jdbi/BindConcat.java @@ -0,0 +1,80 @@ +package org.openmetadata.service.util.jdbi; + +import java.lang.annotation.Annotation; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import java.lang.reflect.Method; +import java.lang.reflect.Parameter; +import java.lang.reflect.Type; +import org.jdbi.v3.sqlobject.customizer.SqlStatementCustomizerFactory; +import org.jdbi.v3.sqlobject.customizer.SqlStatementCustomizingAnnotation; +import org.jdbi.v3.sqlobject.customizer.SqlStatementParameterCustomizer; +import org.openmetadata.service.util.FullyQualifiedName; + +/** Concatenate parts of a string to bind as a parameter, and avoid usage of CONCAT() in where clause */ +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.PARAMETER}) +@SqlStatementCustomizingAnnotation(BindConcat.Factory.class) +public @interface BindConcat { + String value(); // Name of the concatenated parameter to bind + + String original() default + ""; // Optional: Use when both the original and concatenated values are needed + + String[] parts() default {}; // Parts to concatenate (placeholders or static values) + + boolean hash() default false; // Optional: Apply FullyQualifiedName.buildHash if true + + class Factory implements SqlStatementCustomizerFactory { + + @Override + public SqlStatementParameterCustomizer createForParameter( + Annotation annotation, + Class sqlObjectType, + Method method, + Parameter param, + int index, + Type type) { + BindConcat bind = (BindConcat) annotation; + + return (stmt, arg) -> { + String[] partValues = + (arg instanceof String[]) ? (String[]) arg : new String[] {String.valueOf(arg)}; + StringBuilder concatenatedResult = new StringBuilder(); + boolean containsNull = false; + + for (int i = 0; i < bind.parts().length; i++) { + String part = bind.parts()[i]; + if (part.startsWith(":")) { // Dynamic value in argument list to replace placeholder + if (i >= partValues.length) + throw new IllegalArgumentException( + "Not enough values for placeholders in @BindConcat. Expected at least " + + (i + 1) + + " but got " + + partValues.length); + String dynamicValue = partValues[i]; + if (dynamicValue == null) { + containsNull = true; + break; + } + concatenatedResult.append( + bind.hash() ? FullyQualifiedName.buildHash(dynamicValue) : dynamicValue); + } else { // Static part of the string, defined directly in the annotation + concatenatedResult.append(part); + } + } + + String finalValue = containsNull ? null : concatenatedResult.toString(); + stmt.bind(bind.value(), finalValue); + if (!bind.original().isEmpty() && partValues.length > 0) { + String originalValue = partValues[0]; + stmt.bind( + bind.original(), + bind.hash() ? FullyQualifiedName.buildHash(originalValue) : originalValue); + } + }; + } + } +} diff --git a/openmetadata-service/src/main/resources/dataInsights/config.json b/openmetadata-service/src/main/resources/dataInsights/config.json new file mode 100644 index 000000000000..dcf608489841 --- /dev/null +++ b/openmetadata-service/src/main/resources/dataInsights/config.json @@ -0,0 +1,122 @@ +{ + "mappingFields": { + "common": [ + "id", + "description", + "displayName", + "name", + "deleted", + "version", + "owners", + "tags", + "followers", + "extension", + "votes", + "fullyQualifiedName", + "domain", + "dataProducts", + "certification" + ], + "table": [ + "tableType", + "columns", + "databaseSchema", + "database", + "service", + "serviceType" + ], + "storedProcedure": [ + "storedProcedureType", + "databaseSchema", + "database", + "service", + "serviceType" + ], + "databaseSchema": [ + "database", + "service", + "serviceType" + ], + "database": [ + "service", + "serviceType" + ], + "chart": [ + "service", + "serviceType", + "chartType" + ], + "dashboard": [ + "service", + "serviceType", + "dashboardType" + ], + "dashboardDataModel": [ + "service", + "serviceType", + "dataModelType", + "project", + "columns" + ], + "pipeline": [ + "service", + "serviceType", + "pipelineStatus", + "tasks" + ], + "topic": [ + "service", + "serviceType" + ], + "container": [ + "service", + "serviceType", + "numberOfObjects", + "size", + "fileFormats", + "parent", + "children", + "prefix" + ], + "searchIndex": [ + "service", + "serviceType", + "indexType", + "fields" + ], + "mlmodel": [ + "service", + "serviceType", + "mlStore", + "algorithm", + "mlFeatures", + "mlHyperParameters", + "target", + "dashboard", + "server" + ], + "dataProduct": [ + "experts", + "domain", + "assets" + ], + "glossaryTerm": [ + "synonyms", + "glossary", + "parent", + "children", + "relatedTerms", + "references", + "reviewers", + "status", + "usageCount", + "childrenCount" + ], + "tag": [ + "classification", + "parent", + "children", + "usageCount" + ] + } +} \ No newline at end of file diff --git a/openmetadata-service/src/main/resources/dataInsights/elasticsearch/indexMappingsTemplate.json b/openmetadata-service/src/main/resources/dataInsights/elasticsearch/indexMappingsTemplate.json index f8fdd93ff83c..19da7fc9120d 100644 --- a/openmetadata-service/src/main/resources/dataInsights/elasticsearch/indexMappingsTemplate.json +++ b/openmetadata-service/src/main/resources/dataInsights/elasticsearch/indexMappingsTemplate.json @@ -1,138 +1,16 @@ { "template": { - "settings": { - "analysis": { - "normalizer": { - "lowercase_normalizer": { - "type": "custom", - "char_filter": [], - "filter": [ - "lowercase" - ] - } - }, - "analyzer": { - "om_analyzer": { - "tokenizer": "letter", - "filter": [ - "lowercase", - "om_stemmer" - ] - }, - "om_ngram": { - "tokenizer": "ngram", - "min_gram": 3, - "max_gram": 10, - "filter": [ - "lowercase" - ] - } - }, - "filter": { - "om_stemmer": { - "type": "stemmer", - "name": "english" - } - } - } - }, + "settings": {}, "mappings": { "properties": { "@timestamp": { "type": "date" }, - "owners": { - "properties": { - "id": { - "type": "keyword", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 36 - } - } - }, - "type": { - "type": "keyword" - }, - "name": { - "type": "keyword", - "normalizer": "lowercase_normalizer", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "displayName": { - "type": "keyword", - "fields": { - "keyword": { - "type": "keyword", - "normalizer": "lowercase_normalizer", - "ignore_above": 256 - } - } - }, - "fullyQualifiedName": { - "type": "text" - }, - "description": { - "type": "text" - }, - "deleted": { - "type": "text" - }, - "href": { - "type": "text" - } - } - }, - "domain": { - "properties": { - "id": { - "type": "keyword", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 36 - } - } - }, - "type": { - "type": "keyword" - }, - "name": { - "type": "keyword", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "displayName": { - "type": "keyword", - "fields": { - "keyword": { - "type": "keyword", - "normalizer": "lowercase_normalizer", - "ignore_above": 256 - } - } - }, - "fullyQualifiedName": { + "id": { + "type": "text", + "fields": { + "keyword": { "type": "keyword" - }, - "description": { - "type": "text" - }, - "deleted": { - "type": "text" - }, - "href": { - "type": "text" } } } diff --git a/openmetadata-service/src/main/resources/dataInsights/opensearch/indexMappingsTemplate.json b/openmetadata-service/src/main/resources/dataInsights/opensearch/indexMappingsTemplate.json index f8fdd93ff83c..19da7fc9120d 100644 --- a/openmetadata-service/src/main/resources/dataInsights/opensearch/indexMappingsTemplate.json +++ b/openmetadata-service/src/main/resources/dataInsights/opensearch/indexMappingsTemplate.json @@ -1,138 +1,16 @@ { "template": { - "settings": { - "analysis": { - "normalizer": { - "lowercase_normalizer": { - "type": "custom", - "char_filter": [], - "filter": [ - "lowercase" - ] - } - }, - "analyzer": { - "om_analyzer": { - "tokenizer": "letter", - "filter": [ - "lowercase", - "om_stemmer" - ] - }, - "om_ngram": { - "tokenizer": "ngram", - "min_gram": 3, - "max_gram": 10, - "filter": [ - "lowercase" - ] - } - }, - "filter": { - "om_stemmer": { - "type": "stemmer", - "name": "english" - } - } - } - }, + "settings": {}, "mappings": { "properties": { "@timestamp": { "type": "date" }, - "owners": { - "properties": { - "id": { - "type": "keyword", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 36 - } - } - }, - "type": { - "type": "keyword" - }, - "name": { - "type": "keyword", - "normalizer": "lowercase_normalizer", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "displayName": { - "type": "keyword", - "fields": { - "keyword": { - "type": "keyword", - "normalizer": "lowercase_normalizer", - "ignore_above": 256 - } - } - }, - "fullyQualifiedName": { - "type": "text" - }, - "description": { - "type": "text" - }, - "deleted": { - "type": "text" - }, - "href": { - "type": "text" - } - } - }, - "domain": { - "properties": { - "id": { - "type": "keyword", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 36 - } - } - }, - "type": { - "type": "keyword" - }, - "name": { - "type": "keyword", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "displayName": { - "type": "keyword", - "fields": { - "keyword": { - "type": "keyword", - "normalizer": "lowercase_normalizer", - "ignore_above": 256 - } - } - }, - "fullyQualifiedName": { + "id": { + "type": "text", + "fields": { + "keyword": { "type": "keyword" - }, - "description": { - "type": "text" - }, - "deleted": { - "type": "text" - }, - "href": { - "type": "text" } } } diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/api_collection_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/api_collection_index_mapping.json index 4412d0b9d4d0..7d2375914a46 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/en/api_collection_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/en/api_collection_index_mapping.json @@ -152,7 +152,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/api_endpoint_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/api_endpoint_index_mapping.json index 4bad64a563d6..9c3780bde690 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/en/api_endpoint_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/en/api_endpoint_index_mapping.json @@ -236,7 +236,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/api_service_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/api_service_index_mapping.json index e5b166b75d25..09899efe95af 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/en/api_service_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/en/api_service_index_mapping.json @@ -244,7 +244,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/chart_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/chart_index_mapping.json index 4c07d3a82d4c..55563789deab 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/en/chart_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/en/chart_index_mapping.json @@ -323,7 +323,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/container_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/container_index_mapping.json index 9377b8829522..485b08cbe143 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/en/container_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/en/container_index_mapping.json @@ -245,7 +245,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/dashboard_data_model_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/dashboard_data_model_index_mapping.json index cf14f2f82545..02407bd997cc 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/en/dashboard_data_model_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/en/dashboard_data_model_index_mapping.json @@ -129,7 +129,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/dashboard_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/dashboard_index_mapping.json index 531775d00ba4..b37992167d33 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/en/dashboard_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/en/dashboard_index_mapping.json @@ -301,7 +301,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/dashboard_service_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/dashboard_service_index_mapping.json index 8a95a19bfd8b..5650e456a594 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/en/dashboard_service_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/en/dashboard_service_index_mapping.json @@ -243,7 +243,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/database_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/database_index_mapping.json index 6410fcce44bc..dea0e25dae19 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/en/database_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/en/database_index_mapping.json @@ -152,7 +152,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/database_schema_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/database_schema_index_mapping.json index 55cd7e04de18..8975b0caaf02 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/en/database_schema_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/en/database_schema_index_mapping.json @@ -129,7 +129,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/database_service_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/database_service_index_mapping.json index a9a06a28a622..5af7c4fac354 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/en/database_service_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/en/database_service_index_mapping.json @@ -244,7 +244,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/messaging_service_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/messaging_service_index_mapping.json index ac26a4c5697c..93d8f59b8bb2 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/en/messaging_service_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/en/messaging_service_index_mapping.json @@ -243,7 +243,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/mlmodel_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/mlmodel_index_mapping.json index 884ce04be35e..1649ed831452 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/en/mlmodel_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/en/mlmodel_index_mapping.json @@ -145,7 +145,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/mlmodel_service_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/mlmodel_service_index_mapping.json index 1799a171bb40..44cbb4e2df55 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/en/mlmodel_service_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/en/mlmodel_service_index_mapping.json @@ -244,7 +244,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/pipeline_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/pipeline_index_mapping.json index 89aa68e4d2e7..3c0cd989c72c 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/en/pipeline_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/en/pipeline_index_mapping.json @@ -128,7 +128,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" @@ -204,6 +204,9 @@ } } }, + "startDate": { + "type": "text" + }, "tasks": { "properties": { "name": { @@ -234,6 +237,12 @@ }, "taskType": { "type": "text" + }, + "startDate": { + "type": "text" + }, + "endDate": { + "type": "text" } } }, diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/pipeline_service_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/pipeline_service_index_mapping.json index d52eed9436e2..7d26039dad1b 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/en/pipeline_service_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/en/pipeline_service_index_mapping.json @@ -242,7 +242,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/search_entity_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/search_entity_index_mapping.json index f18800d7d4a7..bfb708cabf75 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/en/search_entity_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/en/search_entity_index_mapping.json @@ -438,7 +438,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/search_service_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/search_service_index_mapping.json index e5b166b75d25..09899efe95af 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/en/search_service_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/en/search_service_index_mapping.json @@ -244,7 +244,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/storage_service_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/storage_service_index_mapping.json index e8b9c1087b8e..daa0f2c24838 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/en/storage_service_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/en/storage_service_index_mapping.json @@ -239,7 +239,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/stored_procedure_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/stored_procedure_index_mapping.json index e5381631d395..8a949cac7586 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/en/stored_procedure_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/en/stored_procedure_index_mapping.json @@ -139,7 +139,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/table_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/table_index_mapping.json index 7603a5aad1e5..6894c8b18378 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/en/table_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/en/table_index_mapping.json @@ -451,7 +451,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/topic_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/topic_index_mapping.json index fdbd9874f035..104716d4cfbf 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/en/topic_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/en/topic_index_mapping.json @@ -192,7 +192,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/indexMapping.json b/openmetadata-service/src/main/resources/elasticsearch/indexMapping.json index 20a17107791e..a5b348e7452e 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/indexMapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/indexMapping.json @@ -150,8 +150,8 @@ "indexName": "tag_search_index", "indexMappingFile": "/elasticsearch/%s/tag_index_mapping.json", "alias": "tag", - "parentAliases": ["classification"], - "childAliases": ["all", "dataAsset"] + "parentAliases": ["classification", "all", "dataAsset"], + "childAliases": [] }, "classification": { "indexName": "classification_search_index", diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/api_collection_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/api_collection_index_mapping.json index 963232c61a5c..5f372f0cb654 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/jp/api_collection_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/jp/api_collection_index_mapping.json @@ -201,7 +201,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/api_endpoint_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/api_endpoint_index_mapping.json index 9615b637a06d..6cea22a83d07 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/jp/api_endpoint_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/jp/api_endpoint_index_mapping.json @@ -228,7 +228,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/api_service_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/api_service_index_mapping.json index b129fee387e2..78f6bfae7ed6 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/jp/api_service_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/jp/api_service_index_mapping.json @@ -244,7 +244,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/chart_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/chart_index_mapping.json index 63dc9bf7427a..da41c7a885bf 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/jp/chart_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/jp/chart_index_mapping.json @@ -348,7 +348,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/container_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/container_index_mapping.json index cbc2c0fb9c7a..0ec9e3caf092 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/jp/container_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/jp/container_index_mapping.json @@ -180,7 +180,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/dashboard_data_model_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/dashboard_data_model_index_mapping.json index ac97cf2da7a7..16eab0e04b1e 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/jp/dashboard_data_model_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/jp/dashboard_data_model_index_mapping.json @@ -130,7 +130,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/dashboard_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/dashboard_index_mapping.json index 34220063f91c..6dd71d84ccac 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/jp/dashboard_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/jp/dashboard_index_mapping.json @@ -182,7 +182,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/dashboard_service_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/dashboard_service_index_mapping.json index 7580ed0e585d..b37d23450dea 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/jp/dashboard_service_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/jp/dashboard_service_index_mapping.json @@ -234,7 +234,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/database_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/database_index_mapping.json index 2162b019c7cd..2b8625f71d0c 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/jp/database_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/jp/database_index_mapping.json @@ -201,7 +201,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/database_schema_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/database_schema_index_mapping.json index e3d7d75c1822..f54ea768c1f0 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/jp/database_schema_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/jp/database_schema_index_mapping.json @@ -128,7 +128,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/database_service_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/database_service_index_mapping.json index c5342182cba8..e52673edca0b 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/jp/database_service_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/jp/database_service_index_mapping.json @@ -244,7 +244,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/messaging_service_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/messaging_service_index_mapping.json index a431db25f83d..8555dddbb1f0 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/jp/messaging_service_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/jp/messaging_service_index_mapping.json @@ -239,7 +239,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/mlmodel_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/mlmodel_index_mapping.json index 3c43e7f576f3..0bb981a02a95 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/jp/mlmodel_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/jp/mlmodel_index_mapping.json @@ -137,7 +137,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/mlmodel_service_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/mlmodel_service_index_mapping.json index 8d8950f0e819..a626e291ad11 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/jp/mlmodel_service_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/jp/mlmodel_service_index_mapping.json @@ -244,7 +244,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/pipeline_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/pipeline_index_mapping.json index 719fbb0c696b..0a88086a9349 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/jp/pipeline_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/jp/pipeline_index_mapping.json @@ -116,7 +116,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/pipeline_service_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/pipeline_service_index_mapping.json index 465c23b92fd5..62f40905cf1b 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/jp/pipeline_service_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/jp/pipeline_service_index_mapping.json @@ -234,7 +234,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/search_entity_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/search_entity_index_mapping.json index 3d05a850134c..585f67fce420 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/jp/search_entity_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/jp/search_entity_index_mapping.json @@ -435,7 +435,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/search_service_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/search_service_index_mapping.json index b129fee387e2..78f6bfae7ed6 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/jp/search_service_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/jp/search_service_index_mapping.json @@ -244,7 +244,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/storage_service_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/storage_service_index_mapping.json index c108d4be5700..a8e480a1a7bd 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/jp/storage_service_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/jp/storage_service_index_mapping.json @@ -231,7 +231,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/stored_procedure_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/stored_procedure_index_mapping.json index f7a2c585e4a5..74dc0afdec82 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/jp/stored_procedure_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/jp/stored_procedure_index_mapping.json @@ -131,7 +131,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/table_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/table_index_mapping.json index 0e40e44ba165..91053db24430 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/jp/table_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/jp/table_index_mapping.json @@ -446,7 +446,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/topic_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/topic_index_mapping.json index cc5d423d3333..75e3a1114068 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/jp/topic_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/jp/topic_index_mapping.json @@ -362,7 +362,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/api_collection_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/api_collection_index_mapping.json index 315e65ca7832..401aac8101e9 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/zh/api_collection_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/zh/api_collection_index_mapping.json @@ -140,7 +140,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/api_endpoint_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/api_endpoint_index_mapping.json index 5562831c0941..08c71c439f26 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/zh/api_endpoint_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/zh/api_endpoint_index_mapping.json @@ -226,7 +226,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/api_service_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/api_service_index_mapping.json index 92f798d34c7f..9bf5d4443d15 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/zh/api_service_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/zh/api_service_index_mapping.json @@ -228,7 +228,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/chart_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/chart_index_mapping.json index dc6e1c89ef7d..abea73a8058b 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/zh/chart_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/zh/chart_index_mapping.json @@ -344,7 +344,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/container_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/container_index_mapping.json index 8747dffa51ca..b8456805e43e 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/zh/container_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/zh/container_index_mapping.json @@ -230,7 +230,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/dashboard_data_model_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/dashboard_data_model_index_mapping.json index 6fc67b807ec4..df4f72f7648f 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/zh/dashboard_data_model_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/zh/dashboard_data_model_index_mapping.json @@ -117,7 +117,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/dashboard_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/dashboard_index_mapping.json index f44807f639b1..1c55d305b163 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/zh/dashboard_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/zh/dashboard_index_mapping.json @@ -236,7 +236,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/dashboard_service_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/dashboard_service_index_mapping.json index f0e44beb8d42..7fb238b3e1a6 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/zh/dashboard_service_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/zh/dashboard_service_index_mapping.json @@ -224,7 +224,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/database_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/database_index_mapping.json index 5aeb73fc12a5..df775a54b6b4 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/zh/database_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/zh/database_index_mapping.json @@ -140,7 +140,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/database_schema_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/database_schema_index_mapping.json index 1cb5cafc1255..23f7e115e2a3 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/zh/database_schema_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/zh/database_schema_index_mapping.json @@ -117,7 +117,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/database_service_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/database_service_index_mapping.json index ac4cae0dada7..d7e999223144 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/zh/database_service_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/zh/database_service_index_mapping.json @@ -228,7 +228,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/messaging_service_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/messaging_service_index_mapping.json index 5905c5c4586f..3fc002e1ab9b 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/zh/messaging_service_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/zh/messaging_service_index_mapping.json @@ -226,7 +226,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/mlmodel_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/mlmodel_index_mapping.json index cafb914e9b0d..3fd2a901c7a4 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/zh/mlmodel_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/zh/mlmodel_index_mapping.json @@ -131,7 +131,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/mlmodel_service_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/mlmodel_service_index_mapping.json index 8034e3b1affa..315c7e72fd67 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/zh/mlmodel_service_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/zh/mlmodel_service_index_mapping.json @@ -228,7 +228,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/pipeline_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/pipeline_index_mapping.json index d345bdb6c572..76629230e642 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/zh/pipeline_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/zh/pipeline_index_mapping.json @@ -117,7 +117,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/pipeline_service_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/pipeline_service_index_mapping.json index 30e6757560ae..db4e34b91892 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/zh/pipeline_service_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/zh/pipeline_service_index_mapping.json @@ -224,7 +224,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/search_entity_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/search_entity_index_mapping.json index 385a51afbf0a..fef942a90656 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/zh/search_entity_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/zh/search_entity_index_mapping.json @@ -426,7 +426,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/search_service_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/search_service_index_mapping.json index 92f798d34c7f..9bf5d4443d15 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/zh/search_service_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/zh/search_service_index_mapping.json @@ -228,7 +228,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/storage_service_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/storage_service_index_mapping.json index f6e08d84a619..7c278c1abbf0 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/zh/storage_service_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/zh/storage_service_index_mapping.json @@ -221,7 +221,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/stored_procedure_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/stored_procedure_index_mapping.json index 05962f5a3a01..b658476dbfb0 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/zh/stored_procedure_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/zh/stored_procedure_index_mapping.json @@ -127,7 +127,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/table_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/table_index_mapping.json index a60e851a0516..2aa220c986b8 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/zh/table_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/zh/table_index_mapping.json @@ -442,7 +442,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/topic_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/topic_index_mapping.json index a562e51e4f88..eea4513713e0 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/zh/topic_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/zh/topic_index_mapping.json @@ -180,7 +180,7 @@ } }, "fullyQualifiedName": { - "type": "text" + "type": "keyword" }, "description": { "type": "text" diff --git a/openmetadata-service/src/main/resources/json/data/app/DataRetentionApplication.json b/openmetadata-service/src/main/resources/json/data/app/DataRetentionApplication.json new file mode 100644 index 000000000000..c5e14f24f4e7 --- /dev/null +++ b/openmetadata-service/src/main/resources/json/data/app/DataRetentionApplication.json @@ -0,0 +1,11 @@ +{ + "name": "DataRetentionApplication", + "displayName": "Data Retention", + "appConfiguration": { + "changeEventRetentionPeriod": 7 + }, + "appSchedule": { + "scheduleTimeline": "Custom", + "cronExpression": "0 0 * * 0" + } +} \ No newline at end of file diff --git a/openmetadata-service/src/main/resources/json/data/app/SearchIndexingApplication.json b/openmetadata-service/src/main/resources/json/data/app/SearchIndexingApplication.json index 015d8f6ede2a..debe210dba62 100644 --- a/openmetadata-service/src/main/resources/json/data/app/SearchIndexingApplication.json +++ b/openmetadata-service/src/main/resources/json/data/app/SearchIndexingApplication.json @@ -3,47 +3,7 @@ "displayName": "Search Indexing", "appConfiguration": { "entities": [ - "table", - "dashboard", - "topic", - "pipeline", - "ingestionPipeline", - "searchIndex", - "user", - "team", - "glossary", - "glossaryTerm", - "mlmodel", - "tag", - "classification", - "query", - "container", - "database", - "databaseSchema", - "testCase", - "testSuite", - "chart", - "dashboardDataModel", - "databaseService", - "messagingService", - "dashboardService", - "pipelineService", - "mlmodelService", - "storageService", - "metadataService", - "searchService", - "entityReportData", - "webAnalyticEntityViewReportData", - "webAnalyticUserActivityReportData", - "domain", - "storedProcedure", - "dataProduct", - "testCaseResolutionStatus", - "testCaseResult", - "apiService", - "apiEndpoint", - "apiCollection", - "metric" + "all" ], "recreateIndex": false, "batchSize": "100", diff --git a/openmetadata-service/src/main/resources/json/data/appMarketPlaceDefinition/DataRetentionApplication.json b/openmetadata-service/src/main/resources/json/data/appMarketPlaceDefinition/DataRetentionApplication.json new file mode 100644 index 000000000000..78ff3a06f876 --- /dev/null +++ b/openmetadata-service/src/main/resources/json/data/appMarketPlaceDefinition/DataRetentionApplication.json @@ -0,0 +1,18 @@ +{ + "name": "DataRetentionApplication", + "displayName": "Data Retention", + "description": "The Data Retention App automates the cleanup of OpenMetadata's internal database to maintain performance and efficiency. Based on customizable retention periods and admin-defined retention policies, this application helps manage rapidly growing tables and prevents data bloat by removing outdated records.\n\n**Efficient Record Management:** The Retention Policy App automates the cleanup and deletion of records for selected entities based on specified retention durations. It simplifies compliance with data retention policies and ensures efficient database management.", + "appType": "internal", + "appScreenshots": ["DataRetentionApplication.png"], + "developer": "Collate Inc.", + "developerUrl": "https://www.getcollate.io", + "privacyPolicyUrl": "https://www.getcollate.io", + "supportEmail": "support@getcollate.io", + "scheduleType": "ScheduledOrManual", + "permission": "All", + "className": "org.openmetadata.service.apps.bundles.dataRetention.DataRetention", + "runtime": { + "enabled": "true" + }, + "appConfiguration": {} +} \ No newline at end of file diff --git a/openmetadata-service/src/main/resources/json/data/appMarketPlaceDefinition/SearchIndexingApplication.json b/openmetadata-service/src/main/resources/json/data/appMarketPlaceDefinition/SearchIndexingApplication.json index 7ec557701485..5f323da477b0 100644 --- a/openmetadata-service/src/main/resources/json/data/appMarketPlaceDefinition/SearchIndexingApplication.json +++ b/openmetadata-service/src/main/resources/json/data/appMarketPlaceDefinition/SearchIndexingApplication.json @@ -18,47 +18,7 @@ "supportsInterrupt": true, "appConfiguration": { "entities": [ - "table", - "dashboard", - "topic", - "pipeline", - "ingestionPipeline", - "searchIndex", - "user", - "team", - "glossary", - "glossaryTerm", - "mlmodel", - "tag", - "classification", - "query", - "container", - "database", - "databaseSchema", - "testCase", - "testSuite", - "chart", - "dashboardDataModel", - "databaseService", - "messagingService", - "dashboardService", - "pipelineService", - "mlmodelService", - "storageService", - "metadataService", - "searchService", - "entityReportData", - "webAnalyticEntityViewReportData", - "webAnalyticUserActivityReportData", - "domain", - "storedProcedure", - "dataProduct", - "testCaseResolutionStatus", - "testCaseResult", - "apiService", - "apiEndpoint", - "apiCollection", - "metric" + "all" ], "recreateIndex": false, "batchSize": "100", diff --git a/openmetadata-service/src/main/resources/json/data/eventsubscription/ActivityFeedEvents.json b/openmetadata-service/src/main/resources/json/data/eventsubscription/ActivityFeedEvents.json index ecdaeba28b41..4db1f4b296bf 100644 --- a/openmetadata-service/src/main/resources/json/data/eventsubscription/ActivityFeedEvents.json +++ b/openmetadata-service/src/main/resources/json/data/eventsubscription/ActivityFeedEvents.json @@ -10,7 +10,7 @@ { "name": "matchAnyEventType", "effect": "include", - "condition": "matchAnyEventType({'entityCreated', 'entityDeleted', 'entitySoftDeleted'}) && !matchUpdatedBy({'ingestion-bot'}) && !matchAnySource({'user', 'team', 'owners', 'databaseService', 'messagingService', 'dashboardService', 'pipelineService', 'storageService', 'mlmodelService', 'metadataService', 'searchService', 'apiService', 'ingestionPipeline', 'workflow'})", + "condition": "matchAnyEventType({'entityCreated', 'entityDeleted', 'entitySoftDeleted'}) && !matchUpdatedBy({'ingestion-bot'}) && !matchAnySource({'user', 'team', 'owners', 'databaseService', 'messagingService', 'dashboardService', 'pipelineService', 'storageService', 'mlmodelService', 'metadataService', 'searchService', 'apiService', 'ingestionPipeline', 'workflow', 'testCase', 'testSuite'})", "prefixCondition": "AND" }, { diff --git a/openmetadata-service/src/main/resources/json/data/governance/workflows/GlossaryApprovalWorkflow.json b/openmetadata-service/src/main/resources/json/data/governance/workflows/GlossaryApprovalWorkflow.json index 84a6d293d769..fa83628f3a18 100644 --- a/openmetadata-service/src/main/resources/json/data/governance/workflows/GlossaryApprovalWorkflow.json +++ b/openmetadata-service/src/main/resources/json/data/governance/workflows/GlossaryApprovalWorkflow.json @@ -25,6 +25,12 @@ "name": "ApprovedEnd", "displayName": "Glossary Term Status: Approved" }, + { + "type": "endEvent", + "subType": "endEvent", + "name": "ApprovedEndAfterUserTask", + "displayName": "Glossary Term Status: Approved" + }, { "type": "endEvent", "subType": "endEvent", @@ -93,6 +99,15 @@ "glossaryTermStatus": "Approved" } }, + { + "type": "automatedTask", + "subType": "setGlossaryTermStatusTask", + "name": "SetGlossaryTermStatusToApprovedAfterUserTask", + "displayName": "Set Status to 'Approved'", + "config": { + "glossaryTermStatus": "Approved" + } + }, { "type": "automatedTask", "subType": "setGlossaryTermStatusTask", @@ -138,7 +153,7 @@ }, { "from": "ApproveGlossaryTerm", - "to": "SetGlossaryTermStatusToApproved", + "to": "SetGlossaryTermStatusToApprovedAfterUserTask", "condition": true }, { @@ -150,6 +165,10 @@ "from": "SetGlossaryTermStatusToApproved", "to": "ApprovedEnd" }, + { + "from": "SetGlossaryTermStatusToApprovedAfterUserTask", + "to": "ApprovedEndAfterUserTask" + }, { "from": "SetGlossaryTermStatusToRejected", "to": "RejectedEnd" diff --git a/openmetadata-service/src/main/resources/json/data/policy/DomainAccessPolicy.json b/openmetadata-service/src/main/resources/json/data/policy/DomainAccessPolicy.json index 572760b5ef01..d103fff85265 100644 --- a/openmetadata-service/src/main/resources/json/data/policy/DomainAccessPolicy.json +++ b/openmetadata-service/src/main/resources/json/data/policy/DomainAccessPolicy.json @@ -4,8 +4,8 @@ "fullyQualifiedName": "DomainOnlyAccessPolicy", "description": "This Policy adds restrictions so that users will have access to domain related data. If the user has some domain, then he will be able to access data only for that domain. If the user does not have any domain assigned , he will be able to access only assets which also does not have any domain.", "enabled": true, - "allowDelete": false, - "provider": "system", + "allowDelete": true, + "provider": "user", "rules": [ { "name": "DomainOnlyAccessRule", diff --git a/openmetadata-service/src/main/resources/json/data/role/DomainOnlyAccessRole.json b/openmetadata-service/src/main/resources/json/data/role/DomainOnlyAccessRole.json index b18aeae18424..ec770210e4da 100644 --- a/openmetadata-service/src/main/resources/json/data/role/DomainOnlyAccessRole.json +++ b/openmetadata-service/src/main/resources/json/data/role/DomainOnlyAccessRole.json @@ -2,8 +2,8 @@ "name": "DomainOnlyAccessRole", "displayName": "Domain Only Access Role", "description": "Role Corresponding to Domain Access Restriction.", - "allowDelete": false, - "provider": "system", + "allowDelete": true, + "provider": "user", "policies" : [ { "type" : "policy", diff --git a/openmetadata-service/src/test/java/org/openmetadata/jobs/BackgroundJobWorkerTest.java b/openmetadata-service/src/test/java/org/openmetadata/jobs/BackgroundJobWorkerTest.java new file mode 100644 index 000000000000..50afd197845c --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/jobs/BackgroundJobWorkerTest.java @@ -0,0 +1,207 @@ +package org.openmetadata.jobs; + +import static javax.ws.rs.core.Response.Status.OK; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.openmetadata.service.security.SecurityUtil.getPrincipalName; +import static org.openmetadata.service.util.TestUtils.ADMIN_AUTH_HEADERS; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.security.SecureRandom; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.openmetadata.schema.api.data.CreateTable; +import org.openmetadata.schema.entity.Type; +import org.openmetadata.schema.entity.data.Table; +import org.openmetadata.schema.entity.type.CustomProperty; +import org.openmetadata.schema.jobs.BackgroundJob; +import org.openmetadata.schema.jobs.EnumCleanupArgs; +import org.openmetadata.schema.type.CustomPropertyConfig; +import org.openmetadata.service.Entity; +import org.openmetadata.service.OpenMetadataApplicationTest; +import org.openmetadata.service.jdbi3.CollectionDAO; +import org.openmetadata.service.jobs.BackgroundJobException; +import org.openmetadata.service.jobs.EnumCleanupHandler; +import org.openmetadata.service.jobs.JobDAO; +import org.openmetadata.service.jobs.JobHandler; +import org.openmetadata.service.jobs.JobHandlerRegistry; +import org.openmetadata.service.resources.databases.TableResourceTest; +import org.openmetadata.service.resources.metadata.TypeResourceTest; +import org.openmetadata.service.util.JsonUtils; + +@Slf4j +public class BackgroundJobWorkerTest extends OpenMetadataApplicationTest { + + static JobHandlerRegistry registry; + static JobDAO jobDAO; + static CollectionDAO collectionDao; + static EnumCleanupHandler enumCleanupHandler; + public static TableResourceTest TABLE_RESOURCE_TEST; + public static TypeResourceTest TYPE_RESOURCE_TEST; + + public static CustomProperty customPropertyMulti; + public static CustomProperty customPropertySingle; + + public static Table TABLE4; + + @BeforeAll + public static void setup(TestInfo test) throws IOException, URISyntaxException { + registry = new JobHandlerRegistry(); + jobDAO = Entity.getJobDAO(); + collectionDao = Entity.getCollectionDAO(); + enumCleanupHandler = new EnumCleanupHandler(Entity.getCollectionDAO()); + LOG.info("Registering EnumCleanupHandler {}", enumCleanupHandler); + registry.register("EnumCleanupHandler", enumCleanupHandler); + TABLE_RESOURCE_TEST = new TableResourceTest(); + TABLE_RESOURCE_TEST.setup(test); + TYPE_RESOURCE_TEST = new TypeResourceTest(); + Type enumType = TYPE_RESOURCE_TEST.getEntityByName("enum", "", ADMIN_AUTH_HEADERS); + Type entityType = + TYPE_RESOURCE_TEST.getEntityByName(Entity.TABLE, "customProperties", ADMIN_AUTH_HEADERS); + + customPropertySingle = + new CustomProperty() + .withName("tableEnumCpSingle") + .withDescription("enum type custom property with multiselect = false") + .withPropertyType(enumType.getEntityReference()) + .withCustomPropertyConfig( + new CustomPropertyConfig() + .withConfig( + Map.of( + "values", + List.of("single1", "single2", "single3", "single4", "\"single5\""), + "multiSelect", + false))); + + customPropertyMulti = + new CustomProperty() + .withName("tableEnumCpMulti") + .withDescription("enum type custom property with multiselect = true") + .withPropertyType(enumType.getEntityReference()) + .withCustomPropertyConfig( + new CustomPropertyConfig() + .withConfig( + Map.of( + "values", + List.of("multi1", "multi2", "multi3", "multi4", "\"multi5\""), + "multiSelect", + true))); + CustomProperty[] customProperties = {customPropertySingle, customPropertyMulti}; + for (CustomProperty customProperty : customProperties) { + TYPE_RESOURCE_TEST.addAndCheckCustomProperty( + entityType.getId(), customProperty, OK, ADMIN_AUTH_HEADERS); + } + + CreateTable createTable4 = TABLE_RESOURCE_TEST.createRequest(test); + createTable4.withName("table4"); + TABLE4 = TABLE_RESOURCE_TEST.createAndCheckEntity(createTable4, ADMIN_AUTH_HEADERS); + } + + private BackgroundJob createBackgroundJob(EnumCleanupArgs enumCleanupArgs) { + BackgroundJob job = new BackgroundJob(); + job.setId(new SecureRandom().nextLong()); + job.setJobType(BackgroundJob.JobType.CUSTOM_PROPERTY_ENUM_CLEANUP); + job.setJobArgs(JsonUtils.pojoToJson(enumCleanupArgs)); + job.setCreatedBy(getPrincipalName(ADMIN_AUTH_HEADERS)); + job.setMethodName("EnumCleanupHandler"); + return job; + } + + @Test + public final void testRegisterEventHandler() { + EnumCleanupArgs enumCleanupArgs = + new EnumCleanupArgs() + .withPropertyName("name") + .withRemovedEnumKeys(List.of()) + .withEntityType("type"); + + BackgroundJob job = createBackgroundJob(enumCleanupArgs); + + // Verify that the handler is registered + JobHandler handler = registry.getHandler(job); + assertNotNull(handler); + assertInstanceOf(EnumCleanupHandler.class, handler); + + // Verify that an exception is thrown for a non-existent handler + job.setMethodName("NonExistentHandler"); + Exception exception = + assertThrows(BackgroundJobException.class, () -> registry.getHandler(job)); + assertEquals("No handler registered for NonExistentHandler", exception.getMessage()); + } + + @Test + public final void testBackgroundJobTriggerWithInvalidArgs() { + BackgroundJob job = new BackgroundJob(); + job.setId(new SecureRandom().nextLong()); + job.setJobArgs("invalidArgs"); + job.setMethodName("EnumCleanupHandler"); + + BackgroundJobException exception = + assertThrows(BackgroundJobException.class, () -> enumCleanupHandler.runJob(job)); + assertEquals( + "Failed to run EnumCleanupHandler job. Error:Invalid arguments invalidArgs", + exception.getMessage()); + } + + @Test + public final void testBackgroundJobTriggerWithUnrecognizedField() { + EnumCleanupArgs enumCleanupArgs = + new EnumCleanupArgs() + .withPropertyName("name") + .withRemovedEnumKeys(List.of()) + .withEntityType("type"); + + BackgroundJob job = createBackgroundJob(enumCleanupArgs); + job.setJobArgs( + "{\"bucket\":\"value\",\"propertyName\":\"name\",\"removedEnumKeys\":[],\"entityType\":\"type\"}"); + + BackgroundJobException exception = + assertThrows(BackgroundJobException.class, () -> enumCleanupHandler.runJob(job)); + assertEquals( + "Failed to run EnumCleanupHandler job. Error:Invalid arguments {\"bucket\":\"value\",\"propertyName\":\"name\",\"removedEnumKeys\":[],\"entityType\":\"type\"}", + exception.getMessage()); + } + + @Test + public final void testBackgroundJobTriggerWithValidArgs() { + + EnumCleanupArgs enumCleanupArgs = + new EnumCleanupArgs() + .withPropertyName(customPropertyMulti.getName()) + .withRemovedEnumKeys(List.of()) + .withEntityType("table"); + + BackgroundJob job = createBackgroundJob(enumCleanupArgs); + String jobArgs = JsonUtils.pojoToJson(enumCleanupArgs); + + long jobId = + Entity.getJobDAO() + .insertJob( + job.getJobType(), + new EnumCleanupHandler(collectionDao), + jobArgs, + job.getCreatedBy()); + Optional fetchedJobOptional = Entity.getJobDAO().fetchJobById(jobId); + assertTrue(fetchedJobOptional.isPresent(), "Job should be present"); + + BackgroundJob fetchedJob = fetchedJobOptional.get(); + String fetchedJobArgs = JsonUtils.pojoToJson(fetchedJob.getJobArgs()); + + // Assert the fetched job details + assertEquals(job.getJobType(), fetchedJob.getJobType(), "Job type should match"); + assertEquals(job.getMethodName(), fetchedJob.getMethodName(), "Method name should match"); + + EnumCleanupArgs actualArgs = JsonUtils.readValue(fetchedJobArgs, EnumCleanupArgs.class); + assertEquals(enumCleanupArgs, actualArgs, "Job arguments should match"); + assertEquals(job.getCreatedBy(), fetchedJob.getCreatedBy(), "Created by should match"); + } +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/OpenMetadataApplicationTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/OpenMetadataApplicationTest.java index 87c4e54903aa..35fe934705fa 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/OpenMetadataApplicationTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/OpenMetadataApplicationTest.java @@ -54,6 +54,7 @@ import org.openmetadata.service.jdbi3.CollectionDAO; import org.openmetadata.service.jdbi3.locator.ConnectionAwareAnnotationSqlLocator; import org.openmetadata.service.jdbi3.locator.ConnectionType; +import org.openmetadata.service.jobs.JobDAO; import org.openmetadata.service.migration.api.MigrationWorkflow; import org.openmetadata.service.resources.CollectionRegistry; import org.openmetadata.service.resources.databases.DatasourceConfig; @@ -240,6 +241,7 @@ public static void validateAndRunSystemDataMigrations( SearchRepository searchRepository = new SearchRepository(getEsConfig()); Entity.setSearchRepository(searchRepository); Entity.setCollectionDAO(jdbi.onDemand(CollectionDAO.class)); + Entity.setJobDAO(jdbi.onDemand(JobDAO.class)); Entity.initializeRepositories(config, jdbi); workflow.loadMigrations(); workflow.runMigrationWorkflows(); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/ChangeEventParserResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/ChangeEventParserResourceTest.java index 29bb0247b19e..36ee3fc80adb 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/ChangeEventParserResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/ChangeEventParserResourceTest.java @@ -141,7 +141,7 @@ void testEntityReferenceFormat() { assertEquals(1, threadWithMessages.size()); assertEquals( - "Added **owners**: User One", + "Added **owners**: User One", threadWithMessages.get(0).getMessage()); } @@ -156,8 +156,8 @@ void testUpdateOfString() { assertEquals(1, threadMessages.size()); assertEquals( - "Updated **description**: old " - + "new description", + "Updated **description**: old " + + "new description", threadMessages.get(0).getMessage()); // test if it updates correctly with one add and one delete change @@ -225,7 +225,7 @@ void testMajorSchemaChange() { assertEquals(1, threadWithMessages.size()); assertEquals( - "Updated **columns**: lo_order priority", + "Updated **columns**: lo_order priority", threadWithMessages.get(0).getMessage()); // Simulate a change of datatype change in column @@ -261,7 +261,7 @@ void testMajorSchemaChange() { assertEquals(1, threadWithMessages.size()); assertEquals( - "Updated **columns**: lo_orderpriority , newColumn", + "Updated **columns**: lo_orderpriority , newColumn", threadWithMessages.get(0).getMessage()); } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java index c00e20069def..202abac41da5 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java @@ -2801,8 +2801,7 @@ protected final WebTarget getFollowerResource(UUID id, UUID userId) { return getFollowersCollection(id).path("/" + userId); } - protected final T getEntity(UUID id, Map authHeaders) - throws HttpResponseException { + public final T getEntity(UUID id, Map authHeaders) throws HttpResponseException { WebTarget target = getResource(id); target = target.queryParam("fields", allFields); return TestUtils.get(target, entityClass, authHeaders); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/apps/AppsResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/apps/AppsResourceTest.java index e14532595c78..69fe5eaa3c53 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/apps/AppsResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/apps/AppsResourceTest.java @@ -303,7 +303,7 @@ void validate_data_insights_workflow_is_correct_for_a_simple_case() Request request = new Request("GET", "di-data-assets-*/_search"); String payload = String.format( - "{\"query\":{\"bool\":{\"must\":{\"term\":{\"fullyQualifiedName.keyword\":\"%s\"}}}}}", + "{\"query\":{\"bool\":{\"must\":{\"term\":{\"fullyQualifiedName\":\"%s\"}}}}}", table.getFullyQualifiedName()); request.setJsonEntity(payload); response = searchClient.performRequest(request); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java index a6c51064a08f..9d12cdb0b9d0 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java @@ -2225,7 +2225,7 @@ void test_ownershipInheritance(TestInfo test) throws IOException { CreateTestSuite createTestSuite = testSuiteResourceTest.createRequest(table.getFullyQualifiedName()); TestSuite testSuite = - testSuiteResourceTest.createExecutableTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); + testSuiteResourceTest.createBasicTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); CreateTestCase createTestCase = testCaseResourceTest @@ -2293,7 +2293,7 @@ void test_domainInheritance(TestInfo test) throws IOException { CreateTestSuite createTestSuite = testSuiteResourceTest.createRequest(table.getFullyQualifiedName()); TestSuite testSuite = - testSuiteResourceTest.createExecutableTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); + testSuiteResourceTest.createBasicTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); CreateTestCase createTestCase = testCaseResourceTest @@ -2433,8 +2433,7 @@ void get_tablesWithTestCases(TestInfo test) throws IOException { CreateTestSuite createExecutableTestSuite = testSuiteResourceTest.createRequest(table1.getFullyQualifiedName()); TestSuite executableTestSuite = - testSuiteResourceTest.createExecutableTestSuite( - createExecutableTestSuite, ADMIN_AUTH_HEADERS); + testSuiteResourceTest.createBasicTestSuite(createExecutableTestSuite, ADMIN_AUTH_HEADERS); HashMap queryParams = new HashMap<>(); queryParams.put("includeEmptyTestSuite", "false"); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/docstore/DocStoreResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/docstore/DocStoreResourceTest.java index c138bfc1a4a3..6315fc6df753 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/docstore/DocStoreResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/docstore/DocStoreResourceTest.java @@ -291,7 +291,7 @@ void post_validKnowledgePanels_as_admin_200_OK(TestInfo test) throws IOException page = new Page() - .withPageType(PageType.GLOSSARY_TERM) + .withPageType(PageType.GLOSSARY_TERM_LANDING_PAGE) .withKnowledgePanels( List.of(activityFeed.getEntityReference(), myData.getEntityReference())); fqn = diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/domains/DataProductResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/domains/DataProductResourceTest.java index 4069fbc5a291..a83a5bf224c3 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/domains/DataProductResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/domains/DataProductResourceTest.java @@ -23,8 +23,10 @@ import org.junit.jupiter.api.TestInfo; import org.openmetadata.schema.EntityInterface; import org.openmetadata.schema.api.domains.CreateDataProduct; +import org.openmetadata.schema.api.domains.CreateDomain; import org.openmetadata.schema.entity.data.Topic; import org.openmetadata.schema.entity.domains.DataProduct; +import org.openmetadata.schema.entity.domains.Domain; import org.openmetadata.schema.entity.type.Style; import org.openmetadata.schema.type.ChangeDescription; import org.openmetadata.schema.type.EntityReference; @@ -177,6 +179,74 @@ void testValidateDataProducts() { .hasMessage(String.format("dataProduct instance for %s not found", rdnUUID)); } + @Test + void test_inheritOwnerExpertsFromDomain(TestInfo test) throws IOException { + DomainResourceTest domainResourceTest = new DomainResourceTest(); + + // Create parent domain + CreateDomain parentDomainReq = + domainResourceTest + .createRequest(test, 1) + .withOwners(List.of(USER1_REF)) + .withExperts(List.of(USER2.getFullyQualifiedName())); + Domain parentDomain = domainResourceTest.createEntity(parentDomainReq, ADMIN_AUTH_HEADERS); + parentDomain = domainResourceTest.getEntity(parentDomain.getId(), "*", ADMIN_AUTH_HEADERS); + + // Create data product corresponding to parent domain + CreateDataProduct create = + createRequestWithoutExpertsOwners(getEntityName(test, 1)) + .withDomain(parentDomain.getFullyQualifiedName()); + DataProduct dataProduct = createAndCheckEntity(create, ADMIN_AUTH_HEADERS); + assertOwners(dataProduct.getOwners(), parentDomain.getOwners()); + assertEntityReferences(dataProduct.getExperts(), parentDomain.getExperts()); + + // Create subdomain with no owners and experts + CreateDomain subDomainReq = + domainResourceTest + .createRequestWithoutOwnersExperts(getEntityName(test, 2)) + .withDomain(parentDomain.getFullyQualifiedName()); + Domain subDomain = domainResourceTest.createEntity(subDomainReq, ADMIN_AUTH_HEADERS); + subDomain = domainResourceTest.getEntity(subDomain.getId(), "*", ADMIN_AUTH_HEADERS); + + // Create data product corresponding to subdomain + CreateDataProduct subDomainDataProductCreate = + createRequestWithoutExpertsOwners(getEntityName(test, 2)) + .withDomain(subDomain.getFullyQualifiedName()); + DataProduct subDomainDataProduct = + createAndCheckEntity(subDomainDataProductCreate, ADMIN_AUTH_HEADERS); + + // Subdomain and its data product should inherit owners and experts from parent domain + assertOwners(subDomain.getOwners(), parentDomain.getOwners()); + assertEntityReferences(subDomain.getExperts(), parentDomain.getExperts()); + assertOwners(subDomainDataProduct.getOwners(), parentDomain.getOwners()); + assertEntityReferences(subDomainDataProduct.getExperts(), parentDomain.getExperts()); + + // Add owner and expert to subdomain + Domain updateSubDomainOwner = + JsonUtils.readValue(JsonUtils.pojoToJson(subDomain), Domain.class); + updateSubDomainOwner.setOwners(List.of(TEAM11_REF)); + domainResourceTest.patchEntity( + subDomain.getId(), + JsonUtils.pojoToJson(subDomain), + updateSubDomainOwner, + ADMIN_AUTH_HEADERS); + subDomain = domainResourceTest.getEntity(subDomain.getId(), "*", ADMIN_AUTH_HEADERS); + + Domain updateSubDomainExpert = + JsonUtils.readValue(JsonUtils.pojoToJson(subDomain), Domain.class); + updateSubDomainExpert.setExperts(List.of(USER1_REF)); + domainResourceTest.patchEntity( + subDomain.getId(), + JsonUtils.pojoToJson(subDomain), + updateSubDomainExpert, + ADMIN_AUTH_HEADERS); + subDomain = domainResourceTest.getEntity(subDomain.getId(), "*", ADMIN_AUTH_HEADERS); + + // Data product of subdomain should also have the same changes as its corresponding domain + assertOwners(subDomainDataProduct.getOwners(), subDomain.getOwners()); + assertEntityReferences(subDomainDataProduct.getExperts(), subDomain.getExperts()); + } + private void entityInDataProduct( EntityInterface entity, EntityInterface product, boolean inDataProduct) throws HttpResponseException { @@ -200,6 +270,15 @@ public CreateDataProduct createRequest(String name) { .withAssets(TEST_TABLE1 != null ? listOf(TEST_TABLE1.getEntityReference()) : null); } + public CreateDataProduct createRequestWithoutExpertsOwners(String name) { + return new CreateDataProduct() + .withName(name) + .withDescription(name) + .withDomain(DOMAIN.getFullyQualifiedName()) + .withStyle(new Style().withColor("#40E0D0").withIconURL("https://dataProductIcon")) + .withAssets(TEST_TABLE1 != null ? listOf(TEST_TABLE1.getEntityReference()) : null); + } + @Override public void validateCreatedEntity( DataProduct createdEntity, CreateDataProduct request, Map authHeaders) { diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/domains/DomainResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/domains/DomainResourceTest.java index 7054353f2f6c..178ca066eb98 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/domains/DomainResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/domains/DomainResourceTest.java @@ -3,6 +3,8 @@ import static javax.ws.rs.core.Response.Status.NOT_FOUND; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.openmetadata.common.utils.CommonUtil.listOf; import static org.openmetadata.service.Entity.TABLE; import static org.openmetadata.service.security.SecurityUtil.authHeaders; @@ -17,15 +19,18 @@ import static org.openmetadata.service.util.TestUtils.assertResponse; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.UUID; +import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.Response.Status; import org.apache.http.client.HttpResponseException; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; import org.openmetadata.schema.api.domains.CreateDomain; import org.openmetadata.schema.api.domains.CreateDomain.DomainType; +import org.openmetadata.schema.entity.data.EntityHierarchy; import org.openmetadata.schema.entity.domains.Domain; import org.openmetadata.schema.entity.type.Style; import org.openmetadata.schema.type.ChangeDescription; @@ -35,7 +40,9 @@ import org.openmetadata.service.jdbi3.TableRepository; import org.openmetadata.service.resources.EntityResourceTest; import org.openmetadata.service.resources.domains.DomainResource.DomainList; +import org.openmetadata.service.util.EntityHierarchyList; import org.openmetadata.service.util.JsonUtils; +import org.openmetadata.service.util.TestUtils; public class DomainResourceTest extends EntityResourceTest { public DomainResourceTest() { @@ -143,6 +150,214 @@ void patchWrongExperts(TestInfo test) throws IOException { String.format("user instance for %s not found", expertReference.getId())); } + @Test + void test_buildDomainNestedHierarchyFromSearch() throws HttpResponseException { + CreateDomain createRootDomain = + createRequest("rootDomain") + .withDisplayName("Global Headquarters") + .withDescription("Root Domain") + .withDomainType(DomainType.AGGREGATE) + .withStyle(null) + .withExperts(null); + Domain rootDomain = createEntity(createRootDomain, ADMIN_AUTH_HEADERS); + + CreateDomain createSecondLevelDomain = + createRequest("secondLevelDomain") + .withDisplayName("Operations Hub") + .withDescription("Second Level Domain") + .withDomainType(DomainType.AGGREGATE) + .withStyle(null) + .withExperts(null) + .withParent(rootDomain.getFullyQualifiedName()); + Domain secondLevelDomain = createEntity(createSecondLevelDomain, ADMIN_AUTH_HEADERS); + + CreateDomain createThirdLevelDomain = + createRequest("thirdLevelDomain") + .withDisplayName("Innovation Center") + .withDescription("Third Level Domain") + .withDomainType(DomainType.AGGREGATE) + .withStyle(null) + .withExperts(null) + .withParent(secondLevelDomain.getFullyQualifiedName()); + Domain thirdLevelDomain = createEntity(createThirdLevelDomain, ADMIN_AUTH_HEADERS); + + // Search for the displayName of third-level child domain and verify the hierarchy + String response = getResponseFormSearchWithHierarchy("domain_search_index", "*innovation*"); + List domains = JsonUtils.readObjects(response, EntityHierarchy.class); + + boolean isChild = + domains.stream() + .filter(domain -> "rootDomain".equals(domain.getName())) + .findFirst() + .map( + root -> + root.getChildren().stream() + .filter(domain -> "secondLevelDomain".equals(domain.getName())) + .flatMap(secondLevel -> secondLevel.getChildren().stream()) + .anyMatch(thirdLevel -> "thirdLevelDomain".equals(thirdLevel.getName()))) + .orElse(false); + + assertTrue( + isChild, + "thirdLevelDomain should be a child of secondLevelDomain, which should be a child of rootDomain"); + + // Search for the fqn of third-level child domain and verify the hierarchy + response = getResponseFormSearchWithHierarchy("domain_search_index", "*third*"); + domains = JsonUtils.readObjects(response, EntityHierarchy.class); + + isChild = + domains.stream() + .filter(domain -> "rootDomain".equals(domain.getName())) + .findFirst() + .map( + root -> + root.getChildren().stream() + .filter(domain -> "secondLevelDomain".equals(domain.getName())) + .flatMap(secondLevel -> secondLevel.getChildren().stream()) + .anyMatch(thirdLevel -> "thirdLevelDomain".equals(thirdLevel.getName()))) + .orElse(false); + + assertTrue( + isChild, + "thirdLevelDomain should be a child of secondLevelDomain, which should be a child of rootDomain"); + } + + @Test + void get_hierarchicalListOfDomain(TestInfo test) throws HttpResponseException { + Domain rootDomain = createEntity(createRequest("A_ROOT_DOMAIN"), ADMIN_AUTH_HEADERS); + Domain subDomain1 = + createEntity( + createRequest("A_subDomain1").withParent(rootDomain.getFullyQualifiedName()), + ADMIN_AUTH_HEADERS); + Domain subDomain2 = + createEntity( + createRequest("A_subDomain2").withParent(rootDomain.getFullyQualifiedName()), + ADMIN_AUTH_HEADERS); + Domain subDomain3 = + createEntity( + createRequest("A_subDomain3").withParent(rootDomain.getFullyQualifiedName()), + ADMIN_AUTH_HEADERS); + + // Ensure parent has all the newly created children + rootDomain = getEntity(rootDomain.getId(), "children,parent", ADMIN_AUTH_HEADERS); + assertEntityReferences( + new ArrayList<>( + List.of( + subDomain1.getEntityReference(), + subDomain2.getEntityReference(), + subDomain3.getEntityReference())), + rootDomain.getChildren()); + + Domain subSubDomain1 = + createEntity( + createRequest("A_subSubDomain11").withParent(subDomain1.getFullyQualifiedName()), + ADMIN_AUTH_HEADERS); + Domain subSubDomain2 = + createEntity( + createRequest("A_subSubDomain12").withParent(subDomain1.getFullyQualifiedName()), + ADMIN_AUTH_HEADERS); + Domain subSubDomain3 = + createEntity( + createRequest("A_subSubDomain13").withParent(subDomain1.getFullyQualifiedName()), + ADMIN_AUTH_HEADERS); + + // Ensure parent has all the newly created children + subDomain1 = getEntity(subDomain1.getId(), "children,parent", ADMIN_AUTH_HEADERS); + assertEntityReferences( + new ArrayList<>( + List.of( + subSubDomain1.getEntityReference(), + subSubDomain2.getEntityReference(), + subSubDomain3.getEntityReference())), + subDomain1.getChildren()); + assertParent(subSubDomain1, subDomain1.getEntityReference()); + assertParent(subSubDomain2, subDomain1.getEntityReference()); + assertParent(subSubDomain3, subDomain1.getEntityReference()); + + // Create another root domain without hierarchy + Domain secondRootDomain = createEntity(createRequest("B_ROOT_DOMAIN"), ADMIN_AUTH_HEADERS); + + List hierarchyList = getDomainsHierarchy(ADMIN_AUTH_HEADERS).getData(); + + UUID rootDomainId = rootDomain.getId(); + UUID subDomain1Id = subDomain1.getId(); + UUID subDomain2Id = subDomain2.getId(); + UUID subDomain3Id = subDomain3.getId(); + UUID subSubDomain1Id = subSubDomain1.getId(); + UUID subSubDomain2Id = subSubDomain2.getId(); + UUID subSubDomain3Id = subSubDomain3.getId(); + UUID secondRootDomainId = secondRootDomain.getId(); + + EntityHierarchy rootHierarchy = + hierarchyList.stream().filter(h -> h.getId().equals(rootDomainId)).findAny().orElse(null); + assertNotNull(rootHierarchy); + assertEquals(3, rootHierarchy.getChildren().size()); + + List rootChildren = rootHierarchy.getChildren(); + assertEquals(3, rootChildren.size()); + assertTrue(rootChildren.stream().anyMatch(h -> h.getId().equals(subDomain1Id))); + assertTrue(rootChildren.stream().anyMatch(h -> h.getId().equals(subDomain2Id))); + assertTrue(rootChildren.stream().anyMatch(h -> h.getId().equals(subDomain3Id))); + + EntityHierarchy subDomain1Hierarchy = + rootChildren.stream().filter(h -> h.getId().equals(subDomain1Id)).findAny().orElse(null); + assertNotNull(subDomain1Hierarchy); + assertEquals(3, subDomain1Hierarchy.getChildren().size()); + + List subDomain1Children = subDomain1Hierarchy.getChildren(); + assertTrue(subDomain1Children.stream().anyMatch(h -> h.getId().equals(subSubDomain1Id))); + assertTrue(subDomain1Children.stream().anyMatch(h -> h.getId().equals(subSubDomain2Id))); + assertTrue(subDomain1Children.stream().anyMatch(h -> h.getId().equals(subSubDomain3Id))); + + EntityHierarchy subSubDomain1Hierarchy = + subDomain1Children.stream() + .filter(h -> h.getId().equals(subSubDomain1Id)) + .findAny() + .orElse(null); + assertNotNull(subSubDomain1Hierarchy); + assertEquals(0, subSubDomain1Hierarchy.getChildren().size()); + + EntityHierarchy subSubDomain2Hierarchy = + subDomain1Children.stream() + .filter(h -> h.getId().equals(subSubDomain2Id)) + .findAny() + .orElse(null); + assertNotNull(subSubDomain2Hierarchy); + assertEquals(0, subSubDomain2Hierarchy.getChildren().size()); + + EntityHierarchy subSubDomain3Hierarchy = + subDomain1Children.stream() + .filter(h -> h.getId().equals(subSubDomain3Id)) + .findAny() + .orElse(null); + assertNotNull(subSubDomain3Hierarchy); + assertEquals(0, subSubDomain3Hierarchy.getChildren().size()); + + // Verify the new root domain without hierarchy + EntityHierarchy secondRootDomainHierarchy = + hierarchyList.stream() + .filter(h -> h.getId().equals(secondRootDomainId)) + .findAny() + .orElse(null); + assertNotNull(secondRootDomainHierarchy); + assertEquals(0, secondRootDomainHierarchy.getChildren().size()); + } + + private void assertParent(Domain domain, EntityReference expectedParent) + throws HttpResponseException { + assertEquals(expectedParent, domain.getParent()); + // Ensure the parent has the given domain as a child + Domain parent = getEntity(expectedParent.getId(), "children", ADMIN_AUTH_HEADERS); + assertEntityReferencesContain(parent.getChildren(), domain.getEntityReference()); + } + + private EntityHierarchyList getDomainsHierarchy(Map authHeaders) + throws HttpResponseException { + WebTarget target = getResource("domains/hierarchy"); + target = target.queryParam("limit", 25); + return TestUtils.get(target, EntityHierarchyList.class, authHeaders); + } + @Override public CreateDomain createRequest(String name) { return new CreateDomain() @@ -153,6 +368,14 @@ public CreateDomain createRequest(String name) { .withExperts(listOf(USER1.getFullyQualifiedName())); } + public CreateDomain createRequestWithoutOwnersExperts(String name) { + return new CreateDomain() + .withName(name) + .withDomainType(DomainType.AGGREGATE) + .withDescription("name") + .withStyle(new Style().withColor("#FFA07A").withIconURL("https://domainIcon")); + } + @Override public void validateCreatedEntity( Domain createdEntity, CreateDomain request, Map authHeaders) { diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java index d3cf7b2675c2..0b8a2c3d4a6f 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java @@ -248,7 +248,7 @@ void post_testWithInvalidEntityTestSuite_4xx(TestInfo test) throws IOException { CreateTestSuite createTestSuite = testSuiteResourceTest.createRequest(test).withName(TEST_TABLE1.getFullyQualifiedName()); TestSuite testSuite = - testSuiteResourceTest.createExecutableTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); + testSuiteResourceTest.createBasicTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); create.withEntityLink(INVALID_LINK1).withTestSuite(testSuite.getFullyQualifiedName()); assertResponseContains( @@ -585,7 +585,7 @@ void get_listTestCasesFromSearchWithPagination(TestInfo testInfo) CreateTestSuite createTestSuite = testSuiteResourceTest.createRequest(table.getFullyQualifiedName()); TestSuite testSuite = - testSuiteResourceTest.createExecutableTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); + testSuiteResourceTest.createBasicTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); testSuites.put(table.getFullyQualifiedName(), testSuite); } @@ -676,7 +676,7 @@ void test_getSimpleListFromSearch(TestInfo testInfo) throws IOException, ParseEx CreateTestSuite createTestSuite = testSuiteResourceTest.createRequest(table.getFullyQualifiedName()); TestSuite testSuite = - testSuiteResourceTest.createExecutableTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); + testSuiteResourceTest.createBasicTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); testSuites.put(table.getFullyQualifiedName(), testSuite); } @@ -899,7 +899,7 @@ void test_testCaseInheritedFields(TestInfo testInfo) throws IOException { CreateTestSuite createTestSuite = testSuiteResourceTest.createRequest(table.getFullyQualifiedName()); TestSuite testSuite = - testSuiteResourceTest.createExecutableTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); + testSuiteResourceTest.createBasicTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); CreateTestCase create = createRequest(testInfo) @@ -1133,7 +1133,7 @@ void list_allTestSuitesFromTestCase_200(TestInfo test) throws IOException { CreateTestSuite createTestSuite = testSuiteResourceTest.createRequest(test).withName(TEST_TABLE2.getFullyQualifiedName()); TestSuite executableTestSuite = - testSuiteResourceTest.createExecutableTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); + testSuiteResourceTest.createBasicTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); // Create the test cases (need to be created against an executable test suite) CreateTestCase create = @@ -2171,8 +2171,7 @@ void test_testCaseResultState(TestInfo test) throws IOException, ParseException .withDescription(test.getDisplayName()) .withEntityLink( String.format( - "<#E::table::%s>", - testSuite.getExecutableEntityReference().getFullyQualifiedName())) + "<#E::table::%s>", testSuite.getBasicEntityReference().getFullyQualifiedName())) .withTestSuite(testSuite.getFullyQualifiedName()) .withTestDefinition(TEST_DEFINITION1.getFullyQualifiedName()); TestCase testCase = createAndCheckEntity(createTestCase, ADMIN_AUTH_HEADERS); @@ -2634,7 +2633,7 @@ private TestSuite createExecutableTestSuite(TestInfo test) throws IOException { Table table = tableResourceTest.createAndCheckEntity(tableReq, ADMIN_AUTH_HEADERS); CreateTestSuite createExecutableTestSuite = testSuiteResourceTest.createRequest(table.getFullyQualifiedName()); - return testSuiteResourceTest.createExecutableTestSuite( + return testSuiteResourceTest.createBasicTestSuite( createExecutableTestSuite, ADMIN_AUTH_HEADERS); } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestSuiteResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestSuiteResourceTest.java index ceb09aedca82..ec167504406d 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestSuiteResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestSuiteResourceTest.java @@ -35,6 +35,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; import org.openmetadata.schema.api.data.CreateTable; +import org.openmetadata.schema.api.services.ingestionPipelines.CreateIngestionPipeline; import org.openmetadata.schema.api.teams.CreateTeam; import org.openmetadata.schema.api.teams.CreateUser; import org.openmetadata.schema.api.tests.CreateLogicalTestCases; @@ -42,8 +43,11 @@ import org.openmetadata.schema.api.tests.CreateTestCaseResult; import org.openmetadata.schema.api.tests.CreateTestSuite; import org.openmetadata.schema.entity.data.Table; +import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline; import org.openmetadata.schema.entity.teams.Team; import org.openmetadata.schema.entity.teams.User; +import org.openmetadata.schema.metadataIngestion.SourceConfig; +import org.openmetadata.schema.metadataIngestion.TestSuitePipeline; import org.openmetadata.schema.tests.TestCase; import org.openmetadata.schema.tests.TestSuite; import org.openmetadata.schema.tests.type.TestCaseStatus; @@ -55,6 +59,7 @@ import org.openmetadata.service.Entity; import org.openmetadata.service.resources.EntityResourceTest; import org.openmetadata.service.resources.databases.TableResourceTest; +import org.openmetadata.service.resources.services.ingestionpipelines.IngestionPipelineResourceTest; import org.openmetadata.service.resources.teams.TeamResourceTest; import org.openmetadata.service.resources.teams.UserResourceTest; import org.openmetadata.service.search.models.IndexMapping; @@ -113,10 +118,10 @@ public void setupTestSuites(TestInfo test) throws IOException { TEST_SUITE_TABLE2 = tableResourceTest.createAndCheckEntity(tableReq, ADMIN_AUTH_HEADERS); CREATE_TEST_SUITE1 = createRequest(DATABASE_SCHEMA.getFullyQualifiedName() + "." + TEST_SUITE_TABLE_NAME1); - TEST_SUITE1 = createExecutableTestSuite(CREATE_TEST_SUITE1, ADMIN_AUTH_HEADERS); + TEST_SUITE1 = createBasicTestSuite(CREATE_TEST_SUITE1, ADMIN_AUTH_HEADERS); CREATE_TEST_SUITE2 = createRequest(DATABASE_SCHEMA.getFullyQualifiedName() + "." + TEST_SUITE_TABLE_NAME2); - TEST_SUITE2 = createExecutableTestSuite(CREATE_TEST_SUITE2, ADMIN_AUTH_HEADERS); + TEST_SUITE2 = createBasicTestSuite(CREATE_TEST_SUITE2, ADMIN_AUTH_HEADERS); } @Test @@ -169,7 +174,7 @@ void put_testCaseResults_200() throws IOException, ParseException { verifyTestCases(testSuite.getTests(), testCases1); } } - deleteExecutableTestSuite(TEST_SUITE1.getId(), true, false, ADMIN_AUTH_HEADERS); + deleteBasicTestSuite(TEST_SUITE1.getId(), true, false, ADMIN_AUTH_HEADERS); assertResponse( () -> getEntity(TEST_SUITE1.getId(), ADMIN_AUTH_HEADERS), NOT_FOUND, @@ -182,6 +187,15 @@ void put_testCaseResults_200() throws IOException, ParseException { assertEquals(true, deletedTestSuite.getDeleted()); } + @Test + void create_basicTestSuiteWithoutRef(TestInfo test) { + CreateTestSuite createTestSuite = createRequest(test); + assertResponse( + () -> createBasicEmptySuite(createTestSuite, ADMIN_AUTH_HEADERS), + BAD_REQUEST, + "Cannot create a basic test suite without the BasicEntityReference field informed."); + } + @Test void list_testSuitesIncludeEmpty_200(TestInfo test) throws IOException { List testSuites = new ArrayList<>(); @@ -200,7 +214,7 @@ void list_testSuitesIncludeEmpty_200(TestInfo test) throws IOException { .withDataLength(10))); Table table = tableResourceTest.createEntity(tableReq, ADMIN_AUTH_HEADERS); CreateTestSuite createTestSuite = createRequest(table.getFullyQualifiedName()); - TestSuite testSuite = createExecutableTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); + TestSuite testSuite = createBasicTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); for (int j = 0; j < 3; j++) { CreateTestCase createTestCase = testCaseResourceTest @@ -224,7 +238,7 @@ void list_testSuitesIncludeEmpty_200(TestInfo test) throws IOException { .withDataLength(10))); Table table = tableResourceTest.createEntity(tableReq, ADMIN_AUTH_HEADERS); CreateTestSuite createTestSuite = createRequest(table.getFullyQualifiedName()); - createExecutableTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); + createBasicTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); testSuites.add(createTestSuite); ResultList actualTestSuites = @@ -276,7 +290,7 @@ void test_inheritOwnerFromTable(TestInfo test) throws IOException { table = tableResourceTest.getEntity(table.getId(), "*", ADMIN_AUTH_HEADERS); CreateTestSuite createExecutableTestSuite = createRequest(table.getFullyQualifiedName()); TestSuite executableTestSuite = - createExecutableTestSuite(createExecutableTestSuite, ADMIN_AUTH_HEADERS); + createBasicTestSuite(createExecutableTestSuite, ADMIN_AUTH_HEADERS); TestSuite testSuite = getEntity(executableTestSuite.getId(), "*", ADMIN_AUTH_HEADERS); assertOwners(testSuite.getOwners(), table.getOwners()); Table updateTableOwner = table; @@ -306,7 +320,7 @@ void test_inheritDomainFromTable(TestInfo test) throws IOException { table = tableResourceTest.getEntity(table.getId(), "*", ADMIN_AUTH_HEADERS); CreateTestSuite createExecutableTestSuite = createRequest(table.getFullyQualifiedName()); TestSuite executableTestSuite = - createExecutableTestSuite(createExecutableTestSuite, ADMIN_AUTH_HEADERS); + createBasicTestSuite(createExecutableTestSuite, ADMIN_AUTH_HEADERS); TestSuite testSuite = getEntity(executableTestSuite.getId(), "domain", ADMIN_AUTH_HEADERS); assertEquals(DOMAIN1.getId(), testSuite.getDomain().getId()); ResultList testSuites = @@ -349,7 +363,7 @@ void post_createLogicalTestSuiteAndAddTests_200(TestInfo test) throws IOExceptio CreateTestSuite createExecutableTestSuite = createRequest(table.getFullyQualifiedName()); createExecutableTestSuite.withOwners(List.of(user1Ref)); TestSuite executableTestSuite = - createExecutableTestSuite(createExecutableTestSuite, ADMIN_AUTH_HEADERS); + createBasicTestSuite(createExecutableTestSuite, ADMIN_AUTH_HEADERS); List testCases1 = new ArrayList<>(); // We'll create tests cases for testSuite1 @@ -399,7 +413,7 @@ void post_createLogicalTestSuiteAndAddTests_200(TestInfo test) throws IOExceptio allEntities.getData().stream() .anyMatch(ts -> ts.getId().equals(finalExecutableTestSuite.getId()))); // 2. List only executable test suites - queryParams.put("testSuiteType", "executable"); + queryParams.put("testSuiteType", "basic"); queryParams.put("fields", "tests"); ResultList executableTestSuites = listEntitiesFromSearch(queryParams, 100, 0, ADMIN_AUTH_HEADERS); @@ -489,7 +503,7 @@ void addTestCaseWithLogicalEndPoint(TestInfo test) throws IOException { .withDataLength(10))); Table table = tableResourceTest.createEntity(tableReq, ADMIN_AUTH_HEADERS); CreateTestSuite createTestSuite = createRequest(table.getFullyQualifiedName()); - TestSuite testSuite = createExecutableTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); + TestSuite testSuite = createBasicTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); List testCases1 = new ArrayList<>(); // We'll create tests cases for testSuite1 @@ -511,14 +525,14 @@ void addTestCaseWithLogicalEndPoint(TestInfo test) throws IOException { executableTestSuite, testCases1.stream().map(EntityReference::getId).collect(Collectors.toList())), BAD_REQUEST, - "You are trying to add test cases to an executable test suite."); + "You are trying to add test cases to a basic test suite."); } @Test void post_createExecTestSuiteNonExistingEntity_400(TestInfo test) { CreateTestSuite createTestSuite = createRequest(test); assertResponse( - () -> createExecutableTestSuite(createTestSuite, ADMIN_AUTH_HEADERS), + () -> createBasicTestSuite(createTestSuite, ADMIN_AUTH_HEADERS), NOT_FOUND, String.format("table instance for %s not found", createTestSuite.getName())); } @@ -540,7 +554,7 @@ void get_execTestSuiteFromTable_200(TestInfo test) throws IOException { .withDataLength(10))); Table table = tableResourceTest.createEntity(tableReq, ADMIN_AUTH_HEADERS); CreateTestSuite createTestSuite = createRequest(table.getFullyQualifiedName()); - TestSuite testSuite = createExecutableTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); + TestSuite testSuite = createBasicTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); // We'll create tests cases for testSuite for (int i = 0; i < 5; i++) { @@ -563,7 +577,7 @@ void get_execTestSuiteFromTable_200(TestInfo test) throws IOException { assertEquals(5, tableTestSuite.getTests().size()); // Soft delete entity - deleteExecutableTestSuite(tableTestSuite.getId(), true, false, ADMIN_AUTH_HEADERS); + deleteBasicTestSuite(tableTestSuite.getId(), true, false, ADMIN_AUTH_HEADERS); actualTable = tableResourceTest.getEntity( actualTable.getId(), queryParams, "testSuite", ADMIN_AUTH_HEADERS); @@ -574,7 +588,7 @@ void get_execTestSuiteFromTable_200(TestInfo test) throws IOException { assertEquals(true, tableTestSuite.getDeleted()); // Hard delete entity - deleteExecutableTestSuite(tableTestSuite.getId(), true, true, ADMIN_AUTH_HEADERS); + deleteBasicTestSuite(tableTestSuite.getId(), true, true, ADMIN_AUTH_HEADERS); actualTable = tableResourceTest.getEntity(table.getId(), "testSuite", ADMIN_AUTH_HEADERS); assertNull(actualTable.getTestSuite()); } @@ -596,7 +610,7 @@ void get_execTestSuiteDeletedOnTableDeletion(TestInfo test) throws IOException { .withDataLength(10))); Table table = tableResourceTest.createEntity(tableReq, ADMIN_AUTH_HEADERS); CreateTestSuite createTestSuite = createRequest(table.getFullyQualifiedName()); - TestSuite testSuite = createExecutableTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); + TestSuite testSuite = createBasicTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); HashMap queryParams = new HashMap<>(); queryParams.put("include", Include.ALL.value()); @@ -635,13 +649,13 @@ void get_filterTestSuiteType_200(TestInfo test) throws IOException { ResultList testSuiteResultList = listEntities(queryParams, ADMIN_AUTH_HEADERS); assertEquals(10, testSuiteResultList.getData().size()); - queryParams.put("testSuiteType", "executable"); + queryParams.put("testSuiteType", "basic"); testSuiteResultList = listEntities(queryParams, ADMIN_AUTH_HEADERS); - testSuiteResultList.getData().forEach(ts -> assertEquals(true, ts.getExecutable())); + testSuiteResultList.getData().forEach(ts -> assertEquals(true, ts.getBasic())); queryParams.put("testSuiteType", "logical"); testSuiteResultList = listEntities(queryParams, ADMIN_AUTH_HEADERS); - testSuiteResultList.getData().forEach(ts -> assertEquals(false, ts.getExecutable())); + testSuiteResultList.getData().forEach(ts -> assertEquals(false, ts.getBasic())); queryParams.put("includeEmptyTestSuites", "false"); testSuiteResultList = listEntities(queryParams, ADMIN_AUTH_HEADERS); @@ -691,7 +705,7 @@ void delete_LogicalTestSuite_200(TestInfo test) throws IOException { Table table = tableResourceTest.createEntity(tableReq, ADMIN_AUTH_HEADERS); CreateTestSuite createExecutableTestSuite = createRequest(table.getFullyQualifiedName()); TestSuite executableTestSuite = - createExecutableTestSuite(createExecutableTestSuite, ADMIN_AUTH_HEADERS); + createBasicTestSuite(createExecutableTestSuite, ADMIN_AUTH_HEADERS); List testCases = new ArrayList<>(); // We'll create tests cases for testSuite1 @@ -720,6 +734,80 @@ void delete_LogicalTestSuite_200(TestInfo test) throws IOException { assertEquals(5, actualExecutableTestSuite.getTests().size()); } + @Test + void delete_logicalSuiteWithPipeline(TestInfo test) throws IOException { + TestCaseResourceTest testCaseResourceTest = new TestCaseResourceTest(); + TableResourceTest tableResourceTest = new TableResourceTest(); + CreateTable tableReq = + tableResourceTest + .createRequest(test) + .withColumns( + List.of( + new Column() + .withName(C1) + .withDisplayName("c1") + .withDataType(ColumnDataType.VARCHAR) + .withDataLength(10))); + Table table = tableResourceTest.createEntity(tableReq, ADMIN_AUTH_HEADERS); + CreateTestSuite createExecutableTestSuite = createRequest(table.getFullyQualifiedName()); + TestSuite executableTestSuite = + createBasicTestSuite(createExecutableTestSuite, ADMIN_AUTH_HEADERS); + List testCases1 = new ArrayList<>(); + + // We'll create tests cases for testSuite1 + for (int i = 0; i < 5; i++) { + CreateTestCase createTestCase = + testCaseResourceTest + .createRequest(String.format("test_testSuite_2_%s_", test.getDisplayName()) + i) + .withTestSuite(executableTestSuite.getFullyQualifiedName()); + TestCase testCase = + testCaseResourceTest.createAndCheckEntity(createTestCase, ADMIN_AUTH_HEADERS); + testCases1.add(testCase.getEntityReference()); + } + + // We'll create a logical test suite and associate the test cases to it + CreateTestSuite createTestSuite = createRequest(test); + TestSuite testSuite = createEntity(createTestSuite, ADMIN_AUTH_HEADERS); + addTestCasesToLogicalTestSuite( + testSuite, testCases1.stream().map(EntityReference::getId).collect(Collectors.toList())); + TestSuite logicalTestSuite = getEntity(testSuite.getId(), "*", ADMIN_AUTH_HEADERS); + + // Add ingestion pipeline to the database service + IngestionPipelineResourceTest ingestionPipelineResourceTest = + new IngestionPipelineResourceTest(); + CreateIngestionPipeline createIngestionPipeline = + ingestionPipelineResourceTest + .createRequest(test) + .withService(logicalTestSuite.getEntityReference()); + + TestSuitePipeline testSuitePipeline = new TestSuitePipeline(); + + SourceConfig sourceConfig = new SourceConfig().withConfig(testSuitePipeline); + createIngestionPipeline.withSourceConfig(sourceConfig); + IngestionPipeline ingestionPipeline = + ingestionPipelineResourceTest.createEntity(createIngestionPipeline, ADMIN_AUTH_HEADERS); + + // We can GET the Ingestion Pipeline now + IngestionPipeline actualIngestionPipeline = + ingestionPipelineResourceTest.getEntity(ingestionPipeline.getId(), ADMIN_AUTH_HEADERS); + assertNotNull(actualIngestionPipeline); + + // After deleting the test suite, we can't GET the Ingestion Pipeline + deleteEntity(logicalTestSuite.getId(), true, true, ADMIN_AUTH_HEADERS); + + assertResponse( + () -> + ingestionPipelineResourceTest.getEntity(ingestionPipeline.getId(), ADMIN_AUTH_HEADERS), + NOT_FOUND, + String.format( + "ingestionPipeline instance for %s not found", actualIngestionPipeline.getId())); + + // Test Cases are still there + TestCase testCaseInLogical = + testCaseResourceTest.getEntity(testCases1.get(0).getId(), "*", ADMIN_AUTH_HEADERS); + assertNotNull(testCaseInLogical); + } + @Test void get_listTestSuiteFromSearchWithPagination(TestInfo testInfo) throws IOException { if (supportsSearchIndex) { @@ -750,7 +838,7 @@ void get_listTestSuiteFromSearchWithPagination(TestInfo testInfo) throws IOExcep CreateTestSuite createTestSuite = testSuiteResourceTest.createRequest(table.getFullyQualifiedName()); TestSuite testSuite = - testSuiteResourceTest.createExecutableTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); + testSuiteResourceTest.createBasicTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); testSuites.put(table.getFullyQualifiedName(), testSuite); } validateEntityListFromSearchWithPagination(new HashMap<>(), testSuites.size()); @@ -772,7 +860,7 @@ void create_executableTestSuiteAndCheckSearchClient(TestInfo test) throws IOExce .withDataLength(10))); Table table = tableResourceTest.createEntity(tableReq, ADMIN_AUTH_HEADERS); CreateTestSuite createTestSuite = createRequest(table.getFullyQualifiedName()); - TestSuite testSuite = createExecutableTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); + TestSuite testSuite = createBasicTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); RestClient searchClient = getSearchClient(); IndexMapping index = Entity.getSearchRepository().getIndexMapping(Entity.TABLE); es.org.elasticsearch.client.Response response; @@ -827,10 +915,16 @@ public ResultList getTestSuites( return TestUtils.get(target, TestSuiteResource.TestSuiteList.class, authHeaders); } - public TestSuite createExecutableTestSuite( + public TestSuite createBasicTestSuite( + CreateTestSuite createTestSuite, Map authHeaders) throws IOException { + WebTarget target = getResource("dataQuality/testSuites/basic"); + createTestSuite.setBasicEntityReference(createTestSuite.getName()); + return TestUtils.post(target, createTestSuite, TestSuite.class, authHeaders); + } + + public TestSuite createBasicEmptySuite( CreateTestSuite createTestSuite, Map authHeaders) throws IOException { - WebTarget target = getResource("dataQuality/testSuites/executable"); - createTestSuite.setExecutableEntityReference(createTestSuite.getName()); + WebTarget target = getResource("dataQuality/testSuites/basic"); return TestUtils.post(target, createTestSuite, TestSuite.class, authHeaders); } @@ -844,11 +938,10 @@ public void addTestCasesToLogicalTestSuite(TestSuite testSuite, List testC TestUtils.put(target, createLogicalTestCases, Response.Status.OK, ADMIN_AUTH_HEADERS); } - public void deleteExecutableTestSuite( + public void deleteBasicTestSuite( UUID id, boolean recursive, boolean hardDelete, Map authHeaders) throws IOException { - WebTarget target = - getResource(String.format("dataQuality/testSuites/executable/%s", id.toString())); + WebTarget target = getResource(String.format("dataQuality/testSuites/basic/%s", id.toString())); target = recursive ? target.queryParam("recursive", true) : target; target = hardDelete ? target.queryParam("hardDelete", true) : target; TestUtils.delete(target, TestSuite.class, authHeaders); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java index 2f5e79a12402..431daa668ae2 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java @@ -870,7 +870,12 @@ void testGlossaryImportExport() throws IOException { .withConfig( Map.of( "values", - List.of("single1", "single2", "single3", "single4", "\"single5\""), + List.of( + "\"single val with quotes\"", + "single1", + "single2", + "single3", + "single4"), "multiSelect", false))), new CustomProperty() @@ -903,28 +908,28 @@ void testGlossaryImportExport() throws IOException { } // CSV Header "parent", "name", "displayName", "description", "synonyms", "relatedTerms", // "references", - // "tags", "reviewers", "owners", "status", "extension" + // "tags", "reviewers", "owner", "glossaryStatus", "extension" // Create two records List createRecords = listOf( String.format( - ",g1,dsp1,\"dsc1,1\",h1;h2;h3,,term1;http://term1,PII.None,user:%s,user:%s,%s,\"glossaryTermDateCp:18-09-2024;glossaryTermDateTimeCp:18-09-2024 01:09:34;glossaryTermDurationCp:PT5H30M10S;glossaryTermEmailCp:admin@open-metadata.org;glossaryTermEntRefCp:team:\"\"%s\"\";glossaryTermEntRefListCp:user:\"\"%s\"\"|user:\"\"%s\"\"\"", + ",g1,dsp1,\"dsc1,1\",h1;h2;h3,g1.g1t1;g2.g2t1,term1;http://term1,PII.None,user:%s,user:%s,%s,\"glossaryTermDateCp:18-09-2024;glossaryTermDateTimeCp:18-09-2024 01:09:34;glossaryTermDurationCp:PT5H30M10S;glossaryTermEmailCp:admin@open-metadata.org;glossaryTermEntRefCp:team:\"\"%s\"\";glossaryTermEntRefListCp:user:\"\"%s\"\"|user:\"\"%s\"\"\"", reviewerRef.get(0), user1, "Approved", team11, user1, user2), String.format( - ",g2,dsp2,dsc3,h1;h3;h3,,term2;https://term2,PII.NonSensitive,,user:%s,%s,\"glossaryTermEnumCpMulti:val3|val2|val1|val4|val5;glossaryTermEnumCpSingle:single1;glossaryTermIntegerCp:7777;glossaryTermMarkdownCp:# Sample Markdown Text;glossaryTermNumberCp:123456;\"\"glossaryTermQueryCp:select col,row from table where id ='30';\"\";glossaryTermStringCp:sample string content;glossaryTermTimeCp:10:08:45;glossaryTermTimeIntervalCp:1726142300000:17261420000;glossaryTermTimestampCp:1726142400000\"", + ",g2,dsp2,dsc3,h1;h3;h3,g1.g1t1;g2.g2t1,term2;https://term2,PII.NonSensitive,,user:%s,%s,\"glossaryTermEnumCpMulti:val1|val2|val3|val4|val5;glossaryTermEnumCpSingle:single1;glossaryTermIntegerCp:7777;glossaryTermMarkdownCp:# Sample Markdown Text;glossaryTermNumberCp:123456;\"\"glossaryTermQueryCp:select col,row from table where id ='30';\"\";glossaryTermStringCp:sample string content;glossaryTermTimeCp:10:08:45;glossaryTermTimeIntervalCp:1726142300000:17261420000;glossaryTermTimestampCp:1726142400000\"", user1, "Approved"), String.format( - "importExportTest.g1,g11,dsp2,dsc11,h1;h3;h3,,,,user:%s,team:%s,%s,", + "importExportTest.g1,g11,dsp2,dsc11,h1;h3;h3,g1.g1t1;g2.g2t1,,,user:%s,team:%s,%s,", reviewerRef.get(0), team11, "Draft")); // Update terms with change in description List updateRecords = listOf( String.format( - ",g1,dsp1,new-dsc1,h1;h2;h3,,term1;http://term1,PII.None,user:%s,user:%s,%s,\"glossaryTermDateCp:18-09-2024;glossaryTermDateTimeCp:18-09-2024 01:09:34;glossaryTermDurationCp:PT5H30M10S;glossaryTermEmailCp:admin@open-metadata.org;glossaryTermEntRefCp:team:\"\"%s\"\";glossaryTermEntRefListCp:user:\"\"%s\"\"|user:\"\"%s\"\"\"", + ",g1,dsp1,new-dsc1,h1;h2;h3,g1.g1t1;importExportTest.g2;g2.g2t1,term1;http://term1,PII.None,user:%s,user:%s,%s,\"glossaryTermDateCp:18-09-2024;glossaryTermDateTimeCp:18-09-2024 01:09:34;glossaryTermDurationCp:PT5H30M10S;glossaryTermEmailCp:admin@open-metadata.org;glossaryTermEntRefCp:team:\"\"%s\"\";glossaryTermEntRefListCp:user:\"\"%s\"\"|user:\"\"%s\"\"\"", reviewerRef.get(0), user1, "Approved", team11, user1, user2), String.format( - ",g2,dsp2,new-dsc3,h1;h3;h3,,term2;https://term2,PII.NonSensitive,user:%s,user:%s,%s,\"glossaryTermEnumCpMulti:val3|val2|val1|val4|val5;glossaryTermEnumCpSingle:single1;glossaryTermIntegerCp:7777;glossaryTermMarkdownCp:# Sample Markdown Text;glossaryTermNumberCp:123456;\"\"glossaryTermQueryCp:select col,row from table where id ='30';\"\";glossaryTermStringCp:sample string content;glossaryTermTimeCp:10:08:45;glossaryTermTimeIntervalCp:1726142300000:17261420000;glossaryTermTimestampCp:1726142400000\"", + ",g2,dsp2,new-dsc3,h1;h3;h3,importExportTest.g1;g1.g1t1;g2.g2t1,term2;https://term2,PII.NonSensitive,user:%s,user:%s,%s,\"glossaryTermEnumCpMulti:val1|val2|val3|val4|val5;glossaryTermEnumCpSingle:single1;glossaryTermIntegerCp:7777;glossaryTermMarkdownCp:# Sample Markdown Text;glossaryTermNumberCp:123456;\"\"glossaryTermQueryCp:select col,row from table where id ='30';\"\";glossaryTermStringCp:sample string content;glossaryTermTimeCp:10:08:45;glossaryTermTimeIntervalCp:1726142300000:17261420000;glossaryTermTimestampCp:1726142400000\"", user1, user2, "Approved"), String.format( "importExportTest.g1,g11,dsp2,new-dsc11,h1;h3;h3,,,,user:%s,team:%s,%s,\"\"\"glossaryTermTableCol1Cp:row_1_col1_Value,,\"\";\"\"glossaryTermTableCol3Cp:row_1_col1_Value,row_1_col2_Value,row_1_col3_Value|row_2_col1_Value,row_2_col2_Value,row_2_col3_Value\"\"\"", @@ -933,7 +938,7 @@ void testGlossaryImportExport() throws IOException { // Add new row to existing rows List newRecords = listOf( - ",g3,dsp0,dsc0,h1;h2;h3,,term0;http://term0,PII.Sensitive,,,Approved,\"\"\"glossaryTermTableCol1Cp:row_1_col1_Value,,\"\";\"\"glossaryTermTableCol3Cp:row_1_col1_Value,row_1_col2_Value,row_1_col3_Value|row_2_col1_Value,row_2_col2_Value,row_2_col3_Value\"\"\""); + ",g3,dsp0,dsc0,h1;h2;h3,g1.g1t1;g2.g2t1,term0;http://term0,PII.Sensitive,,,Approved,\"\"\"glossaryTermTableCol1Cp:row_1_col1_Value,,\"\";\"\"glossaryTermTableCol3Cp:row_1_col1_Value,row_1_col2_Value,row_1_col3_Value|row_2_col1_Value,row_2_col2_Value,row_2_col3_Value\"\"\""); Awaitility.await() .atMost(Duration.ofMillis(120 * 1000L)) .pollInterval(Duration.ofMillis(2000L)) diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryTermResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryTermResourceTest.java index c2e15538a3aa..6acb2757e488 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryTermResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryTermResourceTest.java @@ -68,12 +68,14 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; import org.junit.jupiter.api.TestMethodOrder; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; import org.openmetadata.schema.api.data.CreateGlossary; import org.openmetadata.schema.api.data.CreateGlossaryTerm; import org.openmetadata.schema.api.data.CreateTable; import org.openmetadata.schema.api.data.TermReference; import org.openmetadata.schema.api.feed.ResolveTask; -import org.openmetadata.schema.entity.data.EntityHierarchy__1; +import org.openmetadata.schema.entity.data.EntityHierarchy; import org.openmetadata.schema.entity.data.Glossary; import org.openmetadata.schema.entity.data.GlossaryTerm; import org.openmetadata.schema.entity.data.GlossaryTerm.Status; @@ -88,6 +90,7 @@ import org.openmetadata.schema.type.TaskDetails; import org.openmetadata.schema.type.TaskStatus; import org.openmetadata.service.Entity; +import org.openmetadata.service.governance.workflows.WorkflowHandler; import org.openmetadata.service.resources.EntityResourceTest; import org.openmetadata.service.resources.databases.TableResourceTest; import org.openmetadata.service.resources.feeds.FeedResource.ThreadList; @@ -264,7 +267,11 @@ void test_commonPrefixTagLabelCount(TestInfo test) throws IOException { } @Test + @Execution(ExecutionMode.SAME_THREAD) void patch_addDeleteReviewers(TestInfo test) throws IOException { + // Note: We are disabling the GlossaryTermApprovalWorkflow to avoid the Workflow Kicking it and + // adding extra ChangeDescriptions. + WorkflowHandler.getInstance().suspendWorkflow("GlossaryTermApprovalWorkflow"); CreateGlossaryTerm create = createRequest(getEntityName(test), "desc", "", null).withReviewers(null).withSynonyms(null); GlossaryTerm term = createEntity(create, ADMIN_AUTH_HEADERS); @@ -281,14 +288,11 @@ void patch_addDeleteReviewers(TestInfo test) throws IOException { term.withReviewers(List.of(USER1_REF)) .withSynonyms(List.of("synonym1")) .withReferences(List.of(reference1)); - patchEntity(term.getId(), origJson, term, ADMIN_AUTH_HEADERS); - waitForTaskToBeCreated(term.getFullyQualifiedName()); ChangeDescription change = getChangeDescription(term, MINOR_UPDATE); fieldAdded(change, "reviewers", List.of(USER1_REF)); fieldAdded(change, "synonyms", List.of("synonym1")); fieldAdded(change, "references", List.of(reference1)); - fieldUpdated(change, "status", Status.APPROVED, Status.IN_REVIEW); term = patchEntityAndCheck(term, origJson, ADMIN_AUTH_HEADERS, MINOR_UPDATE, change); // Add reviewer USER2, synonym2, reference2 in PATCH request @@ -303,7 +307,6 @@ void patch_addDeleteReviewers(TestInfo test) throws IOException { fieldAdded(change, "reviewers", List.of(USER1_REF, USER2_REF)); fieldAdded(change, "synonyms", List.of("synonym1", "synonym2")); fieldAdded(change, "references", List.of(reference1, reference2)); - fieldUpdated(change, "status", Status.APPROVED, Status.IN_REVIEW); term = patchEntityAndCheck(term, origJson, ADMIN_AUTH_HEADERS, CHANGE_CONSOLIDATED, change); // Remove a reviewer USER1, synonym1, reference1 in PATCH request @@ -316,8 +319,10 @@ void patch_addDeleteReviewers(TestInfo test) throws IOException { fieldAdded(change, "reviewers", List.of(USER2_REF)); fieldAdded(change, "synonyms", List.of("synonym2")); fieldAdded(change, "references", List.of(reference2)); - fieldUpdated(change, "status", Status.APPROVED, Status.IN_REVIEW); patchEntityAndCheck(term, origJson, ADMIN_AUTH_HEADERS, CHANGE_CONSOLIDATED, change); + + // Note: We are re-enabling the GlossaryTermApprovalWorkflow. + WorkflowHandler.getInstance().resumeWorkflow("GlossaryTermApprovalWorkflow"); } @Test @@ -485,12 +490,19 @@ taskId2, new ResolveTask().withNewValue("rejected"), ADMIN_AUTH_HEADERS), String origJson = JsonUtils.pojoToJson(g2t5); // Add reviewer DATA_CONSUMER in PATCH request - g2t5.withReviewers(List.of(DATA_CONSUMER_REF, USER1_REF, USER2_REF)); + List newReviewers = List.of(DATA_CONSUMER_REF, USER1_REF, USER2_REF); + g2t5.withReviewers(newReviewers); + + double previousVersion = g2t5.getVersion(); + g2t5 = patchEntityUsingFqn(g2t5.getFullyQualifiedName(), origJson, g2t5, ADMIN_AUTH_HEADERS); - ChangeDescription change = getChangeDescription(g2t5, MINOR_UPDATE); - fieldAdded(change, "reviewers", List.of(DATA_CONSUMER_REF)); - fieldUpdated(change, "status", Status.DRAFT.value(), Status.IN_REVIEW.value()); - g2t5 = patchEntityUsingFqnAndCheck(g2t5, origJson, ADMIN_AUTH_HEADERS, MINOR_UPDATE, change); + // Due to the Glossary Workflow changing the Status from 'DRAFT' to 'IN_REVIEW' as a + // GovernanceBot, two changes are created. + assertEquals(g2t5.getVersion(), previousVersion + 0.2, 0.0001); + assertTrue( + g2t5.getReviewers().containsAll(newReviewers) + && newReviewers.containsAll(g2t5.getReviewers())); + assertEquals(g2t5.getStatus(), Status.IN_REVIEW); Thread approvalTask1 = assertApprovalTask(g2t5, TaskStatus.Open); // A Request Approval task is opened @@ -862,7 +874,7 @@ public void test_buildGlossaryTermNestedHierarchy(TestInfo test) throws HttpResp GlossaryTerm childGlossaryTerm = createEntity(create, ADMIN_AUTH_HEADERS); String response = getResponseFormSearchWithHierarchy("glossary_term_search_index", "*childGlossaryTerm*"); - List glossaries = JsonUtils.readObjects(response, EntityHierarchy__1.class); + List glossaries = JsonUtils.readObjects(response, EntityHierarchy.class); boolean isChild = glossaries.stream() .filter(glossary -> "g1".equals(glossary.getName())) // Find glossary with name "g1" diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/kpi/KpiResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/kpi/KpiResourceTest.java index 32af712a0293..dd6af914e425 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/kpi/KpiResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/kpi/KpiResourceTest.java @@ -82,7 +82,11 @@ private void createDataAssetsDataStream() { String dataStreamName = String.format("%s-%s", "di-data-assets", dataAssetType).toLowerCase(); if (!searchInterface.dataAssetDataStreamExists(dataStreamName)) { - searchInterface.createDataAssetsDataStream(dataStreamName); + searchInterface.createDataAssetsDataStream( + dataStreamName, + dataAssetType, + getSearchRepository().getIndexMapping(dataAssetType), + "en"); } } } catch (IOException ex) { diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/lineage/LineageResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/lineage/LineageResourceTest.java index 2e5af9e3ba1b..4c794113bb71 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/lineage/LineageResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/lineage/LineageResourceTest.java @@ -504,9 +504,9 @@ void get_dataQualityLineage(TestInfo test) CreateTestSuite createTestSuite6 = testSuiteResourceTest.createRequest(test).withName(TABLES.get(6).getFullyQualifiedName()); TestSuite testSuite4 = - testSuiteResourceTest.createExecutableTestSuite(createTestSuite4, ADMIN_AUTH_HEADERS); + testSuiteResourceTest.createBasicTestSuite(createTestSuite4, ADMIN_AUTH_HEADERS); TestSuite testSuite6 = - testSuiteResourceTest.createExecutableTestSuite(createTestSuite6, ADMIN_AUTH_HEADERS); + testSuiteResourceTest.createBasicTestSuite(createTestSuite6, ADMIN_AUTH_HEADERS); MessageParser.EntityLink TABLE4_LINK = new MessageParser.EntityLink(Entity.TABLE, TABLES.get(4).getFullyQualifiedName()); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/metadata/TypeResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/metadata/TypeResourceTest.java index f36480852e06..1e9c661f7209 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/metadata/TypeResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/metadata/TypeResourceTest.java @@ -244,12 +244,16 @@ void put_patch_customProperty_enum_200() throws IOException { enumFieldA.setCustomPropertyConfig( new CustomPropertyConfig().withConfig(new EnumConfig().withValues(List.of("A", "B")))); ChangeDescription change3 = getChangeDescription(tableEntity, MINOR_UPDATE); - assertResponseContains( - () -> - addCustomPropertyAndCheck( - tableEntity1.getId(), enumFieldA, ADMIN_AUTH_HEADERS, MINOR_UPDATE, change3), - Status.BAD_REQUEST, - "Existing Enum Custom Property values cannot be removed."); + fieldUpdated( + change3, + EntityUtil.getCustomField(enumFieldA, "customPropertyConfig"), + new CustomPropertyConfig().withConfig(new EnumConfig().withValues(List.of("A", "B", "C"))), + new CustomPropertyConfig().withConfig(new EnumConfig().withValues(List.of("A", "B")))); + tableEntity = + addCustomPropertyAndCheck( + tableEntity.getId(), enumFieldA, ADMIN_AUTH_HEADERS, MINOR_UPDATE, change3); + assertCustomProperties(new ArrayList<>(List.of(enumFieldA)), tableEntity.getCustomProperties()); + prevConfig = enumFieldA.getCustomPropertyConfig(); enumFieldA.setCustomPropertyConfig( new CustomPropertyConfig() @@ -264,8 +268,7 @@ void put_patch_customProperty_enum_200() throws IOException { ChangeDescription change5 = getChangeDescription(tableEntity, MINOR_UPDATE); enumFieldA.setCustomPropertyConfig( - new CustomPropertyConfig() - .withConfig(new EnumConfig().withValues(List.of("A", "B", "C", "D")))); + new CustomPropertyConfig().withConfig(new EnumConfig().withValues(List.of("A", "B", "D")))); fieldUpdated( change5, EntityUtil.getCustomField(enumFieldA, "customPropertyConfig"), diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/system/ConfigResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/system/ConfigResourceTest.java index 9cf314ed1755..9a80814133ef 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/system/ConfigResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/system/ConfigResourceTest.java @@ -124,7 +124,7 @@ void get_Login_Configuration_200_OK() throws IOException { LoginConfiguration loginConfiguration = TestUtils.get(target, LoginConfiguration.class, TEST_AUTH_HEADERS); assertEquals(3, loginConfiguration.getMaxLoginFailAttempts()); - assertEquals(600, loginConfiguration.getAccessBlockTime()); + assertEquals(30, loginConfiguration.getAccessBlockTime()); assertEquals(3600, loginConfiguration.getJwtTokenExpiryTime()); } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/system/SystemResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/system/SystemResourceTest.java index c0bd093aa5cc..d7cc9d04cddd 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/system/SystemResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/system/SystemResourceTest.java @@ -47,6 +47,7 @@ import org.openmetadata.schema.auth.JWTAuthMechanism; import org.openmetadata.schema.auth.JWTTokenExpiry; import org.openmetadata.schema.configuration.AssetCertificationSettings; +import org.openmetadata.schema.configuration.WorkflowSettings; import org.openmetadata.schema.email.SmtpSettings; import org.openmetadata.schema.entity.data.Table; import org.openmetadata.schema.entity.teams.AuthenticationMechanism; @@ -80,6 +81,7 @@ import org.openmetadata.service.resources.teams.TeamResourceTest; import org.openmetadata.service.resources.teams.UserResourceTest; import org.openmetadata.service.resources.topics.TopicResourceTest; +import org.openmetadata.service.secrets.masker.PasswordEntityMasker; import org.openmetadata.service.util.JsonUtils; import org.openmetadata.service.util.TestUtils; @@ -189,10 +191,10 @@ void testSystemConfigs() throws HttpResponseException { // Test Email Config Settings emailSettings = getSystemConfig(SettingsType.EMAIL_CONFIGURATION); SmtpSettings smtp = JsonUtils.convertValue(emailSettings.getConfigValue(), SmtpSettings.class); - // Password for Email is encrypted using fernet + // Password for Email is always sent in hidden SmtpSettings expected = config.getSmtpSettings(); - expected.setPassword(smtp.getPassword()); - assertEquals(config.getSmtpSettings(), smtp); + expected.setPassword(PasswordEntityMasker.PASSWORD_MASK); + assertEquals(expected, smtp); // Test Custom Ui Theme Preference Config Settings uiThemeConfigWrapped = getSystemConfig(SettingsType.CUSTOM_UI_THEME_PREFERENCE); @@ -435,7 +437,7 @@ void testLoginConfigurationSettings() throws HttpResponseException { // Assert default values assertEquals(3, loginConfig.getMaxLoginFailAttempts()); - assertEquals(600, loginConfig.getAccessBlockTime()); + assertEquals(30, loginConfig.getAccessBlockTime()); assertEquals(3600, loginConfig.getJwtTokenExpiryTime()); // Update login configuration @@ -551,6 +553,48 @@ void testLineageSettings() throws HttpResponseException { assertEquals(4, updatedLineageConfig.getDownstreamDepth()); } + @Test + void testWorkflowSettings() throws HttpResponseException { + // Retrieve the default workflow settings + Settings setting = getSystemConfig(SettingsType.WORKFLOW_SETTINGS); + WorkflowSettings workflowSettings = + JsonUtils.convertValue(setting.getConfigValue(), WorkflowSettings.class); + + // Assert default values + assertEquals(50, workflowSettings.getExecutorConfiguration().getCorePoolSize()); + assertEquals(1000, workflowSettings.getExecutorConfiguration().getQueueSize()); + assertEquals(100, workflowSettings.getExecutorConfiguration().getMaxPoolSize()); + assertEquals(20, workflowSettings.getExecutorConfiguration().getTasksDuePerAcquisition()); + assertEquals(7, workflowSettings.getHistoryCleanUpConfiguration().getCleanAfterNumberOfDays()); + + // Update workflow settings + workflowSettings.getExecutorConfiguration().setCorePoolSize(100); + workflowSettings.getExecutorConfiguration().setQueueSize(2000); + workflowSettings.getExecutorConfiguration().setMaxPoolSize(200); + workflowSettings.getExecutorConfiguration().setTasksDuePerAcquisition(40); + workflowSettings.getHistoryCleanUpConfiguration().setCleanAfterNumberOfDays(10); + + Settings updatedSetting = + new Settings() + .withConfigType(SettingsType.WORKFLOW_SETTINGS) + .withConfigValue(workflowSettings); + + updateSystemConfig(updatedSetting); + + // Retrieve the updated settings + Settings updatedSettings = getSystemConfig(SettingsType.WORKFLOW_SETTINGS); + WorkflowSettings updateWorkflowSettings = + JsonUtils.convertValue(updatedSettings.getConfigValue(), WorkflowSettings.class); + + // Assert updated values + assertEquals(100, updateWorkflowSettings.getExecutorConfiguration().getCorePoolSize()); + assertEquals(2000, updateWorkflowSettings.getExecutorConfiguration().getQueueSize()); + assertEquals(200, updateWorkflowSettings.getExecutorConfiguration().getMaxPoolSize()); + assertEquals(40, updateWorkflowSettings.getExecutorConfiguration().getTasksDuePerAcquisition()); + assertEquals( + 10, updateWorkflowSettings.getHistoryCleanUpConfiguration().getCleanAfterNumberOfDays()); + } + @Test void globalProfilerConfig(TestInfo test) throws HttpResponseException { // Create a profiler config diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/security/JWTTokenGeneratorTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/security/JWTTokenGeneratorTest.java index b7d935f5c624..555a079ddd30 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/security/JWTTokenGeneratorTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/security/JWTTokenGeneratorTest.java @@ -15,6 +15,7 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; +import org.openmetadata.schema.api.security.AuthenticationConfiguration; import org.openmetadata.schema.api.security.jwt.JWTTokenConfiguration; import org.openmetadata.schema.auth.JWTAuthMechanism; import org.openmetadata.schema.auth.JWTTokenExpiry; @@ -38,7 +39,8 @@ public void setup() { jwtTokenConfiguration.setRsaprivateKeyFilePath(rsaPrivateKeyPath); jwtTokenConfiguration.setRsapublicKeyFilePath(rsaPublicKeyPath); jwtTokenGenerator = JWTTokenGenerator.getInstance(); - jwtTokenGenerator.init(jwtTokenConfiguration); + jwtTokenGenerator.init( + AuthenticationConfiguration.TokenValidationAlgorithm.RS_256, jwtTokenConfiguration); } @Test diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/security/policyevaluator/RuleEvaluatorTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/security/policyevaluator/RuleEvaluatorTest.java index 40c9397316d7..7f930c93fa9b 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/security/policyevaluator/RuleEvaluatorTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/security/policyevaluator/RuleEvaluatorTest.java @@ -25,6 +25,7 @@ import org.openmetadata.schema.entity.teams.Role; import org.openmetadata.schema.entity.teams.Team; import org.openmetadata.schema.entity.teams.User; +import org.openmetadata.schema.type.AssetCertification; import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.Include; import org.openmetadata.schema.type.TagLabel; @@ -187,6 +188,36 @@ void test_matchAnyTag() { assertTrue(evaluateExpression("!matchAnyTag('tag4')")); } + @Test + void test_matchAnyCertification() { + // Certification is not Present + assertFalse(evaluateExpression("!matchAnyCertification('Certification.Gold')")); + assertFalse( + evaluateExpression("!matchAnyCertification('Certification.Gold', 'Certification.Silver')")); + assertFalse(evaluateExpression("matchAnyCertification('Certification.Bronze')")); + + table.withCertification( + new AssetCertification().withTagLabel(new TagLabel().withTagFQN("Certification.Gold"))); + + // Certification is present + assertTrue( + evaluateExpression("matchAnyCertification('Certification.Gold', 'Certification.Silver')")); + assertFalse( + evaluateExpression("!matchAnyCertification('Certification.Gold', 'Certification.Silver')")); + assertTrue(evaluateExpression("matchAnyCertification('Certification.Gold')")); + assertFalse(evaluateExpression("!matchAnyCertification('Certification.Gold')")); + + // Certification is different + assertFalse( + evaluateExpression( + "matchAnyCertification('Certification.Bronze', 'Certification.Silver')")); + assertTrue( + evaluateExpression( + "!matchAnyCertification('Certification.Bronze', 'Certification.Silver')")); + assertFalse(evaluateExpression("matchAnyCertification('Certification.Bronze')")); + assertTrue(evaluateExpression("!matchAnyCertification('Certification.Bronze')")); + } + @Test void test_matchTeam() { // Create a team hierarchy diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/util/TestUtils.java b/openmetadata-service/src/test/java/org/openmetadata/service/util/TestUtils.java index e39f2d05a6e5..534315afa9b0 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/util/TestUtils.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/util/TestUtils.java @@ -105,6 +105,9 @@ public final class TestUtils { public static String LONG_ENTITY_NAME = "a".repeat(256 + 1); public static final Map ADMIN_AUTH_HEADERS = authHeaders(ADMIN_USER_NAME + "@open-metadata.org"); + public static final String GOVERNANCE_BOT = "governance-bot"; + public static final Map GOVERNANCE_BOT_AUTH_HEADERS = + authHeaders(GOVERNANCE_BOT + "@open-metadata.org"); public static final String INGESTION_BOT = "ingestion-bot"; public static final Map INGESTION_BOT_AUTH_HEADERS = authHeaders(INGESTION_BOT + "@open-metadata.org"); diff --git a/openmetadata-shaded-deps/elasticsearch-dep/pom.xml b/openmetadata-shaded-deps/elasticsearch-dep/pom.xml index 156948df99d0..6ada16fc32d5 100644 --- a/openmetadata-shaded-deps/elasticsearch-dep/pom.xml +++ b/openmetadata-shaded-deps/elasticsearch-dep/pom.xml @@ -5,7 +5,7 @@ openmetadata-shaded-deps org.open-metadata - 1.6.0-SNAPSHOT + 1.6.4 4.0.0 elasticsearch-deps diff --git a/openmetadata-shaded-deps/opensearch-dep/pom.xml b/openmetadata-shaded-deps/opensearch-dep/pom.xml index 157e02438587..3b4b02d043ea 100644 --- a/openmetadata-shaded-deps/opensearch-dep/pom.xml +++ b/openmetadata-shaded-deps/opensearch-dep/pom.xml @@ -5,7 +5,7 @@ openmetadata-shaded-deps org.open-metadata - 1.6.0-SNAPSHOT + 1.6.4 4.0.0 opensearch-deps diff --git a/openmetadata-shaded-deps/pom.xml b/openmetadata-shaded-deps/pom.xml index 4c6131ebdad7..befdf462cbac 100644 --- a/openmetadata-shaded-deps/pom.xml +++ b/openmetadata-shaded-deps/pom.xml @@ -5,7 +5,7 @@ platform org.open-metadata - 1.6.0-SNAPSHOT + 1.6.4 4.0.0 openmetadata-shaded-deps diff --git a/openmetadata-spec/pom.xml b/openmetadata-spec/pom.xml index cfaea4fe7159..317336043754 100644 --- a/openmetadata-spec/pom.xml +++ b/openmetadata-spec/pom.xml @@ -5,7 +5,7 @@ platform org.open-metadata - 1.6.0-SNAPSHOT + 1.6.4 4.0.0 diff --git a/openmetadata-spec/src/main/resources/json/schema/api/docStore/createDocument.json b/openmetadata-spec/src/main/resources/json/schema/api/docStore/createDocument.json index d05a04d042e6..3daac09b4965 100644 --- a/openmetadata-spec/src/main/resources/json/schema/api/docStore/createDocument.json +++ b/openmetadata-spec/src/main/resources/json/schema/api/docStore/createDocument.json @@ -1,7 +1,7 @@ { "$id": "https://open-metadata.org/schema/entity/docStore/document.json", "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Document", + "title": "CreateDocumentRequest", "description": "This schema defines Document. A Generic entity to capture any kind of Json Payload.", "type": "object", "javaType": "org.openmetadata.schema.entities.docStore.CreateDocument", diff --git a/openmetadata-spec/src/main/resources/json/schema/api/tests/createTestCaseResult.json b/openmetadata-spec/src/main/resources/json/schema/api/tests/createTestCaseResult.json index 5e008a5e091d..d064bbbe6fde 100644 --- a/openmetadata-spec/src/main/resources/json/schema/api/tests/createTestCaseResult.json +++ b/openmetadata-spec/src/main/resources/json/schema/api/tests/createTestCaseResult.json @@ -6,6 +6,10 @@ "javaType": "org.openmetadata.schema.api.tests.CreateTestCaseResult", "type": "object", "properties": { + "fqn": { + "description": "Fqn of the test case against which this test case result is added.", + "type": "string" + }, "timestamp": { "description": "Data one which test case result is taken.", "$ref": "../../type/basic.json#/definitions/timestamp" diff --git a/openmetadata-spec/src/main/resources/json/schema/api/tests/createTestSuite.json b/openmetadata-spec/src/main/resources/json/schema/api/tests/createTestSuite.json index e3b549e6317f..1d01b9571108 100644 --- a/openmetadata-spec/src/main/resources/json/schema/api/tests/createTestSuite.json +++ b/openmetadata-spec/src/main/resources/json/schema/api/tests/createTestSuite.json @@ -33,10 +33,15 @@ "description": "Owners of this test suite", "$ref": "../../type/entityReferenceList.json" }, - "executableEntityReference": { - "description": "FQN of the entity the test suite is executed against. Only applicable for executable test suites.", + "basicEntityReference": { + "description": "Entity reference the test suite needs to execute the test against. Only applicable if the test suite is basic.", "$ref": "../../type/basic.json#/definitions/fullyQualifiedEntityName" }, + "executableEntityReference": { + "description": "DEPRECATED in 1.6.2: use 'basicEntityReference'", + "$ref": "../../type/basic.json#/definitions/fullyQualifiedEntityName", + "deprecated": true + }, "domain": { "description": "Fully qualified name of the domain the Table belongs to.", "type": "string" diff --git a/openmetadata-spec/src/main/resources/json/schema/auth/emailVerificationToken.json b/openmetadata-spec/src/main/resources/json/schema/auth/emailVerificationToken.json index 964d31bd23ca..ea47256e4ab2 100644 --- a/openmetadata-spec/src/main/resources/json/schema/auth/emailVerificationToken.json +++ b/openmetadata-spec/src/main/resources/json/schema/auth/emailVerificationToken.json @@ -43,12 +43,6 @@ "$ref": "../type/basic.json#/definitions/timestamp" } }, - "required": [ - "token", - "user", - "tokenType", - "tokenStatus", - "expiryDate" - ], + "required": ["token", "userId", "tokenType", "tokenStatus", "expiryDate"], "additionalProperties": false } \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/configuration/authenticationConfiguration.json b/openmetadata-spec/src/main/resources/json/schema/configuration/authenticationConfiguration.json index 670401107ca8..ac7d5075ecae 100644 --- a/openmetadata-spec/src/main/resources/json/schema/configuration/authenticationConfiguration.json +++ b/openmetadata-spec/src/main/resources/json/schema/configuration/authenticationConfiguration.json @@ -46,6 +46,12 @@ "type": "string" } }, + "tokenValidationAlgorithm": { + "description": "Token Validation Algorithm to use.", + "type": "string", + "enum": ["RS256", "RS384", "RS512"], + "default": "RS256" + }, "authority": { "description": "Authentication Authority", "type": "string" diff --git a/openmetadata-spec/src/main/resources/json/schema/configuration/workflowSettings.json b/openmetadata-spec/src/main/resources/json/schema/configuration/workflowSettings.json new file mode 100644 index 000000000000..72280c724b1c --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/configuration/workflowSettings.json @@ -0,0 +1,59 @@ +{ + "$id": "https://open-metadata.org/schema/configuration/workflowSettings.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "WorkflowSettings", + "description": "This schema defines the Workflow Settings.", + "type": "object", + "javaType": "org.openmetadata.schema.configuration.WorkflowSettings", + "definitions": { + "executorConfiguration": { + "type": "object", + "properties": { + "corePoolSize": { + "type": "integer", + "default": 50, + "description": "Default worker Pool Size. The Workflow Executor by default has this amount of workers." + }, + "maxPoolSize": { + "type": "integer", + "default": 100, + "description": "Maximum worker Pool Size. The Workflow Executor could grow up to this number of workers." + }, + "queueSize": { + "type": "integer", + "default": 1000, + "description": "Amount of Tasks that can be queued to be picked up by the Workflow Executor." + }, + "tasksDuePerAcquisition": { + "type": "integer", + "default": 20, + "description": "The amount of Tasks that the Workflow Executor is able to pick up each time it looks for more." + } + }, + "additionalProperties": false + }, + "historyCleanUpConfiguration": { + "type": "object", + "properties": { + "cleanAfterNumberOfDays": { + "type": "integer", + "default": 7, + "description": "Cleans the Workflow Task that were finished, after given number of days." + } + }, + "additionalProperties": false + } + }, + "properties": { + "executorConfiguration": { + "$ref": "#/definitions/executorConfiguration", + "description": "Used to set up the Workflow Executor Settings." + }, + "historyCleanUpConfiguration": { + "$ref": "#/definitions/historyCleanUpConfiguration", + "description": "Used to set up the History CleanUp Settings." + } + }, + "required": [], + "additionalProperties": false +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/appRunRecord.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/appRunRecord.json index 122ee052ef92..6dde1cf433e1 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/appRunRecord.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/appRunRecord.json @@ -34,7 +34,8 @@ "active", "activeError", "stopped", - "success" + "success", + "pending" ] }, "runType": { @@ -63,6 +64,10 @@ }, "scheduleInfo": { "$ref": "./app.json#/definitions/appSchedule" + }, + "config": { + "descripton": "The configuration used for this application run. It's type will be based on the application type. Old runs might not be compatible with schema of app configuration.", + "$ref": "../../type/basic.json#/definitions/map" } }, "additionalProperties": false diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/applicationConfig.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/applicationConfig.json index 74de4576ef66..947d54a309fc 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/applicationConfig.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/applicationConfig.json @@ -24,6 +24,9 @@ }, { "$ref": "external/slackAppTokenConfiguration.json" + }, + { + "$ref": "internal/dataRetentionConfiguration.json" } ] }, diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/automator/addCustomProperties.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/automator/addCustomProperties.json new file mode 100644 index 000000000000..476192e9e7d2 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/automator/addCustomProperties.json @@ -0,0 +1,36 @@ +{ + "$id": "https://open-metadata.org/schema/entity/applications/configuration/external/automator/addCustomPropertiesAction.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "AddCustomPropertiesAction", + "description": "Add a Custom Property to the selected assets.", + "type": "object", + "definitions": { + "AddCustomPropertiesActionType": { + "description": "Add Custom Properties Action Type.", + "type": "string", + "enum": ["AddCustomPropertiesAction"], + "default": "AddCustomPropertiesAction" + } + }, + "properties": { + "type": { + "title": "Application Type", + "description": "Application Type", + "$ref": "#/definitions/AddCustomPropertiesActionType", + "default": "AddCustomPropertiesAction" + }, + "customProperties": { + "description": "Owners to apply", + "$ref": "../../../../../type/basic.json#/definitions/entityExtension", + "default": null + }, + "overwriteMetadata": { + "title": "Overwrite Metadata", + "description": "Update the Custom Property even if it is defined in the asset. By default, we will only apply the owners to assets without the given Custom Property informed.", + "type": "boolean", + "default": false + } + }, + "required": ["type", "customProperties"], + "additionalProperties": false +} diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/automator/removeCustomPropertiesAction.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/automator/removeCustomPropertiesAction.json new file mode 100644 index 000000000000..e975ab6c1622 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/automator/removeCustomPropertiesAction.json @@ -0,0 +1,32 @@ +{ + "$id": "https://open-metadata.org/schema/entity/applications/configuration/external/automator/removeCustomPropertiesAction.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "RemoveCustomPropertiesAction", + "description": "Remove Custom Properties Action Type", + "type": "object", + "definitions": { + "removeCustomPropertiesActionType": { + "description": "Remove Custom Properties Action Type.", + "type": "string", + "enum": ["RemoveCustomPropertiesAction"], + "default": "RemoveCustomPropertiesAction" + } + }, + "properties": { + "type": { + "title": "Application Type", + "description": "Application Type", + "$ref": "#/definitions/removeCustomPropertiesActionType", + "default": "removeCustomPropertiesActionType" + }, + "customProperties": { + "description": "Custom Properties keys to remove", + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": ["type", "customProperties"], + "additionalProperties": false +} diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/automator/removeDescriptionAction.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/automator/removeDescriptionAction.json index 06abeffb7805..9dcd2a4ecf99 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/automator/removeDescriptionAction.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/automator/removeDescriptionAction.json @@ -21,12 +21,18 @@ }, "applyToChildren": { "title": "Apply to Children", - "description": "Remove descriptions from all children of the selected assets. E.g., columns, tasks, topic fields,...", + "description": "Remove descriptions from the children of the selected assets. E.g., columns, tasks, topic fields,...", "type": "array", "items": { "$ref": "../../../../../type/basic.json#/definitions/entityName" }, "default": null + }, + "applyToAll": { + "title": "Apply to All", + "description": "Remove descriptions from all the children and parent of the selected assets.", + "type": "boolean", + "default": null } }, "required": ["type"], diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/automator/removeTagsAction.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/automator/removeTagsAction.json index a255e0f9551c..12ebeec4a4aa 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/automator/removeTagsAction.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/automator/removeTagsAction.json @@ -10,6 +10,15 @@ "type": "string", "enum": ["RemoveTagsAction"], "default": "RemoveTagsAction" + }, + "labelType": { + "description" : "Remove tags by its label type", + "type": "string", + "enum": [ + "Manual", + "Propagated", + "Automated" + ] } }, "properties": { @@ -26,16 +35,30 @@ "$ref": "../../../../../type/tagLabel.json" } }, + "labels": { + "description": "Remove tags by its label type", + "type": "array", + "items": { + "$ref": "#/definitions/labelType" + }, + "default": null + }, "applyToChildren": { "title": "Apply to Children", - "description": "Remove tags from all the children of the selected assets. E.g., columns, tasks, topic fields,...", + "description": "Remove tags from the children of the selected assets. E.g., columns, tasks, topic fields,...", "type": "array", "items": { "$ref": "../../../../../type/basic.json#/definitions/entityName" }, "default": null + }, + "applyToAll": { + "title": "Apply to All", + "description": "Remove tags from all the children and parent of the selected assets.", + "type": "boolean", + "default": null } }, - "required": ["type", "tags"], + "required": ["type"], "additionalProperties": false } diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/automatorAppConfig.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/automatorAppConfig.json index a47c48c933fd..9985ec39e63e 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/automatorAppConfig.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/automatorAppConfig.json @@ -47,6 +47,9 @@ { "$ref": "automator/addDescriptionAction.json" }, + { + "$ref": "automator/addCustomProperties.json" + }, { "$ref": "automator/removeDescriptionAction.json" }, @@ -62,6 +65,9 @@ { "$ref": "automator/removeOwnerAction.json" }, + { + "$ref": "automator/removeCustomPropertiesAction.json" + }, { "$ref": "automator/lineagePropagationAction.json" }, diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/internal/dataRetentionConfiguration.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/internal/dataRetentionConfiguration.json new file mode 100644 index 000000000000..bdb4745559fa --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/internal/dataRetentionConfiguration.json @@ -0,0 +1,19 @@ +{ + "$id": "https://open-metadata.org/schema/entity/applications/configuration/dataRetentionConfiguration.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Retention Configuration", + "description": "Configure retention policies for each entity.", + "properties": { + "changeEventRetentionPeriod": { + "title": "Change Event Retention Period (days)", + "description": "Enter the retention period for change event records in days (e.g., 7 for one week, 30 for one month).", + "type": "integer", + "default": 7, + "minimum": 1 + } + }, + "required": [ + "changeEventRetentionPeriod" + ], + "additionalProperties": false +} diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/data/table.json b/openmetadata-spec/src/main/resources/json/schema/entity/data/table.json index 4ad8fb950ed4..d7e26c644327 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/data/table.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/table.json @@ -193,7 +193,8 @@ "PRIMARY_KEY", "FOREIGN_KEY", "SORT_KEY", - "DIST_KEY" + "DIST_KEY", + "CLUSTER_KEY" ] }, "columns": { diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/databricksConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/databricksConnection.json index cb8346c5b840..22756827f76d 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/databricksConnection.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/databricksConnection.json @@ -64,6 +64,12 @@ "type": "integer", "default": 120 }, + "queryHistoryTable":{ + "title": "Query History Table", + "description": "Table name to fetch the query history.", + "type": "string", + "default": "system.query.history" + }, "connectionOptions": { "title": "Connection Options", "$ref": "../connectionBasicType.json#/definitions/connectionOptions" diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/snowflakeConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/snowflakeConnection.json index 593d8105a1a6..0ff17d9e3d80 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/snowflakeConnection.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/snowflakeConnection.json @@ -68,6 +68,12 @@ "description": "Session query tag used to monitor usage on snowflake. To use a query tag snowflake user should have enough privileges to alter the session.", "type": "string" }, + "accountUsageSchema":{ + "title": "Account Usage Schema Name", + "description": "Full name of the schema where the account usage data is stored.", + "type": "string", + "default": "SNOWFLAKE.ACCOUNT_USAGE" + }, "privateKey": { "title": "Private Key", "description": "Connection to Snowflake instance via Private Key", diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/messaging/kafkaConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/messaging/kafkaConnection.json index e5590befd006..649b5404c31a 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/messaging/kafkaConnection.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/messaging/kafkaConnection.json @@ -88,6 +88,11 @@ "type": "string", "default": "-value" }, + "consumerConfigSSL": { + "title": "Consumer Config SSL", + "description": "Consumer Config SSL Config. Configuration for enabling SSL for the Consumer Config connection.", + "$ref": "../../../../security/ssl/verifySSLConfig.json#/definitions/sslConfig" + }, "schemaRegistrySSL": { "title": "Schema Registry SSL", "description": "Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry connection.", diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/airbyteConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/airbyteConnection.json index 867292cfe4ea..7ce93dc3eae3 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/airbyteConnection.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/airbyteConnection.json @@ -37,6 +37,12 @@ "type": "string", "format": "password" }, + "apiVersion": { + "title": "API Version", + "description": "Airbyte API version.", + "type": "string", + "default": "api/v1" + }, "supportsMetadataExtraction": { "title": "Supports Metadata Extraction", "$ref": "../connectionBasicType.json#/definitions/supportsMetadataExtraction" diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/matillion/matillionETL.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/matillion/matillionETL.json new file mode 100644 index 000000000000..183cbb45013e --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/matillion/matillionETL.json @@ -0,0 +1,43 @@ +{ + "$id": "https://open-metadata.org/schema/entity/services/connections/pipeline/matillion/matillionETL.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Matillion ETL Auth Config", + "description": "Matillion ETL Auth Config.", + "javaType": "org.openmetadata.schema.services.connections.pipeline.matillion.MatillionETLAuth", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "MatillionETL" + ], + "default": "MatillionETL" + }, + "hostPort": { + "type": "string", + "title": "Host", + "description": "Matillion Host", + "default": "localhost" + }, + "username": { + "title": "Username", + "description": "Username to connect to the Matillion. This user should have privileges to read all the metadata in Matillion.", + "type": "string" + }, + "password": { + "title": "Password", + "description": "Password to connect to the Matillion.", + "type": "string", + "format": "password" + }, + "sslConfig": { + "$ref": "../../../../../security/ssl/verifySSLConfig.json#/definitions/sslConfig" + } + }, + "required": [ + "hostPort", + "username", + "password" + ], + "additionalProperties": false +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/matillionConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/matillionConnection.json index 07598d15cbd8..3b8517c20461 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/matillionConnection.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/matillionConnection.json @@ -13,45 +13,6 @@ "Matillion" ], "default": "Matillion" - }, - "matillionETL": { - "description": "Matillion ETL Auth Config", - "type": "object", - "title": "Matillion ETL Auth Config", - "properties": { - "type": { - "type": "string", - "enum": [ - "MatillionETL" - ], - "default": "MatillionETL" - }, - "hostPort": { - "type": "string", - "title": "Host", - "description": "Matillion Host", - "default": "localhost" - }, - "username": { - "title": "Username", - "description": "Username to connect to the Matillion. This user should have privileges to read all the metadata in Matillion.", - "type": "string" - }, - "password": { - "title": "Password", - "description": "Password to connect to the Matillion.", - "type": "string", - "format": "password" - }, - "sslConfig": { - "$ref": "../../../../security/ssl/verifySSLConfig.json#/definitions/sslConfig" - } - }, - "required": [ - "hostPort", - "username", - "password" - ] } }, "properties": { @@ -66,7 +27,7 @@ "description": "Matillion Auth Configuration", "oneOf": [ { - "$ref": "#/definitions/matillionETL" + "$ref": "matillion/matillionETL.json" } ] }, diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/nifi/basicAuth.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/nifi/basicAuth.json new file mode 100644 index 000000000000..63512a45f828 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/nifi/basicAuth.json @@ -0,0 +1,28 @@ +{ + "$id": "https://open-metadata.org/schema/entity/services/connections/pipeline/nifi/basicAuth.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Nifi Basic Auth", + "description": "Configuration for connecting to Nifi Basic Auth.", + "javaType": "org.openmetadata.schema.services.connections.pipeline.nifi.BasicAuth", + "type": "object", + "properties": { + "username": { + "title": "Username", + "description": "Nifi user to authenticate to the API.", + "type": "string" + }, + "password": { + "title": "Password", + "description": "Nifi password to authenticate to the API.", + "type": "string", + "format": "password" + }, + "verifySSL": { + "title": "Verify SSL", + "description": "Boolean marking if we need to verify the SSL certs for Nifi. False by default.", + "type": "boolean", + "default": false + } + }, + "additionalProperties": false +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/nifi/clientCertificateAuth.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/nifi/clientCertificateAuth.json new file mode 100644 index 000000000000..a540eddcaa7b --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/nifi/clientCertificateAuth.json @@ -0,0 +1,26 @@ +{ + "$id": "https://open-metadata.org/schema/entity/services/connections/pipeline/nifi/clientCertificateAuth.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Nifi Client Certificate Auth", + "description": "Configuration for connecting to Nifi Client Certificate Auth.", + "javaType": "org.openmetadata.schema.services.connections.pipeline.nifi.ClientCertificateAuth", + "type": "object", + "properties": { + "certificateAuthorityPath": { + "title": "Certificat Authority Path", + "description": "Path to the root CA certificate", + "type": "string" + }, + "clientCertificatePath": { + "title": "Client Certificat", + "description": "Path to the client certificate", + "type": "string" + }, + "clientkeyPath": { + "title": "Client Key", + "description": "Path to the client key", + "type": "string" + } + }, + "additionalProperties": false +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/nifiConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/nifiConnection.json index 843e36b8122b..ce90643ba3ff 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/nifiConnection.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/nifiConnection.json @@ -15,6 +15,7 @@ "basicAuthentication": { "title": "Username/Password Authentication", "description": "username/password auth", + "javaType": "org.openmetadata.schema.services.connections.pipeline.NifiBasicAuth", "type":"object", "properties": { "username": { @@ -41,6 +42,7 @@ "title": "Client Certificate Authentication", "description": "client certificate auth", "type":"object", + "javaType": "org.openmetadata.schema.services.connections.pipeline.NifiClientAuth", "properties": { "certificateAuthorityPath":{ "title":"Certificat Authority Path", @@ -80,10 +82,10 @@ "description": "We support username/password or client certificate authentication", "oneOf": [ { - "$ref": "#/definitions/basicAuthentication" + "$ref": "nifi/basicAuth.json" }, { - "$ref": "#/definitions/clientCertificateAuthentication" + "$ref": "nifi/clientCertificateAuth.json" } ] }, diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/ingestionPipelines/ingestionPipeline.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/ingestionPipelines/ingestionPipeline.json index 314c78b2e555..3c5bf2fa11ef 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/ingestionPipelines/ingestionPipeline.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/ingestionPipelines/ingestionPipeline.json @@ -43,6 +43,10 @@ "status": { "description": "Ingestion Pipeline summary status. Informed at the end of the execution.", "$ref": "status.json#/definitions/ingestionStatus" + }, + "config": { + "description": "Pipeline configuration for this particular execution.", + "$ref": "../../../type/basic.json#/definitions/map" } }, "additionalProperties": false diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/teams/persona.json b/openmetadata-spec/src/main/resources/json/schema/entity/teams/persona.json index ac8d15ea4d2a..16588c532f3b 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/teams/persona.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/teams/persona.json @@ -1,7 +1,7 @@ { "$id": "https://open-metadata.org/schema/entity/teams/persona.json", "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Team", + "title": "Persona", "description": "This schema defines the Persona entity. A `Persona` is a job function associated with a user. An Example, Data Engineer or Data Consumer is a Persona of a user in Metadata world.", "type": "object", "javaType": "org.openmetadata.schema.entity.teams.Persona", @@ -38,10 +38,6 @@ "description": "Link to the resource corresponding to this entity.", "$ref": "../../type/basic.json#/definitions/href" }, - "uiCustomization": { - "description": "Reference to the UI customization configuration.", - "$ref": "../../type/entityReference.json" - }, "users": { "description": "Users that are assigned a persona.", "$ref": "../../type/entityReferenceList.json", diff --git a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/automatedTask/setGlossaryTermStatusTask.json b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/automatedTask/setGlossaryTermStatusTask.json index a7543b37f0d8..16d4d4dff26c 100644 --- a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/automatedTask/setGlossaryTermStatusTask.json +++ b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/automatedTask/setGlossaryTermStatusTask.json @@ -36,7 +36,7 @@ "$ref": "../../../../../entity/data/glossaryTerm.json#/definitions/status" } }, - "required": ["status"], + "required": [], "additionalProperties": false }, "input": { diff --git a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/userTask/userApprovalTask.json b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/userTask/userApprovalTask.json index fe4ae4636a7c..b175fba1d6e8 100644 --- a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/userTask/userApprovalTask.json +++ b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/userTask/userApprovalTask.json @@ -40,13 +40,6 @@ "description": "Add the Reviewers to the assignees List.", "type": "boolean", "default": false - }, - "extraAssignees": { - "description": "Manually add Specific Assignees.", - "type": "array", - "items": { - "$ref": "../../../../../type/entityReference.json" - } } } } diff --git a/openmetadata-spec/src/main/resources/json/schema/jobs/backgroundJob.json b/openmetadata-spec/src/main/resources/json/schema/jobs/backgroundJob.json new file mode 100644 index 000000000000..f5e95e69d8fe --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/jobs/backgroundJob.json @@ -0,0 +1,50 @@ +{ + "$id": "https://open-metadata.org/schema/jobs/backgroundJob.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "BackgroundJob", + "description": "Defines a background job that is triggered on insertion of new record in background_jobs table.", + "javaType": "org.openmetadata.schema.jobs.BackgroundJob", + "type": "object", + "properties": { + "id": { + "existingJavaType": "java.lang.Long", + "description": "Unique identifier for the job. This field is auto-incremented." + }, + "jobType": { + "type": "string", + "enum": ["CUSTOM_PROPERTY_ENUM_CLEANUP"], + "description": "Type of the job." + }, + "methodName": { + "type": "string", + "description": "JobHandler name of the method that will be executed for this job." + }, + "jobArgs": { + "oneOf": [ + { + "$ref": "./enumCleanupArgs.json" + } + ], + "description": "Object containing job arguments." + }, + "status": { + "type": "string", + "enum": ["COMPLETED", "FAILED", "RUNNING","PENDING"], + "description": "Current status of the job." + }, + "createdBy": { + "type": "string", + "description": "User or Bot who triggered the background job." + }, + "createdAt": { + "description": "Timestamp when the job was created in Unix epoch time milliseconds.", + "$ref": "../type/basic.json#/definitions/timestamp" + }, + "updatedAt": { + "description": "Time when job was last updated in Unix epoch time milliseconds.", + "$ref": "../type/basic.json#/definitions/timestamp" + } + }, + "required": ["id", "jobType", "methodName", "jobArgs", "status", "createdBy", "createdAt", "updatedAt"], + "additionalProperties": false +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/jobs/enumCleanupArgs.json b/openmetadata-spec/src/main/resources/json/schema/jobs/enumCleanupArgs.json new file mode 100644 index 000000000000..2c8c6246f308 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/jobs/enumCleanupArgs.json @@ -0,0 +1,26 @@ +{ + "$id": "https://open-metadata.org/schema/jobs/enumCleanupArgs.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "EnumCleanupArgs", + "description": "Arguments for enum removal job.", + "type": "object", + "properties": { + "propertyName": { + "type": "string", + "description": "Name of the property." + }, + "removedEnumKeys": { + "type": "array", + "items": { + "type": "string" + }, + "description": "List of removed enum keys." + }, + "entityType": { + "type": "string", + "description": "Type of the entity." + } + }, + "required": ["propertyName", "removedEnumKeys", "entityType"], + "additionalProperties": false +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/databaseServiceQueryLineagePipeline.json b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/databaseServiceQueryLineagePipeline.json index 490a49d1822b..d53b197af74c 100644 --- a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/databaseServiceQueryLineagePipeline.json +++ b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/databaseServiceQueryLineagePipeline.json @@ -91,6 +91,12 @@ "default": 1, "title": "Number of Threads", "minimum": 1 + }, + "enableTempTableLineage": { + "title": "Enable Temp Table Lineage", + "description": "Handle Lineage for Snowflake Temporary and Transient Tables. ", + "type": "boolean", + "default": false } }, "additionalProperties": false diff --git a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dbtPipeline.json b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dbtPipeline.json index 985d16aab572..55b9a9c56c7b 100644 --- a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dbtPipeline.json +++ b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dbtPipeline.json @@ -43,11 +43,21 @@ } ] }, + "searchAcrossDatabases": { + "description": "Optional configuration to search across databases for tables or not", + "type": "boolean", + "default": false + }, "dbtUpdateDescriptions": { "description": "Optional configuration to update the description from DBT or not", "type": "boolean", "default": false }, + "dbtUpdateOwners": { + "description": "Optional configuration to update the owners from DBT or not", + "type": "boolean", + "default": false + }, "includeTags": { "description": "Optional configuration to toggle the tags ingestion.", "type": "boolean", diff --git a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/searchServiceMetadataPipeline.json b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/searchServiceMetadataPipeline.json index 01e0f88f3666..5127fc30483b 100644 --- a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/searchServiceMetadataPipeline.json +++ b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/searchServiceMetadataPipeline.json @@ -21,37 +21,33 @@ "searchIndexFilterPattern": { "title": "Search Index Filter Pattern", "description": "Regex to only fetch search indexes that matches the pattern.", - "$ref": "../type/filterPattern.json#/definitions/filterPattern", - "title": "Search Index Filter Pattern" + "$ref": "../type/filterPattern.json#/definitions/filterPattern" }, "markDeletedSearchIndexes": { "title": "Mark Deleted Search Indexes", "description": "Optional configuration to soft delete search indexes in OpenMetadata if the source search indexes are deleted. Also, if the search index is deleted, all the associated entities like lineage, etc., with that search index will be deleted", "type": "boolean", - "default": true, - "title": "Mark Deleted Search Index" + "default": true }, "includeSampleData": { "title": "Include Sample Data", "description": "Optional configuration to turn off fetching sample data for search index.", "type": "boolean", - "default": true, - "title": "Include Sample Data" + "default": true }, "sampleSize": { "title": "Sample Size", "description": "No. of records of sample data we want to ingest.", "default": 10, - "type": "integer", - "title": "Sample Size" + "type": "integer" }, - "overrideMetadata":{ + "overrideMetadata": { "title": "Override Metadata", "description": "Set the 'Override Metadata' toggle to control whether to override the existing metadata in the OpenMetadata server with the metadata fetched from the source. If the toggle is set to true, the metadata fetched from the source will override the existing metadata in the OpenMetadata server. If the toggle is set to false, the metadata fetched from the source will not override the existing metadata in the OpenMetadata server. This is applicable for fields like description, tags, owner and displayName", "type": "boolean", "default": false }, - "includeIndexTemplate":{ + "includeIndexTemplate": { "title": "Include Index Template", "description": "Enable the 'Include Index Template' toggle to manage the ingestion of index template data.", "type": "boolean", diff --git a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/storage/containerMetadataConfig.json b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/storage/containerMetadataConfig.json index 24942df3b962..0cd65ceba3da 100644 --- a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/storage/containerMetadataConfig.json +++ b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/storage/containerMetadataConfig.json @@ -31,6 +31,12 @@ }, "default": null }, + "depth":{ + "title": "Depth", + "description": "Depth of the data path in the container", + "type": "integer", + "default": 0 + }, "separator": { "title": "Separator", "description": "For delimited files such as CSV, what is the separator being used?", diff --git a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/testSuitePipeline.json b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/testSuitePipeline.json index 9ca076b5f734..f43ca42a1d30 100644 --- a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/testSuitePipeline.json +++ b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/testSuitePipeline.json @@ -10,6 +10,21 @@ "type": "string", "enum": ["TestSuite"], "default": "TestSuite" + }, + "serviceConnections": { + "description": "Service connections available for the logical test suite.", + "type": "object", + "properties": { + "serviceName": { + "type": "string" + }, + "serviceConnection": { + "description": "Connection configuration for the source. ex: mysql , tableau connection.", + "$ref": "../entity/services/connections/serviceConnection.json#/definitions/serviceConnection" + } + }, + "additionalProperties": false, + "required": ["serviceName", "serviceConnection"] } }, "properties": { @@ -19,9 +34,17 @@ "default": "TestSuite" }, "entityFullyQualifiedName": { - "description": "Fully qualified name of the entity to be tested.", + "description": "Fully qualified name of the entity to be tested, if we're working with a basic suite.", "$ref": "../type/basic.json#/definitions/fullyQualifiedEntityName" }, + "serviceConnections": { + "description": "Service connections to be used for the logical test suite.", + "type": "array", + "items": { + "$ref": "#/definitions/serviceConnections" + }, + "default": null + }, "profileSample": { "description": "Percentage of data or no. of rows we want to execute the profiler and tests on", "type": "number", @@ -45,6 +68,6 @@ "default": null } }, - "required": ["type", "entityFullyQualifiedName"], + "required": ["type"], "additionalProperties": false } diff --git a/openmetadata-spec/src/main/resources/json/schema/settings/settings.json b/openmetadata-spec/src/main/resources/json/schema/settings/settings.json index 0811f5e8fc8f..5a98c9d9552a 100644 --- a/openmetadata-spec/src/main/resources/json/schema/settings/settings.json +++ b/openmetadata-spec/src/main/resources/json/schema/settings/settings.json @@ -32,7 +32,8 @@ "profilerConfiguration", "searchSettings", "assetCertificationSettings", - "lineageSettings" + "lineageSettings", + "workflowSettings" ] } }, @@ -84,6 +85,9 @@ }, { "$ref": "../configuration/lineageSettings.json" + }, + { + "$ref": "../configuration/workflowSettings.json" } ] } diff --git a/openmetadata-spec/src/main/resources/json/schema/system/ui/navigationItem.json b/openmetadata-spec/src/main/resources/json/schema/system/ui/navigationItem.json deleted file mode 100644 index 60bb7a44088a..000000000000 --- a/openmetadata-spec/src/main/resources/json/schema/system/ui/navigationItem.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "$id": "https://open-metadata.org/schema/system/ui/navigationItem.json", - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "NavigationItem", - "description": "Defines a navigation item in the UI navigation menu.", - "type": "object", - "javaType": "org.openmetadata.schema.system.ui.NavigationItem", - "properties": { - "id": { - "description": "Unique identifier for the navigation item.", - "$ref": "../../type/basic.json#/definitions/uuid" - }, - "title": { - "description": "Display title of the navigation item.", - "type": "string" - }, - "pageId": { - "description": "Reference to a Page ID that this navigation item links to.", - "type": "string" - }, - "order": { - "description": "Order of the navigation item in the menu.", - "type": "integer" - }, - "children": { - "description": "Optional sub-navigation items.", - "type": "array", - "items": { "$ref": "#" }, - "default": [] - } - }, - "required": ["id", "title", "pageId", "order"], - "additionalProperties": false -} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/system/ui/page.json b/openmetadata-spec/src/main/resources/json/schema/system/ui/page.json index c2f2f247fe28..803050d5d88b 100644 --- a/openmetadata-spec/src/main/resources/json/schema/system/ui/page.json +++ b/openmetadata-spec/src/main/resources/json/schema/system/ui/page.json @@ -12,19 +12,19 @@ "type": "string", "enum": [ "LandingPage", - "Table", - "StoredProcedure", - "Database", - "DatabaseSchema", - "Topic", - "Pipeline", - "Dashboard", - "DashboardDataModel", - "Container", - "SearchIndex", - "Glossary", - "GlossaryTerm", - "Domain" + "TableLandingPage", + "StoredProcedureLandingPage", + "DatabaseLandingPage", + "DatabaseSchemaLandingPage", + "TopicLandingPage", + "PipelineLandingPage", + "DashboardLandingPage", + "DashboardDataModelLandingPage", + "ContainerLandingPage", + "SearchIndexLandingPage", + "GlossaryLandingPage", + "GlossaryTermLandingPage", + "DomainLandingPage" ] } }, @@ -41,12 +41,6 @@ "description": "Configuration for the Knowledge Panel.", "type": "object" }, - "tabs": { - "description": "Tabs included in this page.", - "type": "array", - "items": { "$ref": "tab.json" }, - "default": [] - }, "persona": { "description": "Persona this page belongs to.", "$ref": "../../type/entityReference.json" diff --git a/openmetadata-spec/src/main/resources/json/schema/system/ui/tab.json b/openmetadata-spec/src/main/resources/json/schema/system/ui/tab.json deleted file mode 100644 index 93805686b474..000000000000 --- a/openmetadata-spec/src/main/resources/json/schema/system/ui/tab.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "$id": "https://open-metadata.org/schema/system/ui/tab.json", - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Tab", - "description": "This schema defines a Tab within a Page.", - "type": "object", - "javaType": "org.openmetadata.schema.system.ui.Tab", - "properties": { - "id": { "$ref": "../../type/basic.json#/definitions/uuid" }, - "name": { - "description": "Name of the tab.", - "type": "string" - }, - "displayName": { - "description": "DisplayName of the tab.", - "type": "string" - }, - "layout": { - "description": "Layout configuration for this tab.", - "type": "object" - }, - "editable": { - "description": "Weather tab can be edit by the user or not.", - "type": "boolean" - }, - "knowledgePanels": { - "description": "KnowledgePanels that are part of this Tab.", - "$ref": "../../type/entityReferenceList.json" - } - }, - "required": ["id", "name", "layout"], - "additionalProperties": false -} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/system/ui/uiCustomization.json b/openmetadata-spec/src/main/resources/json/schema/system/ui/uiCustomization.json deleted file mode 100644 index bd32c3b90b4c..000000000000 --- a/openmetadata-spec/src/main/resources/json/schema/system/ui/uiCustomization.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "$id": "https://open-metadata.org/schema/system/ui/uiCustomization.json", - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "UICustomization", - "description": "Contains UI customization details for a Persona.", - "type": "object", - "javaType": "org.openmetadata.schema.system.ui.UICustomization", - "properties": { - "id": { "$ref": "../../type/basic.json#/definitions/uuid" }, - "name": { - "description": "A unique name for the UI customization configuration.", - "$ref": "../../type/basic.json#/definitions/entityName" - }, - "displayName": { - "description": "Name used for display purposes.", - "type": "string" - }, - "description": { - "description": "Description of the UI customization.", - "$ref": "../../type/basic.json#/definitions/markdown" - }, - "pages": { - "description": "List of Pages in the UI customization.", - "type": "array", - "items": { "$ref": "page.json" } - }, - "navigation": { - "description": "Site-wide navigation configuration.", - "type": "array", - "items": { "$ref": "navigationItem.json" } - }, - "updatedAt": { - "$ref": "../../type/basic.json#/definitions/timestamp" - }, - "updatedBy": { "type": "string" }, - "version": { - "$ref": "../../type/entityHistory.json#/definitions/entityVersion" - }, - "changeDescription": { - "$ref": "../../type/entityHistory.json#/definitions/changeDescription" - }, - "href": { "$ref": "../../type/basic.json#/definitions/href" } - }, - "required": ["id", "name", "pages"], - "additionalProperties": false -} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/tests/testCase.json b/openmetadata-spec/src/main/resources/json/schema/tests/testCase.json index 9de6c5b776c9..7d79359093a5 100644 --- a/openmetadata-spec/src/main/resources/json/schema/tests/testCase.json +++ b/openmetadata-spec/src/main/resources/json/schema/tests/testCase.json @@ -56,11 +56,12 @@ "type": "string" }, "testSuite": { - "description": "Test Suite that this test case belongs to.", + "description": "Basic Test Suite that this test case belongs to.", "$ref": "../type/entityReference.json" }, "testSuites": { "type": "array", + "description": "Basic and Logical Test Suites this test case belongs to", "items": { "$ref": "./testSuite.json" } diff --git a/openmetadata-spec/src/main/resources/json/schema/tests/testSuite.json b/openmetadata-spec/src/main/resources/json/schema/tests/testSuite.json index 51d46d165325..87d7ab558d13 100644 --- a/openmetadata-spec/src/main/resources/json/schema/tests/testSuite.json +++ b/openmetadata-spec/src/main/resources/json/schema/tests/testSuite.json @@ -75,7 +75,7 @@ "$ref": "../entity/services/connections/testConnectionResult.json" }, "pipelines": { - "description": "References to pipelines deployed for this database service to extract metadata, usage, lineage etc..", + "description": "References to pipelines deployed for this Test Suite to execute the tests.", "$ref": "../type/entityReferenceList.json", "default": null }, @@ -120,15 +120,25 @@ "type": "boolean", "default": false }, - "executable": { - "description": "Indicates if the test suite is executable. Set on the backend.", + "basic": { + "description": "Indicates if the test suite is basic, i.e., the parent suite of a test and linked to an entity. Set on the backend.", "type": "boolean", "default": false }, - "executableEntityReference": { - "description": "Entity reference the test suite is executed against. Only applicable if the test suite is executable.", + "executable": { + "description": "DEPRECATED in 1.6.2: Use 'basic'", + "type": "boolean", + "deprecated": true + }, + "basicEntityReference": { + "description": "Entity reference the test suite needs to execute the test against. Only applicable if the test suite is basic.", "$ref": "../type/entityReference.json" }, + "executableEntityReference": { + "description": "DEPRECATED in 1.6.2: Use 'basicEntityReference'.", + "$ref": "../type/entityReference.json", + "deprecated": true + }, "summary": { "description": "Summary of the previous day test cases execution for this test suite.", "$ref": "./basic.json#/definitions/testSummary" diff --git a/openmetadata-spec/src/main/resources/json/schema/type/basic.json b/openmetadata-spec/src/main/resources/json/schema/type/basic.json index 314954800e07..fd732ad7e1c7 100644 --- a/openmetadata-spec/src/main/resources/json/schema/type/basic.json +++ b/openmetadata-spec/src/main/resources/json/schema/type/basic.json @@ -173,6 +173,12 @@ ".{1,}": { "type": "string" } } }, + "map": { + "description": "A generic map that can be deserialized later.", + "existingJavaType" : "java.util.Map", + "type" : "object", + "additionalProperties": true + }, "status" : { "javaType": "org.openmetadata.schema.type.ApiStatus", "description": "State of an action over API.", diff --git a/openmetadata-spec/src/main/resources/json/schema/type/entityHierarchy.json b/openmetadata-spec/src/main/resources/json/schema/type/entityHierarchy.json index 73fbe9afce77..859b97024b95 100644 --- a/openmetadata-spec/src/main/resources/json/schema/type/entityHierarchy.json +++ b/openmetadata-spec/src/main/resources/json/schema/type/entityHierarchy.json @@ -7,38 +7,12 @@ "type": "object", "javaType": "org.openmetadata.schema.entity.data.EntityHierarchy", "definitions": { - "EntityHierarchy": { - "type": "object", - "properties": { - "id": { - "description": "Unique identifier of an entity hierarchy instance.", - "$ref": "../type/basic.json#/definitions/uuid" - }, - "name": { - "description": "Preferred name for the entity hierarchy.", - "$ref": "../type/basic.json#/definitions/entityName" - }, - "displayName": { - "description": "Display name that identifies this hierarchy.", - "type": "string" - }, - "description": { - "description": "Description of the entity hierarchy.", - "$ref": "../type/basic.json#/definitions/markdown" - }, - "fullyQualifiedName": { - "description": "A unique name that identifies an entity within the hierarchy. It captures name hierarchy in the form of `rootEntity.childEntity`.", - "$ref": "../type/basic.json#/definitions/fullyQualifiedEntityName" - }, - "children": { - "description": "Other entities that are children of this entity.", - "type": "array", - "items": { - "$ref": "#/definitions/EntityHierarchy" - } - } + "entityHierarchyList": { + "type": "array", + "items": { + "$ref": "entityHierarchy.json" }, - "required": ["id", "name", "description"] + "default": [] } }, "properties": { @@ -64,12 +38,8 @@ }, "children": { "description": "Other entities that are children of this entity.", - "type": "array", - "items": { - "$ref": "#/definitions/EntityHierarchy" - } + "$ref" : "#/definitions/entityHierarchyList" } }, - "required": ["id", "name", "description"], - "additionalProperties": false + "required": ["id", "name", "description"] } diff --git a/openmetadata-ui/pom.xml b/openmetadata-ui/pom.xml index 7dbbb62784df..710afbb0e2d3 100644 --- a/openmetadata-ui/pom.xml +++ b/openmetadata-ui/pom.xml @@ -5,7 +5,7 @@ platform org.open-metadata - 1.6.0-SNAPSHOT + 1.6.4 4.0.0 @@ -182,6 +182,7 @@ gz woff woff2 + ttf eot svg png diff --git a/openmetadata-ui/src/main/resources/ui/package.json b/openmetadata-ui/src/main/resources/ui/package.json index 162e0c2d705e..359b30ed6518 100644 --- a/openmetadata-ui/src/main/resources/ui/package.json +++ b/openmetadata-ui/src/main/resources/ui/package.json @@ -78,10 +78,10 @@ "classnames": "^2.3.1", "codemirror": "^5.65.16", "cookie-storage": "^6.1.0", - "cronstrue": "^1.122.0", + "cronstrue": "^2.53.0", "crypto-random-string-with-promisify-polyfill": "^5.0.0", "diff": "^5.0.0", - "dompurify": "^3.1.5", + "dompurify": "^3.2.4", "eventemitter3": "^5.0.1", "fast-json-patch": "^3.1.1", "history": "4.5.1", @@ -161,7 +161,6 @@ "@types/codemirror": "^0.0.104", "@types/dagre": "^0.7.47", "@types/diff": "^5.0.2", - "@types/dompurify": "^3.0.5", "@types/jest": "^26.0.23", "@types/katex": "^0.16.7", "@types/lodash": "^4.14.167", @@ -249,6 +248,8 @@ "terser-webpack-plugin": "5.1.1", "cookie": "0.7.0", "jsonpath-plus": "10.2.0", - "cross-spawn": "7.0.5" + "cross-spawn": "7.0.5", + "serialize-javascript": "6.0.2", + "dompurify": "3.2.4" } } diff --git a/openmetadata-ui/src/main/resources/ui/playwright.config.ts b/openmetadata-ui/src/main/resources/ui/playwright.config.ts index 4108f316d4af..c47a3699213b 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright.config.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright.config.ts @@ -63,6 +63,11 @@ export default defineConfig({ { name: 'setup', testMatch: '**/*.setup.ts', + teardown: 'restore-policies', + }, + { + name: 'restore-policies', + testMatch: '**/auth.teardown.ts', }, { name: 'chromium', diff --git a/openmetadata-ui/src/main/resources/ui/playwright/constant/conditionalPermissions.ts b/openmetadata-ui/src/main/resources/ui/playwright/constant/conditionalPermissions.ts new file mode 100644 index 000000000000..bf32712c3fe2 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/playwright/constant/conditionalPermissions.ts @@ -0,0 +1,174 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { PolicyClass } from '../support/access-control/PoliciesClass'; +import { RolesClass } from '../support/access-control/RolesClass'; +import { ApiCollectionClass } from '../support/entity/ApiCollectionClass'; +import { ContainerClass } from '../support/entity/ContainerClass'; +import { DashboardClass } from '../support/entity/DashboardClass'; +import { MlModelClass } from '../support/entity/MlModelClass'; +import { PipelineClass } from '../support/entity/PipelineClass'; +import { SearchIndexClass } from '../support/entity/SearchIndexClass'; +import { TableClass } from '../support/entity/TableClass'; +import { TopicClass } from '../support/entity/TopicClass'; +import { EntityData } from '../support/interfaces/ConditionalPermissions.interface'; +import { UserClass } from '../support/user/UserClass'; +import { ServiceTypes } from './settings'; + +export const isOwnerPolicy = new PolicyClass(); +export const matchAnyTagPolicy = new PolicyClass(); +export const isOwnerRole = new RolesClass(); +export const matchAnyTagRole = new RolesClass(); +export const userWithOwnerPermission = new UserClass(); +export const userWithTagPermission = new UserClass(); +export const apiCollectionWithOwner = new ApiCollectionClass(); +export const apiCollectionWithTag = new ApiCollectionClass(); +export const containerWithOwner = new ContainerClass(); +export const containerWithTag = new ContainerClass(); +export const dashboardWithOwner = new DashboardClass(); +export const dashboardWithTag = new DashboardClass(); +export const mlModelWithOwner = new MlModelClass(); +export const mlModelWithTag = new MlModelClass(); +export const pipelineWithOwner = new PipelineClass(); +export const pipelineWithTag = new PipelineClass(); +export const searchIndexWithOwner = new SearchIndexClass(); +export const searchIndexWithTag = new SearchIndexClass(); +export const tableWithOwner = new TableClass(); +export const tableWithTag = new TableClass(); +export const topicWithOwner = new TopicClass(); +export const topicWithTag = new TopicClass(); + +const withOwner = { + apiCollectionWithOwner, + containerWithOwner, + dashboardWithOwner, + mlModelWithOwner, + pipelineWithOwner, + searchIndexWithOwner, + tableWithOwner, + topicWithOwner, +}; + +const withTag = { + apiCollectionWithTag, + containerWithTag, + dashboardWithTag, + mlModelWithTag, + pipelineWithTag, + searchIndexWithTag, + tableWithTag, + topicWithTag, +}; + +export const assetsData = [ + { + asset: ServiceTypes.API_SERVICES, + withOwner: apiCollectionWithOwner, + withTag: apiCollectionWithTag, + childTabId: 'collections', + assetOwnerUrl: `/service/${apiCollectionWithOwner.serviceType}`, + assetTagUrl: `/service/${apiCollectionWithTag.serviceType}`, + }, + { + asset: ServiceTypes.STORAGE_SERVICES, + withOwner: containerWithOwner, + withTag: containerWithTag, + childTabId: 'containers', + assetOwnerUrl: `/service/${containerWithOwner.serviceType}`, + assetTagUrl: `/service/${containerWithTag.serviceType}`, + }, + { + asset: ServiceTypes.DASHBOARD_SERVICES, + withOwner: dashboardWithOwner, + withTag: dashboardWithTag, + childTabId: 'dashboards', + childTabId2: 'data-model', + childTableId2: 'data-models-table', + assetOwnerUrl: `/service/${dashboardWithOwner.serviceType}`, + assetTagUrl: `/service/${dashboardWithTag.serviceType}`, + }, + { + asset: ServiceTypes.ML_MODEL_SERVICES, + withOwner: mlModelWithOwner, + withTag: mlModelWithTag, + childTabId: 'ml models', + assetOwnerUrl: `/service/${mlModelWithOwner.serviceType}`, + assetTagUrl: `/service/${mlModelWithTag.serviceType}`, + }, + { + asset: ServiceTypes.PIPELINE_SERVICES, + withOwner: pipelineWithOwner, + withTag: pipelineWithTag, + childTabId: 'pipelines', + assetOwnerUrl: `/service/${pipelineWithOwner.serviceType}`, + assetTagUrl: `/service/${pipelineWithTag.serviceType}`, + }, + { + asset: ServiceTypes.SEARCH_SERVICES, + withOwner: searchIndexWithOwner, + withTag: searchIndexWithTag, + childTabId: 'search indexes', + assetOwnerUrl: `/service/${searchIndexWithOwner.serviceType}`, + assetTagUrl: `/service/${searchIndexWithTag.serviceType}`, + }, + { + asset: ServiceTypes.DATABASE_SERVICES, + withOwner: tableWithOwner, + withTag: tableWithTag, + childTabId: 'databases', + assetOwnerUrl: `/service/${tableWithOwner.serviceType}`, + assetTagUrl: `/service/${tableWithTag.serviceType}`, + }, + { + asset: ServiceTypes.MESSAGING_SERVICES, + withOwner: topicWithOwner, + withTag: topicWithTag, + childTabId: 'topics', + assetOwnerUrl: `/service/${topicWithOwner.serviceType}`, + assetTagUrl: `/service/${topicWithTag.serviceType}`, + }, + { + asset: 'database', + withOwner: tableWithOwner, + withTag: tableWithTag, + childTabId: 'schema', + assetOwnerUrl: `/database`, + assetTagUrl: `/database`, + }, + { + asset: 'databaseSchema', + withOwner: tableWithOwner, + withTag: tableWithTag, + childTabId: 'table', + assetOwnerUrl: `/databaseSchema`, + assetTagUrl: `/databaseSchema`, + }, + { + asset: 'container', + withOwner: containerWithOwner, + withTag: containerWithTag, + childTabId: 'children', + assetOwnerUrl: `/container`, + assetTagUrl: `/container`, + }, +]; + +export const conditionalPermissionsEntityData: EntityData = { + isOwnerPolicy, + matchAnyTagPolicy, + isOwnerRole, + matchAnyTagRole, + userWithOwnerPermission, + userWithTagPermission, + withOwner, + withTag, +}; diff --git a/openmetadata-ui/src/main/resources/ui/playwright/constant/login.ts b/openmetadata-ui/src/main/resources/ui/playwright/constant/login.ts index 1d2923e25127..d71e9ae86146 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/constant/login.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/constant/login.ts @@ -11,6 +11,7 @@ * limitations under the License. */ export const JWT_EXPIRY_TIME_MAP = { + '3 minutes': 180, '1 hour': 3600, '2 hours': 7200, '3 hours': 10800, diff --git a/openmetadata-ui/src/main/resources/ui/playwright/constant/permission.ts b/openmetadata-ui/src/main/resources/ui/playwright/constant/permission.ts index b4dccdce8030..66e87f4079a4 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/constant/permission.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/constant/permission.ts @@ -77,6 +77,45 @@ export const DATA_CONSUMER_RULES: PolicyRulesType[] = [ }, ]; +export const VIEW_ALL_RULE: PolicyRulesType[] = [ + { + name: 'OrganizationPolicy-ViewAll-Rule', + description: 'Allow all users to view all metadata', + resources: ['All'], + operations: ['ViewAll'], + effect: 'allow', + }, +]; + +export const VIEW_ALL_WITH_IS_OWNER: PolicyRulesType[] = [ + { + name: 'viewAll-IsOwner', + resources: ['All'], + operations: ['ViewAll'], + effect: 'allow', + condition: 'isOwner()', + }, +]; + +export const VIEW_ALL_WITH_MATCH_TAG_CONDITION: PolicyRulesType[] = [ + { + name: 'viewAll-MatchTag', + resources: ['All'], + operations: ['ViewAll'], + effect: 'allow', + condition: "matchAnyTag('PersonalData.Personal')", + }, +]; + +export const EDIT_USER_FOR_TEAM_RULES: PolicyRulesType[] = [ + { + name: 'EditUserTeams-EditRule', + resources: ['team'], + operations: ['EditUsers'], + effect: 'allow', + }, +]; + export const ORGANIZATION_POLICY_RULES: PolicyRulesType[] = [ { name: 'OrganizationPolicy-NoOwner-Rule', @@ -95,13 +134,6 @@ export const ORGANIZATION_POLICY_RULES: PolicyRulesType[] = [ resources: ['All'], condition: 'isOwner()', }, - { - name: 'OrganizationPolicy-ViewAll-Rule', - description: 'Allow all users to discover data assets.', - effect: 'allow', - operations: ['ViewAll'], - resources: ['All'], - }, ]; export const GLOBAL_SETTING_PERMISSIONS: Record< diff --git a/openmetadata-ui/src/main/resources/ui/playwright/constant/service.ts b/openmetadata-ui/src/main/resources/ui/playwright/constant/service.ts index 3d5bc91321c4..6f40d87c1e6d 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/constant/service.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/constant/service.ts @@ -11,7 +11,7 @@ * limitations under the License. */ import { uuid } from '../utils/common'; -import { GlobalSettingOptions } from './settings'; +import { GlobalSettingOptions, ServiceTypes } from './settings'; export const SERVICE_TYPE = { Database: GlobalSettingOptions.DATABASES, @@ -26,49 +26,38 @@ export const SERVICE_TYPE = { ApiService: GlobalSettingOptions.APIS, }; -export const SERVICE_CATEGORIES = { - DATABASE_SERVICES: 'databaseServices', - MESSAGING_SERVICES: 'messagingServices', - PIPELINE_SERVICES: 'pipelineServices', - DASHBOARD_SERVICES: 'dashboardServices', - ML_MODEL_SERVICES: 'mlmodelServices', - STORAGE_SERVICES: 'storageServices', - METADATA_SERVICES: 'metadataServices', - SEARCH_SERVICES: 'searchServices', -}; - export const VISIT_SERVICE_PAGE_DETAILS = { [SERVICE_TYPE.Database]: { settingsMenuId: GlobalSettingOptions.DATABASES, - serviceCategory: SERVICE_CATEGORIES.DATABASE_SERVICES, + serviceCategory: ServiceTypes.DATABASE_SERVICES, }, [SERVICE_TYPE.Messaging]: { settingsMenuId: GlobalSettingOptions.MESSAGING, - serviceCategory: SERVICE_CATEGORIES.MESSAGING_SERVICES, + serviceCategory: ServiceTypes.MESSAGING_SERVICES, }, [SERVICE_TYPE.Dashboard]: { settingsMenuId: GlobalSettingOptions.DASHBOARDS, - serviceCategory: SERVICE_CATEGORIES.DASHBOARD_SERVICES, + serviceCategory: ServiceTypes.DASHBOARD_SERVICES, }, [SERVICE_TYPE.Pipeline]: { settingsMenuId: GlobalSettingOptions.PIPELINES, - serviceCategory: SERVICE_CATEGORIES.PIPELINE_SERVICES, + serviceCategory: ServiceTypes.PIPELINE_SERVICES, }, [SERVICE_TYPE.MLModels]: { settingsMenuId: GlobalSettingOptions.MLMODELS, - serviceCategory: SERVICE_CATEGORIES.ML_MODEL_SERVICES, + serviceCategory: ServiceTypes.ML_MODEL_SERVICES, }, [SERVICE_TYPE.Storage]: { settingsMenuId: GlobalSettingOptions.STORAGES, - serviceCategory: SERVICE_CATEGORIES.STORAGE_SERVICES, + serviceCategory: ServiceTypes.STORAGE_SERVICES, }, [SERVICE_TYPE.Search]: { settingsMenuId: GlobalSettingOptions.SEARCH, - serviceCategory: SERVICE_CATEGORIES.SEARCH_SERVICES, + serviceCategory: ServiceTypes.SEARCH_SERVICES, }, [SERVICE_TYPE.Metadata]: { settingsMenuId: GlobalSettingOptions.METADATA, - serviceCategory: SERVICE_CATEGORIES.METADATA_SERVICES, + serviceCategory: ServiceTypes.METADATA_SERVICES, }, }; diff --git a/openmetadata-ui/src/main/resources/ui/playwright/constant/settings.ts b/openmetadata-ui/src/main/resources/ui/playwright/constant/settings.ts index 703f32e05358..2025a11dc015 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/constant/settings.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/constant/settings.ts @@ -21,6 +21,18 @@ export enum GlobalSettingsMenuCategory { APPLICATIONS = 'apps', } +export enum ServiceTypes { + API_SERVICES = 'apiServices', + DATABASE_SERVICES = 'databaseServices', + MESSAGING_SERVICES = 'messagingServices', + PIPELINE_SERVICES = 'pipelineServices', + DASHBOARD_SERVICES = 'dashboardServices', + ML_MODEL_SERVICES = 'mlmodelServices', + STORAGE_SERVICES = 'storageServices', + METADATA_SERVICES = 'metadataServices', + SEARCH_SERVICES = 'searchServices', +} + export enum GlobalSettingOptions { USERS = 'users', ADMINS = 'admins', @@ -80,6 +92,10 @@ export enum GlobalSettingOptions { export const SETTINGS_OPTIONS_PATH = { // Services + [GlobalSettingOptions.API_COLLECTIONS]: [ + GlobalSettingsMenuCategory.SERVICES, + `${GlobalSettingsMenuCategory.SERVICES}.${GlobalSettingOptions.API_COLLECTIONS}`, + ], [GlobalSettingOptions.DATABASES]: [ GlobalSettingsMenuCategory.SERVICES, `${GlobalSettingsMenuCategory.SERVICES}.${GlobalSettingOptions.DATABASES}`, @@ -149,7 +165,10 @@ export const SETTINGS_OPTIONS_PATH = { GlobalSettingsMenuCategory.MEMBERS, `${GlobalSettingsMenuCategory.MEMBERS}.${GlobalSettingOptions.ADMINS}`, ], - [GlobalSettingOptions.PERSONA]: [GlobalSettingOptions.PERSONA], + [GlobalSettingOptions.PERSONA]: [ + GlobalSettingsMenuCategory.MEMBERS, + `${GlobalSettingsMenuCategory.MEMBERS}.${GlobalSettingOptions.PERSONA}`, + ], // Access Control @@ -164,6 +183,10 @@ export const SETTINGS_OPTIONS_PATH = { // Open-metadata + [GlobalSettingOptions.CUSTOMIZE_LANDING_PAGE]: [ + GlobalSettingsMenuCategory.PREFERENCES, + `${GlobalSettingsMenuCategory.PREFERENCES}.${GlobalSettingOptions.CUSTOMIZE_LANDING_PAGE}`, + ], [GlobalSettingOptions.EMAIL]: [ GlobalSettingsMenuCategory.PREFERENCES, `${GlobalSettingsMenuCategory.PREFERENCES}.${GlobalSettingOptions.EMAIL}`, diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/ActivityFeed.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/ActivityFeed.spec.ts index 954bd2c3f272..3327819b67f2 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/ActivityFeed.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/ActivityFeed.spec.ts @@ -381,7 +381,9 @@ test.describe('Activity feed', () => { // Task 1 - Resolved the task + const resolveTask2 = page.waitForResponse('/api/v1/feed/tasks/*/resolve'); await page.getByText('Accept Suggestion').click(); + await resolveTask2; await toastNotification(page, /Task resolved successfully/); @@ -590,6 +592,100 @@ test.describe('Activity feed', () => { } ); }); + + test('Check Task Filter in Landing Page Widget', async ({ browser }) => { + const { page: page1, afterAction: afterActionUser1 } = + await performUserLogin(browser, user1); + const { page: page2, afterAction: afterActionUser2 } = + await performUserLogin(browser, user2); + + await base.step('Create and Assign Task to User 2', async () => { + await redirectToHomePage(page1); + await entity.visitEntityPage(page1); + + // Create task for the user 2 + await page1.getByTestId('request-description').click(); + await createDescriptionTask(page1, { + term: entity.entity.displayName, + assignee: user2.responseData.name, + }); + + await afterActionUser1(); + }); + + await base.step('Create and Validate Task as per Filters', async () => { + await redirectToHomePage(page2); + await entity.visitEntityPage(page2); + + // Create task for the user 1 + await page2.getByTestId('request-entity-tags').click(); + await createTagTask(page2, { + term: entity.entity.displayName, + tag: 'PII.None', + assignee: user1.responseData.name, + }); + + await redirectToHomePage(page2); + const taskResponse = page2.waitForResponse( + '/api/v1/feed?type=Task&filterType=OWNER&taskStatus=Open&userId=*' + ); + + await page2 + .getByTestId('activity-feed-widget') + .getByText('Tasks') + .click(); + + await taskResponse; + + await expect( + page2.locator( + '[data-testid="activity-feed-widget"] [data-testid="no-data-placeholder"]' + ) + ).not.toBeVisible(); + + // Check the Task based on ALL task filter + await expect(page2.getByTestId('message-container')).toHaveCount(2); + + // Check the Task based on Assigned task filter + await page2.getByTestId('filter-button').click(); + await page2.waitForSelector('.ant-popover ', { state: 'visible' }); + + const taskAssignedResponse = page2.waitForResponse( + '/api/v1/feed?type=Task&filterType=ASSIGNED_TO&taskStatus=Open&userId=*' + ); + await page2.getByText('Assigned').click(); + await page2.getByTestId('selectable-list-update-btn').click(); + + await taskAssignedResponse; + + await expect(page2.getByTestId('message-container')).toHaveCount(1); + + await expect(page2.getByTestId('owner-link')).toContainText( + user2.responseData.displayName + ); + + // Check the Task based on Created by me task filter + + await page2.getByTestId('filter-button').click(); + await page2.waitForSelector('.ant-popover ', { state: 'visible' }); + + const taskCreatedByResponse = page2.waitForResponse( + '/api/v1/feed?type=Task&filterType=ASSIGNED_BY&taskStatus=Open&userId=*' + ); + await page2.getByText('Created By').click(); + await page2.getByTestId('selectable-list-update-btn').click(); + + await taskCreatedByResponse; + + await expect(page2.getByTestId('message-container')).toHaveCount(1); + + await expect(page2.getByTestId('owner-link')).toContainText( + user1.responseData.displayName + ); + + await afterActionUser2(); + }); + }); }); base.describe('Activity feed with Data Consumer User', () => { diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/AdvanceSearchCustomProperty.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/AdvanceSearchCustomProperty.spec.ts new file mode 100644 index 000000000000..1642742a5a22 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/AdvanceSearchCustomProperty.spec.ts @@ -0,0 +1,140 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import test, { expect } from '@playwright/test'; +import { CUSTOM_PROPERTIES_ENTITIES } from '../../constant/customProperty'; +import { GlobalSettingOptions } from '../../constant/settings'; +import { SidebarItem } from '../../constant/sidebar'; +import { TableClass } from '../../support/entity/TableClass'; +import { + selectOption, + showAdvancedSearchDialog, +} from '../../utils/advancedSearch'; +import { advanceSearchSaveFilter } from '../../utils/advancedSearchCustomProperty'; +import { createNewPage, redirectToHomePage, uuid } from '../../utils/common'; +import { addCustomPropertiesForEntity } from '../../utils/customProperty'; +import { settingClick, sidebarClick } from '../../utils/sidebar'; + +// use the admin user to login +test.use({ storageState: 'playwright/.auth/admin.json' }); + +test.describe('Advanced Search Custom Property', () => { + const table = new TableClass(); + const durationPropertyName = `pwCustomPropertyDurationTest${uuid()}`; + const durationPropertyValue = 'PT1H30M'; + + test.beforeAll('Setup pre-requests', async ({ browser }) => { + const { apiContext, afterAction } = await createNewPage(browser); + await table.create(apiContext); + await afterAction(); + }); + + test.afterAll('Cleanup', async ({ browser }) => { + const { apiContext, afterAction } = await createNewPage(browser); + await table.delete(apiContext); + await afterAction(); + }); + + test('Create, Assign and Test Advance Search for Duration', async ({ + page, + }) => { + test.slow(true); + + await redirectToHomePage(page); + + await test.step('Create and Assign Custom Property Value', async () => { + await settingClick(page, GlobalSettingOptions.TABLES, true); + + await addCustomPropertiesForEntity({ + page, + propertyName: durationPropertyName, + customPropertyData: CUSTOM_PROPERTIES_ENTITIES['entity_table'], + customType: 'Duration', + }); + + await table.visitEntityPage(page); + + await page.getByTestId('custom_properties').click(); // Tab Click + + await page + .getByTestId(`custom-property-${durationPropertyName}-card`) + .locator('svg') + .click(); // Add Custom Property Value + + await page.getByTestId('duration-input').fill(durationPropertyValue); + + const saveResponse = page.waitForResponse('/api/v1/tables/*'); + await page.getByTestId('inline-save-btn').click(); + await saveResponse; + }); + + await test.step('Verify Duration Type in Advance Search ', async () => { + await sidebarClick(page, SidebarItem.EXPLORE); + + await showAdvancedSearchDialog(page); + + const ruleLocator = page.locator('.rule').nth(0); + + // Perform click on rule field + await selectOption( + page, + ruleLocator.locator('.rule--field .ant-select'), + 'Custom Properties' + ); + + // Perform click on custom property type to filter + await selectOption( + page, + ruleLocator.locator('.rule--field .ant-select'), + durationPropertyName + ); + + const inputElement = ruleLocator.locator( + '.rule--widget--TEXT input[type="text"]' + ); + + await inputElement.fill(durationPropertyValue); + + await advanceSearchSaveFilter(page, durationPropertyValue); + + await expect( + page.getByTestId( + `table-data-card_${table.entityResponseData.fullyQualifiedName}` + ) + ).toBeVisible(); + + // Check around the Partial Search Value + const partialSearchValue = durationPropertyValue.slice(0, 3); + + await page.getByTestId('advance-search-filter-btn').click(); + + await expect(page.locator('[role="dialog"].ant-modal')).toBeVisible(); + + // Perform click on operator + await selectOption( + page, + ruleLocator.locator('.rule--operator .ant-select'), + 'Contains' + ); + + await inputElement.fill(partialSearchValue); + + await advanceSearchSaveFilter(page, partialSearchValue); + + await expect( + page.getByTestId( + `table-data-card_${table.entityResponseData.fullyQualifiedName}` + ) + ).toBeVisible(); + }); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/AdvancedSearch.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/AdvancedSearch.spec.ts index 3dda28a884d6..8f04b37c7e93 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/AdvancedSearch.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/AdvancedSearch.spec.ts @@ -12,9 +12,10 @@ */ import test from '@playwright/test'; import { SidebarItem } from '../../constant/sidebar'; +import { EntityDataClass } from '../../support/entity/EntityDataClass'; import { TableClass } from '../../support/entity/TableClass'; -import { TopicClass } from '../../support/entity/TopicClass'; -import { TagClass } from '../../support/tag/TagClass'; +import { Glossary } from '../../support/glossary/Glossary'; +import { GlossaryTerm } from '../../support/glossary/GlossaryTerm'; import { UserClass } from '../../support/user/UserClass'; import { FIELDS, @@ -23,91 +24,208 @@ import { verifyAllConditions, } from '../../utils/advancedSearch'; import { createNewPage, redirectToHomePage } from '../../utils/common'; -import { addMultiOwner, assignTag, assignTier } from '../../utils/entity'; +import { assignTier } from '../../utils/entity'; import { sidebarClick } from '../../utils/sidebar'; +test.describe.configure({ + // 4 minutes to avoid test timeout happening some times in AUTs + timeout: 4 * 60 * 1000, +}); + +const user = new UserClass(); +const table = new TableClass(undefined, 'Regular'); +let glossaryEntity: Glossary; + test.describe('Advanced Search', { tag: '@advanced-search' }, () => { // use the admin user to login test.use({ storageState: 'playwright/.auth/admin.json' }); - const user1 = new UserClass(); - const user2 = new UserClass(); - const table1 = new TableClass(); - const table2 = new TableClass(); - const topic1 = new TopicClass(); - const topic2 = new TopicClass(); - const tierTag1 = new TagClass({ classification: 'Tier' }); - const tierTag2 = new TagClass({ classification: 'Tier' }); - let searchCriteria: Record = {}; test.beforeAll('Setup pre-requests', async ({ browser }) => { - test.setTimeout(150000); - const { page, apiContext, afterAction } = await createNewPage(browser); - await Promise.all([ - user1.create(apiContext), - user2.create(apiContext), - table1.create(apiContext), - table2.create(apiContext), - topic1.create(apiContext), - topic2.create(apiContext), - tierTag1.create(apiContext), - tierTag2.create(apiContext), + await EntityDataClass.preRequisitesForTests(apiContext); + await user.create(apiContext); + glossaryEntity = new Glossary(undefined, [ + { + id: user.responseData.id, + type: 'user', + name: user.responseData.name, + displayName: user.responseData.displayName, + }, ]); + const glossaryTermEntity = new GlossaryTerm(glossaryEntity); + + await glossaryEntity.create(apiContext); + await glossaryTermEntity.create(apiContext); + await table.create(apiContext); // Add Owner & Tag to the table - await table1.visitEntityPage(page); - await addMultiOwner({ - page, - ownerNames: [user1.getUserName()], - activatorBtnDataTestId: 'edit-owner', - resultTestId: 'data-assets-header', - endpoint: table1.endpoint, - type: 'Users', + await EntityDataClass.table1.visitEntityPage(page); + await EntityDataClass.table1.patch({ + apiContext, + patchData: [ + { + op: 'add', + value: { + type: 'user', + id: EntityDataClass.user1.responseData.id, + }, + path: '/owners/0', + }, + { + op: 'add', + value: { + tagFQN: 'PersonalData.Personal', + }, + path: '/tags/0', + }, + { + op: 'add', + path: '/domain', + value: { + id: EntityDataClass.domain1.responseData.id, + type: 'domain', + name: EntityDataClass.domain1.responseData.name, + displayName: EntityDataClass.domain1.responseData.displayName, + }, + }, + ], }); - await assignTag(page, 'PersonalData.Personal'); - await table2.visitEntityPage(page); - await addMultiOwner({ - page, - ownerNames: [user2.getUserName()], - activatorBtnDataTestId: 'edit-owner', - resultTestId: 'data-assets-header', - endpoint: table1.endpoint, - type: 'Users', + await EntityDataClass.table2.visitEntityPage(page); + await EntityDataClass.table2.patch({ + apiContext, + patchData: [ + { + op: 'add', + value: { + type: 'user', + id: EntityDataClass.user2.responseData.id, + }, + path: '/owners/0', + }, + { + op: 'add', + value: { + tagFQN: 'PII.None', + }, + path: '/tags/0', + }, + { + op: 'add', + path: '/domain', + value: { + id: EntityDataClass.domain2.responseData.id, + type: 'domain', + name: EntityDataClass.domain2.responseData.name, + displayName: EntityDataClass.domain2.responseData.displayName, + }, + }, + ], }); - await assignTag(page, 'PII.None'); // Add Tier To the topic 1 - await topic1.visitEntityPage(page); - await assignTier(page, tierTag1.data.displayName, topic1.endpoint); + await EntityDataClass.topic1.visitEntityPage(page); + await assignTier( + page, + EntityDataClass.tierTag1.data.displayName, + EntityDataClass.topic1.endpoint + ); // Add Tier To the topic 2 - await topic2.visitEntityPage(page); - await assignTier(page, tierTag2.data.displayName, topic2.endpoint); + await EntityDataClass.topic2.visitEntityPage(page); + await assignTier( + page, + EntityDataClass.tierTag2.data.displayName, + EntityDataClass.topic2.endpoint + ); // Update Search Criteria here searchCriteria = { - 'owners.displayName.keyword': [user1.getUserName(), user2.getUserName()], + 'owners.displayName.keyword': [ + EntityDataClass.user1.getUserName(), + EntityDataClass.user2.getUserName(), + ], 'tags.tagFQN': ['PersonalData.Personal', 'PII.None'], 'tier.tagFQN': [ - tierTag1.responseData.fullyQualifiedName, - tierTag2.responseData.fullyQualifiedName, + EntityDataClass.tierTag1.responseData.fullyQualifiedName, + EntityDataClass.tierTag2.responseData.fullyQualifiedName, + ], + 'service.displayName.keyword': [ + EntityDataClass.table1.service.name, + EntityDataClass.table2.service.name, ], - 'service.displayName.keyword': [table1.service.name, table2.service.name], 'database.displayName.keyword': [ - table1.database.name, - table2.database.name, + EntityDataClass.table1.database.name, + EntityDataClass.table2.database.name, ], 'databaseSchema.displayName.keyword': [ - table1.schema.name, - table2.schema.name, + EntityDataClass.table1.schema.name, + EntityDataClass.table2.schema.name, + ], + 'columns.name.keyword': [ + EntityDataClass.table1.entity.columns[2].name, + EntityDataClass.table2.entity.columns[3].name, ], - 'columns.name.keyword': ['email', 'shop_id'], 'displayName.keyword': [ - table1.entity.displayName, - table2.entity.displayName, + EntityDataClass.table1.entity.displayName, + EntityDataClass.table2.entity.displayName, + ], + serviceType: [ + EntityDataClass.table1.service.serviceType, + EntityDataClass.topic1.service.serviceType, + ], + 'messageSchema.schemaFields.name.keyword': [ + EntityDataClass.topic1.entity.messageSchema.schemaFields[0].name, + EntityDataClass.topic2.entity.messageSchema.schemaFields[1].name, + ], + 'dataModel.columns.name.keyword': [ + EntityDataClass.container1.entity.dataModel.columns[0].name, + EntityDataClass.container2.entity.dataModel.columns[1].name, + ], + dataModelType: [ + EntityDataClass.dashboard1.dataModel.dataModelType, + EntityDataClass.dashboard2.dataModel.dataModelType, + ], + 'fields.name.keyword': [ + EntityDataClass.searchIndex1.entity.fields[1].name, + EntityDataClass.searchIndex2.entity.fields[3].name, + ], + 'tasks.displayName.keyword': [ + EntityDataClass.pipeline1.entity.tasks[0].displayName, + EntityDataClass.pipeline2.entity.tasks[1].displayName, + ], + 'domain.displayName.keyword': [ + EntityDataClass.domain1.data.displayName, + EntityDataClass.domain2.data.displayName, + ], + 'responseSchema.schemaFields.name.keyword': [ + EntityDataClass.apiCollection1.apiEndpoint.responseSchema + .schemaFields[0].name, + EntityDataClass.apiCollection2.apiEndpoint.responseSchema + .schemaFields[1].name, + ], + 'requestSchema.schemaFields.name.keyword': [ + EntityDataClass.apiCollection1.apiEndpoint.requestSchema.schemaFields[0] + .name, + EntityDataClass.apiCollection2.apiEndpoint.requestSchema.schemaFields[1] + .name, + ], + 'name.keyword': [ + EntityDataClass.table1.entity.name, + EntityDataClass.table2.entity.name, + ], + 'project.keyword': [ + EntityDataClass.dashboardDataModel1.entity.project, + EntityDataClass.dashboardDataModel2.entity.project, + ], + status: ['Approved', 'In Review'], + tableType: [table.entity.tableType, 'MaterializedView'], + entityType: ['dashboard', 'mlmodel'], + 'charts.displayName.keyword': [ + EntityDataClass.dashboard1.charts.displayName, + EntityDataClass.dashboard2.charts.displayName, ], }; @@ -116,16 +234,10 @@ test.describe('Advanced Search', { tag: '@advanced-search' }, () => { test.afterAll('Cleanup', async ({ browser }) => { const { apiContext, afterAction } = await createNewPage(browser); - await Promise.all([ - user1.delete(apiContext), - user2.delete(apiContext), - table1.delete(apiContext), - table2.delete(apiContext), - topic1.delete(apiContext), - topic2.delete(apiContext), - tierTag1.delete(apiContext), - tierTag2.delete(apiContext), - ]); + await EntityDataClass.postRequisitesForTests(apiContext); + await glossaryEntity.delete(apiContext); + await user.delete(apiContext); + await table.delete(apiContext); await afterAction(); }); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/CronValidations.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/CronValidations.spec.ts new file mode 100644 index 000000000000..788ad7a5840b --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/CronValidations.spec.ts @@ -0,0 +1,182 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { expect, Page, test } from '@playwright/test'; +import { GlobalSettingOptions } from '../../constant/settings'; +import { redirectToHomePage } from '../../utils/common'; +import { settingClick } from '../../utils/sidebar'; + +const inputCronExpression = async (page: Page, cron: string) => { + await page + .locator('[data-testid="cron-container"] #schedular-form_cron') + .click(); + await page + .locator('[data-testid="cron-container"] #schedular-form_cron') + .clear(); + await page + .locator('[data-testid="cron-container"] #schedular-form_cron') + .fill(cron); +}; + +// use the admin user to login +test.use({ + storageState: 'playwright/.auth/admin.json', +}); + +test.describe('Cron Validations', () => { + test('Validate different cron expressions', async ({ page }) => { + await redirectToHomePage(page); + + await settingClick(page, GlobalSettingOptions.APPLICATIONS); + + const applicationResponse = page.waitForResponse( + '/api/v1/apps/name/SearchIndexingApplication/status?offset=0&limit=1' + ); + + await page + .locator( + '[data-testid="search-indexing-application-card"] [data-testid="config-btn"]' + ) + .click(); + + await applicationResponse; + + await page.click('[data-testid="edit-button"]'); + await page.waitForSelector('[data-testid="schedular-card-container"]'); + await page + .getByTestId('schedular-card-container') + .getByText('Schedule', { exact: true }) + .click(); + + await page + .getByTestId('time-dropdown-container') + .getByTestId('cron-type') + .click(); + + await page.click('.ant-select-dropdown:visible [title="Custom"]'); + + await page.waitForSelector( + '[data-testid="cron-container"] #schedular-form_cron' + ); + + // Check Valid Crons + + // Check '0 0 * * *' to be valid + await inputCronExpression(page, '0 0 * * *'); + + await expect( + page.getByTestId('cron-container').getByText('At 12:00 AM, every day') + ).toBeAttached(); + await expect(page.locator('#schedular-form_cron_help')).not.toBeAttached(); + + // Check '0 0 1/3 * * 1' to be valid + await inputCronExpression(page, '0 0 1/3 * * 1'); + + await expect( + page + .getByTestId('cron-container') + .getByText( + 'At 0 minutes past the hour, every 3 hours, starting at 01:00 AM, only on Monday' + ) + ).toBeAttached(); + await expect(page.locator('#schedular-form_cron_help')).not.toBeAttached(); + + // Check '0 0 * * 1-6' to be valid + await inputCronExpression(page, '0 0 * * 1-6'); + + await expect( + page + .getByTestId('cron-container') + .getByText('At 12:00 AM, Monday through Saturday') + ).toBeAttached(); + await expect(page.locator('#schedular-form_cron_help')).not.toBeAttached(); + + // Check Invalid crons + + // Check every minute frequency throws an error + await inputCronExpression(page, '0/1 0 * * *'); + + await expect( + page + .locator('#schedular-form_cron_help') + .getByText( + 'Cron schedule too frequent. Please choose at least 1-hour intervals.' + ) + ).toBeAttached(); + + // Check every second frequency throws an error + await inputCronExpression(page, '0/1 0 * * * 1'); + + await expect( + page + .locator('#schedular-form_cron_help') + .getByText( + 'Cron schedule too frequent. Please choose at least 1-hour intervals.' + ) + ).toBeAttached(); + + // Check '0 0 * * 7' to be invalid + await inputCronExpression(page, '0 0 * * 7'); + + await expect( + page + .locator('#schedular-form_cron_help') + .getByText('DOW part must be >= 0 and <= 6') + ).toBeAttached(); + + // Check '0 0 * * 1 7' to be invalid + await inputCronExpression(page, '0 0 * * 1 7'); + + await expect( + page + .locator('#schedular-form_cron_help') + .getByText('DOW part must be >= 0 and <= 6') + ).toBeAttached(); + + // Check '0 0 * * 1 7 67' to be invalid + await inputCronExpression(page, '0 0 * * 1 7 67'); + + await expect( + page + .locator('#schedular-form_cron_help') + .getByText('DOW part must be >= 0 and <= 6') + ).toBeAttached(); + + // Check '0 0 * * 0-7' to be invalid + await inputCronExpression(page, '0 0 * * 0-7'); + + await expect( + page + .locator('#schedular-form_cron_help') + .getByText('DOW part must be >= 0 and <= 6') + ).toBeAttached(); + + // Check '0 0 * * 7-9' to be invalid + await inputCronExpression(page, '0 0 * * 7-9'); + + await expect( + page + .locator('#schedular-form_cron_help') + .getByText('DOW part must be >= 0 and <= 6') + ).toBeAttached(); + + // Check '0 0 * * -1-9' to be invalid + await inputCronExpression(page, '0 0 * * -1-9'); + + await expect( + page + .locator('#schedular-form_cron_help') + .getByText('Error: DOW part must be >= 0 and <= 6') + ).toBeAttached(); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/ExploreQuickFilters.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/ExploreQuickFilters.spec.ts index 3c941f8d775e..e166c74b501f 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/ExploreQuickFilters.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/ExploreQuickFilters.spec.ts @@ -10,12 +10,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import test from '@playwright/test'; +import test, { expect } from '@playwright/test'; import { SidebarItem } from '../../constant/sidebar'; import { Domain } from '../../support/domain/Domain'; import { TableClass } from '../../support/entity/TableClass'; import { assignDomain, + clickOutside, createNewPage, redirectToHomePage, } from '../../utils/common'; @@ -91,7 +92,7 @@ test('should search for empty or null filters', async ({ page }) => { } }); -test('should search for multiple values alongwith null filters', async ({ +test('should search for multiple values along with null filters', async ({ page, }) => { const items = [ @@ -111,3 +112,37 @@ test('should search for multiple values alongwith null filters', async ({ await selectNullOption(page, filter); } }); + +test('should persist quick filter on global search', async ({ page }) => { + const items = [{ label: 'Owners', key: 'owners.displayName.keyword' }]; + + for (const filter of items) { + await selectNullOption(page, filter, false); + } + + const waitForSearchResponse = page.waitForResponse( + '/api/v1/search/query?q=*index=dataAsset*' + ); + + await page + .getByTestId('searchBox') + .fill(table.entityResponseData.fullyQualifiedName); + await waitForSearchResponse; + + await clickOutside(page); + + // expect the quick filter to be persisted + await expect( + page.getByRole('button', { name: 'Owners : No Owners' }) + ).toBeVisible(); + + await page.getByTestId('searchBox').click(); + await page.keyboard.down('Enter'); + + await page.waitForLoadState('networkidle'); + + // expect the quick filter to be persisted + await expect( + page.getByRole('button', { name: 'Owners : No Owners' }) + ).toBeVisible(); +}); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/GlossaryPagination.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/GlossaryPagination.spec.ts new file mode 100644 index 000000000000..e3256e339d80 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/GlossaryPagination.spec.ts @@ -0,0 +1,69 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import test, { expect } from '@playwright/test'; +import { SidebarItem } from '../../constant/sidebar'; +import { Glossary } from '../../support/glossary/Glossary'; +import { getApiContext, redirectToHomePage } from '../../utils/common'; +import { sidebarClick } from '../../utils/sidebar'; + +test.use({ + storageState: 'playwright/.auth/admin.json', +}); + +test.describe('Glossary tests', () => { + test.beforeEach(async ({ page }) => { + await redirectToHomePage(page); + }); + + test('should check for glossary term pagination', async ({ page }) => { + test.slow(true); + + const { apiContext, afterAction } = await getApiContext(page); + const glossaries = []; + for (let i = 0; i < 60; i++) { + const glossary = new Glossary(`Z_PW_GLOSSARY_TEST_${i + 1}`); + await glossary.create(apiContext); + glossaries.push(glossary); + } + + try { + await redirectToHomePage(page); + const glossaryRes = page.waitForResponse('/api/v1/glossaries?*'); + await sidebarClick(page, SidebarItem.GLOSSARY); + await glossaryRes; + + const glossaryAfterRes = page.waitForResponse( + '/api/v1/glossaries?*after=*' + ); + await page + .getByTestId('glossary-left-panel-scroller') + .scrollIntoViewIfNeeded(); + + const res = await glossaryAfterRes; + const json = await res.json(); + + const firstGlossaryName = json.data[0].displayName; + + await expect( + page + .getByTestId('glossary-left-panel') + .getByRole('menuitem', { name: firstGlossaryName }) + ).toBeVisible(); + } finally { + for (const glossary of glossaries) { + await glossary.delete(apiContext); + } + await afterAction(); + } + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/IncidentManager.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/IncidentManager.spec.ts index b7b5a7f38fbd..b00da9cc961d 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/IncidentManager.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/IncidentManager.spec.ts @@ -15,6 +15,7 @@ import { PLAYWRIGHT_INGESTION_TAG_OBJ } from '../../constant/config'; import { SidebarItem } from '../../constant/sidebar'; import { TableClass } from '../../support/entity/TableClass'; import { UserClass } from '../../support/user/UserClass'; +import { resetTokenFromBotPage } from '../../utils/bot'; import { createNewPage, descriptionBox, @@ -47,6 +48,12 @@ test.describe('Incident Manager', PLAYWRIGHT_INGESTION_TAG_OBJ, () => { const { afterAction, apiContext, page } = await createNewPage(browser); + // Todo: Remove this patch once the issue is fixed #19140 + await resetTokenFromBotPage(page, { + name: 'testsuite', + testId: 'bot-link-TestSuiteBot', + }); + for (const user of users) { await user.create(apiContext); } diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/QueryEntity.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/QueryEntity.spec.ts index b16719709108..b35180a99e1b 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/QueryEntity.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/QueryEntity.spec.ts @@ -92,6 +92,7 @@ test('Query Entity', async ({ page }) => { await page .locator('div') .filter({ hasText: new RegExp(`^${queryData.queryUsedIn.table1}$`) }) + .first() .click(); await clickOutside(page); @@ -105,7 +106,9 @@ test('Query Entity', async ({ page }) => { }); await test.step('Update owner, description and tag', async () => { - const ownerListResponse = page.waitForResponse('/api/v1/users?*'); + const ownerListResponse = page.waitForResponse( + '/api/v1/search/query?q=*isBot:false*index=user_search_index*' + ); await page .getByTestId( 'entity-summary-resizable-right-panel-container entity-resizable-panel-container' @@ -140,7 +143,7 @@ test('Query Entity', async ({ page }) => { // Update Description await page.click(`[data-testid="edit-description"]`); - await page.fill(descriptionBox, 'updated description'); + await page.locator(descriptionBox).fill('updated description'); const updateDescriptionResponse = page.waitForResponse( (response) => response.url().includes('/api/v1/queries/') && @@ -178,7 +181,11 @@ test('Query Entity', async ({ page }) => { ); await page.keyboard.type(queryData.queryUsedIn.table2); await tableSearchResponse; - await page.click(`[title="${queryData.queryUsedIn.table2}"]`); + await page + .locator('div') + .filter({ hasText: new RegExp(`^${queryData.queryUsedIn.table2}$`) }) + .first() + .click(); await clickOutside(page); const updateQueryResponse = page.waitForResponse( (response) => diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/Table.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/Table.spec.ts index c908c3bbd674..2bcabd03624d 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/Table.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/Table.spec.ts @@ -86,4 +86,10 @@ test.describe('Table pagination sorting search scenarios ', () => { await expect(page.getByTestId('search-error-placeholder')).toBeVisible(); }); + + test('Table page should show schema tab with count', async ({ page }) => { + await table1.visitEntityPage(page); + + await expect(page.getByRole('tab', { name: 'Schema' })).toContainText('4'); + }); }); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/TableConstraint.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/TableConstraint.spec.ts index 9678b28c4301..4491f4b3aa61 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/TableConstraint.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/TableConstraint.spec.ts @@ -31,6 +31,11 @@ test.use({ storageState: 'playwright/.auth/admin.json' }); const table = new TableClass(); test.describe('Table Constraints', {}, () => { + const columnName1 = table.children[0].name; + const columnName2 = table.children[1].name; + const columnName3 = table.children[2].name; + const columnName4 = table.children[3].name; + test.beforeAll('Prerequisite', async ({ browser }) => { const { apiContext, afterAction } = await createNewPage(browser); await table.create(apiContext); @@ -76,23 +81,21 @@ test.describe('Table Constraints', {}, () => { await page .getByTestId('primary-constraint-type-select') - .locator('div') - .nth(1) - .type('user_id'); + .getByRole('combobox') + .fill(columnName1, { force: true }); // select 1st value from dropdown - const firstPrimaryKeyColumn = page.getByTitle('user_id'); + const firstPrimaryKeyColumn = page.getByTitle(columnName1); await firstPrimaryKeyColumn.hover(); await firstPrimaryKeyColumn.click(); // select 2nd value from dropdown await page .getByTestId('primary-constraint-type-select') - .locator('div') - .nth(1) - .type('shop_id'); + .getByRole('combobox') + .fill(columnName2, { force: true }); - const secondPrimaryKeyColumn = page.getByTitle('shop_id'); + const secondPrimaryKeyColumn = page.getByTitle(columnName2); await secondPrimaryKeyColumn.hover(); await secondPrimaryKeyColumn.click(); await clickOutside(page); @@ -100,7 +103,7 @@ test.describe('Table Constraints', {}, () => { await expect( page .getByTestId('primary-constraint-type-select') - .getByText('user_idshop_id') + .getByText(`${columnName1}${columnName2}`) ).toBeVisible(); // Foreign Key Constraint Section @@ -170,29 +173,31 @@ test.describe('Table Constraints', {}, () => { await page .getByTestId('unique-constraint-type-select') - .locator('div') - .nth(1) - .type('name'); + .getByRole('combobox') + .fill(columnName3, { force: true }); // select 1st value from dropdown - const firstUniqueKeyColumn = page.getByTitle('name', { exact: true }); + const firstUniqueKeyColumn = page.getByTitle(columnName3, { + exact: true, + }); await firstUniqueKeyColumn.hover(); await firstUniqueKeyColumn.click(); // select 2nd value from dropdown await page .getByTestId('unique-constraint-type-select') - .locator('div') - .nth(1) - .type('email'); + .getByRole('combobox') + .fill(columnName4, { force: true }); - const secondUniqueKeyColumn = page.getByTitle('email'); + const secondUniqueKeyColumn = page.getByTitle(columnName4); await secondUniqueKeyColumn.hover(); await secondUniqueKeyColumn.click(); await clickOutside(page); await expect( - page.getByTestId('unique-constraint-type-select').getByText('nameemail') + page + .getByTestId('unique-constraint-type-select') + .getByText(`${columnName3}${columnName4}`) ).toBeVisible(); // Dist Constraint Section @@ -206,23 +211,21 @@ test.describe('Table Constraints', {}, () => { await page .getByTestId('dist-constraint-type-select') - .locator('div') - .nth(1) - .type('user_id'); + .getByRole('combobox') + .fill(columnName1, { force: true }); // select 1st value from dropdown - const firstDistKeyColumn = page.getByTitle('user_id'); + const firstDistKeyColumn = page.getByTitle(columnName1); await firstDistKeyColumn.hover(); await firstDistKeyColumn.click(); // select 2nd value from dropdown await page .getByTestId('dist-constraint-type-select') - .locator('div') - .nth(1) - .type('shop_id'); + .getByRole('combobox') + .fill(columnName2, { force: true }); - const secondDistKeyColumn = page.getByTitle('shop_id'); + const secondDistKeyColumn = page.getByTitle(columnName2); await secondDistKeyColumn.hover(); await secondDistKeyColumn.click(); await clickOutside(page); @@ -230,7 +233,7 @@ test.describe('Table Constraints', {}, () => { await expect( page .getByTestId('dist-constraint-type-select') - .getByText('user_idshop_id') + .getByText(`${columnName1}${columnName2}`) ).toBeVisible(); // Sort Constraint Section @@ -244,29 +247,29 @@ test.describe('Table Constraints', {}, () => { await page .getByTestId('sort-constraint-type-select') - .locator('div') - .nth(1) - .type('name'); + .getByRole('combobox') + .fill(columnName3, { force: true }); // select 1st value from dropdown - const firstSortKeyColumn = page.getByTitle('name', { exact: true }); + const firstSortKeyColumn = page.getByTitle(columnName3, { exact: true }); await firstSortKeyColumn.hover(); await firstSortKeyColumn.click(); // select 2nd value from dropdown await page .getByTestId('sort-constraint-type-select') - .locator('div') - .nth(1) - .type('email'); + .getByRole('combobox') + .fill(columnName4, { force: true }); - const secondSortKeyColumn = page.getByTitle('email'); + const secondSortKeyColumn = page.getByTitle(columnName4); await secondSortKeyColumn.hover(); await secondSortKeyColumn.click(); await clickOutside(page); await expect( - page.getByTestId('sort-constraint-type-select').getByText('nameemail') + page + .getByTestId('sort-constraint-type-select') + .getByText(`${columnName3}${columnName4}`) ).toBeVisible(); const saveResponse = page.waitForResponse('/api/v1/tables/*'); @@ -288,31 +291,31 @@ test.describe('Table Constraints', {}, () => { // Verify Primary Key await expect(page.getByTestId('PRIMARY_KEY-container')).toContainText( - 'shop_iduser_id' + `${columnName2}${columnName1}` ); await expect(page.getByTestId('PRIMARY_KEY-icon')).toBeVisible(); // Verify Foreign Key await expect(page.getByTestId('FOREIGN_KEY-container')).toContainText( - `user_id${table.additionalEntityTableResponseData[0]?.['columns'][1].fullyQualifiedName}` + `${columnName1}${table.additionalEntityTableResponseData[0]?.['columns'][1].fullyQualifiedName}` ); await expect(page.getByTestId('FOREIGN_KEY-icon')).toBeVisible(); // Verify Unique Key await expect(page.getByTestId('UNIQUE-container')).toContainText( - 'emailname' + `${columnName4}${columnName3}` ); await expect(page.getByTestId('UNIQUE-icon')).toBeVisible(); // Verify Sort Key await expect(page.getByTestId('SORT_KEY-container')).toContainText( - 'emailname' + `${columnName4}${columnName3}` ); await expect(page.getByTestId('SORT_KEY-icon')).toBeVisible(); // Verify Dist Key await expect(page.getByTestId('DIST_KEY-container')).toContainText( - 'shop_iduser_id' + `${columnName2}${columnName1}` ); await expect(page.getByTestId('DIST_KEY-icon')).toBeVisible(); }); @@ -345,11 +348,11 @@ test.describe('Table Constraints', {}, () => { // Verify Sort and Dist Key to be available await expect(page.getByTestId('SORT_KEY-container')).toContainText( - 'emailname' + `${columnName4}${columnName3}` ); await expect(page.getByTestId('SORT_KEY-icon')).toBeVisible(); await expect(page.getByTestId('DIST_KEY-container')).toContainText( - 'shop_iduser_id' + `${columnName2}${columnName1}` ); // Remove the pending constraints diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/Topic.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/Topic.spec.ts new file mode 100644 index 000000000000..4fab3aa230f2 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/Topic.spec.ts @@ -0,0 +1,50 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import test, { expect } from '@playwright/test'; +import { TopicClass } from '../../support/entity/TopicClass'; +import { createNewPage, redirectToHomePage } from '../../utils/common'; + +// use the admin user to login +test.use({ storageState: 'playwright/.auth/admin.json' }); + +const topic = new TopicClass(); + +test.slow(true); + +test.describe('Topic entity specific tests ', () => { + test.beforeAll('Setup pre-requests', async ({ browser }) => { + const { afterAction, apiContext } = await createNewPage(browser); + + await topic.create(apiContext); + + await afterAction(); + }); + + test.afterAll('Clean up', async ({ browser }) => { + const { afterAction, apiContext } = await createNewPage(browser); + + await topic.delete(apiContext); + + await afterAction(); + }); + + test.beforeEach('Visit home page', async ({ page }) => { + await redirectToHomePage(page); + }); + + test('Topic page should show schema tab with count', async ({ page }) => { + await topic.visitEntityPage(page); + + await expect(page.getByRole('tab', { name: 'Schema' })).toContainText('2'); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Flow/AddRoleAndAssignToUser.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Flow/AddRoleAndAssignToUser.spec.ts index 0158fc12adb7..96ee92ff9295 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Flow/AddRoleAndAssignToUser.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Flow/AddRoleAndAssignToUser.spec.ts @@ -53,7 +53,7 @@ test.describe.serial('Add role and assign it to the user', () => { await page.click('[data-testid="add-role"]'); await page.fill('[data-testid="name"]', roleName); - await page.fill(descriptionBox, `description for ${roleName}`); + await page.locator(descriptionBox).fill(`description for ${roleName}`); await page.click('[data-testid="policies"]'); await page.click('[title="Data Consumer Policy"]'); @@ -87,7 +87,7 @@ test.describe.serial('Add role and assign it to the user', () => { await page.fill('[data-testid="email"]', user.email); await page.fill('[data-testid="displayName"]', userDisplayName); - await page.fill(descriptionBox, 'Adding user'); + await page.locator(descriptionBox).fill('Adding user'); const generatePasswordResponse = page.waitForResponse( `/api/v1/users/generateRandomPwd` ); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Flow/ApiServiceRest.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Flow/ApiServiceRest.spec.ts index d9b46f3d5148..0f66d1dd1f05 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Flow/ApiServiceRest.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Flow/ApiServiceRest.spec.ts @@ -40,7 +40,7 @@ test.describe('API service', () => { // step 1 await page.getByTestId('service-name').fill(apiServiceConfig.name); - await page.fill(descriptionBox, apiServiceConfig.description); + await page.locator(descriptionBox).fill(apiServiceConfig.description); await page.getByTestId('next-button').click(); // step 2 diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Flow/ConditionalPermissions.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Flow/ConditionalPermissions.spec.ts new file mode 100644 index 000000000000..15965848cfe0 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Flow/ConditionalPermissions.spec.ts @@ -0,0 +1,113 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Page, test as base } from '@playwright/test'; +import { startCase } from 'lodash'; +import { + assetsData, + userWithOwnerPermission, + userWithTagPermission, +} from '../../constant/conditionalPermissions'; +import { performAdminLogin } from '../../utils/admin'; +import { + checkViewAllPermission, + conditionalPermissionsCleanup, + conditionalPermissionsPrerequisites, + getEntityFQN, +} from '../../utils/conditionalPermissions'; + +const test = base.extend<{ + user1Page: Page; + user2Page: Page; +}>({ + user1Page: async ({ browser }, use) => { + const page = await browser.newPage(); + await userWithOwnerPermission.login(page); + await use(page); + await page.close(); + }, + user2Page: async ({ browser }, use) => { + const page = await browser.newPage(); + await userWithTagPermission.login(page); + await use(page); + await page.close(); + }, +}); + +test.beforeAll(async ({ browser }) => { + test.slow(); + + const { apiContext, afterAction } = await performAdminLogin(browser); + await conditionalPermissionsPrerequisites(apiContext); + await afterAction(); +}); + +test.afterAll(async ({ browser }) => { + test.slow(); + + const { apiContext, afterAction } = await performAdminLogin(browser); + await conditionalPermissionsCleanup(apiContext); + await afterAction(); +}); + +for (const serviceData of assetsData) { + const { + asset, + withOwner, + withTag, + assetOwnerUrl, + assetTagUrl, + childTabId, + childTabId2, + childTableId2, + } = serviceData; + + test(`User with owner permission can only view owned ${startCase( + asset + )}`, async ({ user1Page: page }) => { + // Get the FQNs of both assets + const ownerAssetName = getEntityFQN(asset, withOwner); + const tagAssetName = getEntityFQN(asset, withTag); + + const ownerAssetURL = `${assetOwnerUrl}/${ownerAssetName}`; + const tagAssetURL = `${assetTagUrl}/${tagAssetName}`; + + await checkViewAllPermission({ + page, + url1: ownerAssetURL, + url2: tagAssetURL, + childTabId, + childTabId2, + childTableId2, + }); + }); + + test(`User with matchAnyTag permission can only view ${startCase( + asset + )} with the tag`, async ({ user2Page: page }) => { + // Get the FQNs of both assets + const ownerAssetName = getEntityFQN(asset, withOwner); + const tagAssetName = getEntityFQN(asset, withTag); + + const ownerAssetURL = `${assetOwnerUrl}/${ownerAssetName}`; + const tagAssetURL = `${assetTagUrl}/${tagAssetName}`; + + await checkViewAllPermission({ + page, + url1: tagAssetURL, + url2: ownerAssetURL, + childTabId, + childTabId2, + childTableId2, + }); + }); +} diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Flow/CustomizeLandingPage.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Flow/CustomizeLandingPage.spec.ts index b44f9ff31731..462deb32659a 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Flow/CustomizeLandingPage.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Flow/CustomizeLandingPage.spec.ts @@ -14,7 +14,7 @@ import { expect, Page, test as base } from '@playwright/test'; import { PersonaClass } from '../../support/persona/PersonaClass'; import { UserClass } from '../../support/user/UserClass'; import { performAdminLogin } from '../../utils/admin'; -import { redirectToHomePage, toastNotification } from '../../utils/common'; +import { redirectToHomePage } from '../../utils/common'; import { checkAllDefaultWidgets, navigateToCustomizeLandingPage, @@ -239,7 +239,11 @@ test.describe('Customize Landing Page Flow', () => { .click(); // Verify the toast notification - await toastNotification(adminPage, 'Page layout updated successfully.'); + const toastNotification = adminPage.locator('.Toastify__toast-body'); + + await expect(toastNotification).toContainText( + 'Page layout updated successfully.' + ); // Check if all widgets are present after resetting the layout await checkAllDefaultWidgets(adminPage, true); @@ -247,6 +251,9 @@ test.describe('Customize Landing Page Flow', () => { // Check if all widgets are present on landing page await redirectToHomePage(adminPage); + // Ensures the page is fully loaded + await adminPage.waitForLoadState('networkidle'); + await checkAllDefaultWidgets(adminPage); } ); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Flow/Navbar.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Flow/Navbar.spec.ts index a67465bb3cea..9512e14fff9f 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Flow/Navbar.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Flow/Navbar.spec.ts @@ -35,7 +35,7 @@ for (const searchItem of navbarSearchItems) { ); const searchRes = page.waitForResponse( - `/api/v1/search/query?q=*&index=${searchIndex}` + `/api/v1/search/query?q=*&index=${searchIndex}**` ); await page.getByTestId('searchBox').fill('dim'); await searchRes; diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Flow/PersonaFlow.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Flow/PersonaFlow.spec.ts index 7bd48c50c546..692b66700fd2 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Flow/PersonaFlow.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Flow/PersonaFlow.spec.ts @@ -102,7 +102,15 @@ test.describe.serial('Persona operations', () => { // Verify created persona details await expect( - page.getByTestId(`persona-details-card-${PERSONA_DETAILS.name}`) + page + .getByTestId('persona-details-card') + .getByText(PERSONA_DETAILS.displayName) + ).toBeVisible(); + + await expect( + page + .getByTestId('persona-details-card') + .getByText(PERSONA_DETAILS.description) ).toBeVisible(); const personaResponse = page.waitForResponse( @@ -112,7 +120,8 @@ test.describe.serial('Persona operations', () => { ); await page - .getByTestId(`persona-details-card-${PERSONA_DETAILS.name}`) + .locator('[data-testid="persona-details-card"]') + .getByText(PERSONA_DETAILS.displayName) .click(); await personaResponse; @@ -144,7 +153,8 @@ test.describe.serial('Persona operations', () => { page, }) => { await page - .getByTestId(`persona-details-card-${PERSONA_DETAILS.name}`) + .locator('[data-testid="persona-details-card"]') + .getByText(PERSONA_DETAILS.displayName) .click(); await page.getByTestId('edit-description').click(); @@ -170,7 +180,9 @@ test.describe.serial('Persona operations', () => { test('Persona rename flow should work properly', async ({ page }) => { await page - .getByTestId(`persona-details-card-${PERSONA_DETAILS.name}`) + .locator('[data-testid="persona-details-card"]') + .getByText(PERSONA_DETAILS.displayName) + .click(); await updatePersonaDisplayName({ page, displayName: 'Test Persona' }); @@ -191,7 +203,8 @@ test.describe.serial('Persona operations', () => { test('Remove users in persona should work properly', async ({ page }) => { await page - .getByTestId(`persona-details-card-${PERSONA_DETAILS.name}`) + .locator('[data-testid="persona-details-card"]') + .getByText(PERSONA_DETAILS.displayName) .click(); await page @@ -218,7 +231,8 @@ test.describe.serial('Persona operations', () => { test('Delete persona should work properly', async ({ page }) => { await page - .getByTestId(`persona-details-card-${PERSONA_DETAILS.name}`) + .locator('[data-testid="persona-details-card"]') + .getByText(PERSONA_DETAILS.displayName) .click(); await page.click('[data-testid="manage-button"]'); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/DataQualityAndProfiler.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/DataQualityAndProfiler.spec.ts index 6d767449500c..cc1a65106d88 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/DataQualityAndProfiler.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/DataQualityAndProfiler.spec.ts @@ -45,7 +45,7 @@ test.beforeAll(async ({ browser }) => { ); await table2.createTestCase(apiContext, { name: `email_column_values_to_be_in_set_${uuid()}`, - entityLink: `<#E::table::${table2.entityResponseData?.['fullyQualifiedName']}::columns::email>`, + entityLink: `<#E::table::${table2.entityResponseData?.['fullyQualifiedName']}::columns::${table2.entity?.columns[3].name}>`, parameterValues: [ { name: 'allowedValues', value: '["gmail","yahoo","collate"]' }, ], @@ -90,7 +90,7 @@ test('Table test case', PLAYWRIGHT_INGESTION_TAG_OBJ, async ({ page }) => { '#tableTestForm_params_columnName', NEW_TABLE_TEST_CASE.field ); - await page.fill(descriptionBox, NEW_TABLE_TEST_CASE.description); + await page.locator(descriptionBox).fill(NEW_TABLE_TEST_CASE.description); await page.click('[data-testid="submit-test"]'); await page.waitForSelector('[data-testid="success-line"]'); @@ -170,7 +170,7 @@ test('Column test case', PLAYWRIGHT_INGESTION_TAG_OBJ, async ({ page }) => { const NEW_COLUMN_TEST_CASE = { name: 'email_column_value_lengths_to_be_between', - column: 'email', + column: table1.entity?.columns[3].name, type: 'columnValueLengthsToBeBetween', label: 'Column Value Lengths To Be Between', min: '3', @@ -200,7 +200,7 @@ test('Column test case', PLAYWRIGHT_INGESTION_TAG_OBJ, async ({ page }) => { '#tableTestForm_params_maxLength', NEW_COLUMN_TEST_CASE.max ); - await page.fill(descriptionBox, NEW_COLUMN_TEST_CASE.description); + await page.locator(descriptionBox).fill(NEW_COLUMN_TEST_CASE.description); await page.click('[data-testid="submit-test"]'); await page.waitForSelector('[data-testid="success-line"]'); @@ -365,7 +365,9 @@ test( await expect(page.locator('#tableTestForm_table')).toHaveValue( table2.entityResponseData?.['name'] ); - await expect(page.locator('#tableTestForm_column')).toHaveValue('email'); + await expect(page.locator('#tableTestForm_column')).toHaveValue( + table2.entity?.columns[3].name + ); await expect(page.locator('#tableTestForm_name')).toHaveValue( testCaseName ); @@ -395,7 +397,7 @@ test( // Edit test case description await page.click(`[data-testid="edit-${testCaseName}"]`); - await page.fill(descriptionBox, 'Test case description'); + await page.locator(descriptionBox).fill('Test case description'); const updateTestCaseResponse2 = page.waitForResponse( (response) => response.url().includes('/api/v1/dataQuality/testCases/') && @@ -407,7 +409,11 @@ test( expect(body2).toEqual( JSON.stringify([ - { op: 'add', path: '/description', value: 'Test case description' }, + { + op: 'add', + path: '/description', + value: '

Test case description

', + }, ]) ); @@ -482,9 +488,9 @@ test( profileSample: '60', sampleDataCount: '100', profileQuery: 'select * from table', - excludeColumns: 'user_id', - includeColumns: 'shop_id', - partitionColumnName: 'name', + excludeColumns: table1.entity?.columns[0].name, + includeColumns: table1.entity?.columns[1].name, + partitionColumnName: table1.entity?.columns[2].name, partitionIntervalType: 'COLUMN-VALUE', partitionValues: 'test', }; @@ -557,13 +563,13 @@ test( expect(requestBody).toEqual( JSON.stringify({ - excludeColumns: ['user_id'], + excludeColumns: [table1.entity?.columns[0].name], profileQuery: 'select * from table', profileSample: 60, profileSampleType: 'PERCENTAGE', - includeColumns: [{ columnName: 'shop_id' }], + includeColumns: [{ columnName: table1.entity?.columns[1].name }], partitioning: { - partitionColumnName: 'name', + partitionColumnName: table1.entity?.columns[2].name, partitionIntervalType: 'COLUMN-VALUE', partitionValues: ['test'], enablePartitioning: true, @@ -633,7 +639,7 @@ test('TestCase filters', PLAYWRIGHT_INGESTION_TAG_OBJ, async ({ page }) => { await filterTable1.createTestSuiteAndPipelines(apiContext); const { testSuiteData: testSuite2Response } = await filterTable1.createTestSuiteAndPipelines(apiContext, { - executableEntityReference: filterTable2Response?.['fullyQualifiedName'], + basicEntityReference: filterTable2Response?.['fullyQualifiedName'], }); const testCaseResult = { @@ -937,10 +943,13 @@ test('TestCase filters', PLAYWRIGHT_INGESTION_TAG_OBJ, async ({ page }) => { await expect(page.locator('[value="tier"]')).not.toBeVisible(); // Apply domain globally - await page.locator('[data-testid="domain-dropdown"]').click(); + await page.getByTestId('domain-dropdown').click(); + await page - .locator(`li[data-menu-id*='${domain.responseData?.['name']}']`) + .getByTestId(`tag-${domain.responseData.fullyQualifiedName}`) .click(); + await page.getByTestId('saveAssociatedTag').click(); + await sidebarClick(page, SidebarItem.DATA_QUALITY); const getTestCaseList = page.waitForResponse( '/api/v1/dataQuality/testCases/search/list?*' diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Domains.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Domains.spec.ts index 4a6ee2337773..219790217f74 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Domains.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Domains.spec.ts @@ -10,7 +10,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import test, { expect } from '@playwright/test'; +import base, { expect, Page } from '@playwright/test'; import { Operation } from 'fast-json-patch'; import { get } from 'lodash'; import { SidebarItem } from '../../constant/sidebar'; @@ -34,16 +34,59 @@ import { selectDomain, selectSubDomain, setupAssetsForDomain, + verifyDataProductAssetsAfterDelete, verifyDomain, } from '../../utils/domain'; import { sidebarClick } from '../../utils/sidebar'; import { performUserLogin, visitUserProfilePage } from '../../utils/user'; -test.describe('Domains', () => { - test.use({ storageState: 'playwright/.auth/admin.json' }); +const user = new UserClass(); + +const domain = new Domain(); + +const test = base.extend<{ + page: Page; + userPage: Page; +}>({ + page: async ({ browser }, use) => { + const { page } = await performAdminLogin(browser); + await use(page); + await page.close(); + }, + userPage: async ({ browser }, use) => { + const page = await browser.newPage(); + await user.login(page); + await use(page); + await page.close(); + }, +}); +test.describe('Domains', () => { test.slow(true); + test.beforeAll('Setup pre-requests', async ({ browser }) => { + const { apiContext, afterAction } = await performAdminLogin(browser); + await user.create(apiContext); + + await domain.create(apiContext); + + await domain.patch({ + apiContext, + patchData: [ + { + op: 'add', + path: '/owners/0', + value: { + id: user.responseData.id, + type: 'user', + }, + }, + ], + }); + + await afterAction(); + }); + test.beforeEach('Visit home page', async ({ page }) => { await redirectToHomePage(page); }); @@ -61,7 +104,7 @@ test.describe('Domains', () => { await test.step('Add assets to domain', async () => { await redirectToHomePage(page); await sidebarClick(page, SidebarItem.DOMAIN); - await addAssetsToDomain(page, domain.data, assets); + await addAssetsToDomain(page, domain, assets); }); await test.step('Delete domain using delete modal', async () => { @@ -98,9 +141,13 @@ test.describe('Domains', () => { const dataProduct1 = new DataProduct(domain); const dataProduct2 = new DataProduct(domain); await domain.create(apiContext); - await sidebarClick(page, SidebarItem.DOMAIN); await page.reload(); - await addAssetsToDomain(page, domain.data, assets); + + await test.step('Add assets to domain', async () => { + await redirectToHomePage(page); + await sidebarClick(page, SidebarItem.DOMAIN); + await addAssetsToDomain(page, domain, assets); + }); await test.step('Create DataProducts', async () => { await selectDomain(page, domain.data); @@ -115,7 +162,11 @@ test.describe('Domains', () => { await redirectToHomePage(page); await sidebarClick(page, SidebarItem.DOMAIN); await selectDataProduct(page, domain.data, dataProduct1.data); - await addAssetsToDataProduct(page, dataProduct1.data, assets); + await addAssetsToDataProduct( + page, + dataProduct1.data.fullyQualifiedName ?? '', + assets + ); }); await test.step('Remove assets from DataProducts', async () => { @@ -189,11 +240,13 @@ test.describe('Domains', () => { await domain.create(apiContext); await page.reload(); await page.getByTestId('domain-dropdown').click(); + await page - .locator( - `[data-menu-id*="${domain.data.name}"] > .ant-dropdown-menu-title-content` - ) + .getByTestId(`tag-${domain.responseData.fullyQualifiedName}`) .click(); + await page.getByTestId('saveAssociatedTag').click(); + + await page.waitForLoadState('networkidle'); await redirectToHomePage(page); @@ -220,7 +273,7 @@ test.describe('Domains', () => { await domain.create(apiContext); await page.reload(); await sidebarClick(page, SidebarItem.DOMAIN); - await addAssetsToDomain(page, domain.data, assets); + await addAssetsToDomain(page, domain, assets); await page.getByTestId('documentation').click(); const updatedDomainName = 'PW Domain Updated'; @@ -264,6 +317,161 @@ test.describe('Domains', () => { await domain.delete(apiContext); await afterAction(); }); + + test('Should clear assets from data products after deletion of data product in Domain', async ({ + page, + }) => { + const { afterAction, apiContext } = await getApiContext(page); + const { assets, assetCleanup } = await setupAssetsForDomain(page); + const domain = new Domain({ + name: 'PW_Domain_Delete_Testing', + displayName: 'PW_Domain_Delete_Testing', + description: 'playwright domain description', + domainType: 'Aggregate', + fullyQualifiedName: 'PW_Domain_Delete_Testing', + }); + const dataProduct1 = new DataProduct(domain, 'PW_DataProduct_Sales'); + const dataProduct2 = new DataProduct(domain, 'PW_DataProduct_Finance'); + + const domain1 = new Domain({ + name: 'PW_Domain_Delete_Testing', + displayName: 'PW_Domain_Delete_Testing', + description: 'playwright domain description', + domainType: 'Aggregate', + fullyQualifiedName: 'PW_Domain_Delete_Testing', + }); + const newDomainDP1 = new DataProduct(domain1, 'PW_DataProduct_Sales'); + const newDomainDP2 = new DataProduct(domain1, 'PW_DataProduct_Finance'); + + try { + await domain.create(apiContext); + await dataProduct1.create(apiContext); + await dataProduct2.create(apiContext); + await sidebarClick(page, SidebarItem.DOMAIN); + await page.reload(); + await addAssetsToDomain(page, domain, assets); + await verifyDataProductAssetsAfterDelete(page, { + domain, + dataProduct1, + dataProduct2, + assets, + }); + + await test.step( + 'Delete domain & recreate the same domain and data product', + async () => { + await domain.delete(apiContext); + await domain1.create(apiContext); + await newDomainDP1.create(apiContext); + await newDomainDP2.create(apiContext); + await page.reload(); + await redirectToHomePage(page); + await sidebarClick(page, SidebarItem.DOMAIN); + await selectDataProduct(page, domain1.data, newDomainDP1.data); + await checkAssetsCount(page, 0); + await sidebarClick(page, SidebarItem.DOMAIN); + await selectDataProduct(page, domain1.data, newDomainDP2.data); + await checkAssetsCount(page, 0); + } + ); + } finally { + await newDomainDP1.delete(apiContext); + await newDomainDP2.delete(apiContext); + await domain1.delete(apiContext); + await assetCleanup(); + await afterAction(); + } + }); + + test('Should inherit owners and experts from parent domain', async ({ + page, + }) => { + const { afterAction, apiContext } = await getApiContext(page); + const user1 = new UserClass(); + const user2 = new UserClass(); + let domain; + let dataProduct; + try { + await user1.create(apiContext); + await user2.create(apiContext); + + domain = new Domain({ + name: 'PW_Domain_Inherit_Testing', + displayName: 'PW_Domain_Inherit_Testing', + description: 'playwright domain description', + domainType: 'Aggregate', + fullyQualifiedName: 'PW_Domain_Inherit_Testing', + owners: [ + { + name: user1.responseData.name, + type: 'user', + fullyQualifiedName: user1.responseData.fullyQualifiedName ?? '', + id: user1.responseData.id, + }, + ], + experts: [user2.responseData.name], + }); + dataProduct = new DataProduct(domain); + await domain.create(apiContext); + await dataProduct.create(apiContext); + + await page.reload(); + await redirectToHomePage(page); + + await sidebarClick(page, SidebarItem.DOMAIN); + await selectDomain(page, domain.data); + await selectDataProduct(page, domain.data, dataProduct.data); + + await expect( + page.getByTestId('domain-owner-name').getByTestId('owner-label') + ).toContainText(user1.responseData.displayName); + + await expect( + page.getByTestId('domain-expert-name').getByTestId('owner-label') + ).toContainText(user2.responseData.displayName); + } finally { + await dataProduct?.delete(apiContext); + await domain?.delete(apiContext); + await user1.delete(apiContext); + await user2.delete(apiContext); + await afterAction(); + } + }); + + test('Domain owner should able to edit description of domain', async ({ + page, + userPage, + }) => { + const { afterAction, apiContext } = await getApiContext(page); + try { + await sidebarClick(userPage, SidebarItem.DOMAIN); + await selectDomain(userPage, domain.data); + + await expect(userPage.getByTestId('edit-description')).toBeInViewport(); + + await userPage.getByTestId('edit-description').click(); + + await expect(userPage.getByTestId('editor')).toBeInViewport(); + + const descriptionInputBox = '.om-block-editor[contenteditable="true"]'; + + await userPage.fill(descriptionInputBox, 'test description'); + + await userPage.getByTestId('save').click(); + + await userPage.waitForTimeout(3000); + + const descriptionBox = '.om-block-editor[contenteditable="false"]'; + + await expect(userPage.locator(descriptionBox)).toHaveText( + 'test description' + ); + } finally { + await domain?.delete(apiContext); + await user.delete(apiContext); + } + await afterAction(); + }); }); test.describe('Domains Rbac', () => { @@ -343,13 +551,13 @@ test.describe('Domains Rbac', () => { await redirectToHomePage(page); await sidebarClick(page, SidebarItem.DOMAIN); await selectDomain(page, domain1.data); - await addAssetsToDomain(page, domain1.data, domainAssset1); + await addAssetsToDomain(page, domain1, domainAssset1); // Add assets to domain 2 await redirectToHomePage(page); await sidebarClick(page, SidebarItem.DOMAIN); await selectDomain(page, domain2.data); - await addAssetsToDomain(page, domain2.data, domainAssset2); + await addAssetsToDomain(page, domain2, domainAssset2); }); await test.step('User with access to multiple domains', async () => { @@ -360,14 +568,11 @@ test.describe('Domains Rbac', () => { .click(); await expect( - userPage - .getByRole('menuitem', { name: domain1.data.displayName }) - .locator('span') + userPage.getByTestId(`tag-${domain1.responseData.fullyQualifiedName}`) ).toBeVisible(); + await expect( - userPage - .getByRole('menuitem', { name: domain3.data.displayName }) - .locator('span') + userPage.getByTestId(`tag-${domain3.responseData.fullyQualifiedName}`) ).toBeVisible(); // Visit explore page and verify if domain is passed in the query @@ -380,7 +585,7 @@ test.describe('Domains Rbac', () => { const urlParams = new URLSearchParams(queryString); const qParam = urlParams.get('q'); - expect(qParam).toContain(`domain.fullyQualifiedName:`); + expect(qParam).toEqual(''); }); for (const asset of domainAssset2) { diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Entity.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Entity.spec.ts index 5e5294e3d321..f0aa8825e374 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Entity.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Entity.spec.ts @@ -10,7 +10,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { test } from '@playwright/test'; +import { expect, test } from '@playwright/test'; import { isUndefined } from 'lodash'; import { CustomPropertySupportedEntityList } from '../../constant/customProperty'; import { ApiEndpointClass } from '../../support/entity/ApiEndpointClass'; @@ -65,8 +65,9 @@ test.use({ storageState: 'playwright/.auth/admin.json' }); entities.forEach((EntityClass) => { const entity = new EntityClass(); const deleteEntity = new EntityClass(); + const entityName = entity.getType(); - test.describe(entity.getType(), () => { + test.describe(entityName, () => { test.beforeAll('Setup pre-requests', async ({ browser }) => { const { apiContext, afterAction } = await createNewPage(browser); @@ -115,7 +116,7 @@ entities.forEach((EntityClass) => { }, false ); - await removeDomain(page); + await removeDomain(page, EntityDataClass.domain1.responseData); } }); @@ -192,6 +193,16 @@ entities.forEach((EntityClass) => { ); }); + if (['Dashboard', 'Dashboard Data Model'].includes(entityName)) { + test(`${entityName} page should show the project name`, async ({ + page, + }) => { + await expect( + page.getByText((entity.entity as { project: string }).project) + ).toBeVisible(); + }); + } + test('Update description', async ({ page }) => { await entity.descriptionUpdate(page); }); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/ExploreTree.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/ExploreTree.spec.ts index 9db6cfb2d1c9..346131401181 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/ExploreTree.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/ExploreTree.spec.ts @@ -190,3 +190,28 @@ test.describe('Explore Tree scenarios ', () => { await afterAction(); }); }); + +test.describe('Explore page', () => { + test('Check the listing of tags', async ({ page }) => { + await page + .locator('div') + .filter({ hasText: /^Governance$/ }) + .locator('svg') + .first() + .click(); + + await expect(page.getByRole('tree')).toContainText('Glossaries'); + await expect(page.getByRole('tree')).toContainText('Tags'); + + const res = page.waitForResponse( + '/api/v1/search/query?q=&index=dataAsset*' + ); + // click on tags + await page.getByTestId('explore-tree-title-Tags').click(); + + const response = await res; + const jsonResponse = await response.json(); + + expect(jsonResponse.hits.hits.length).toBeGreaterThan(0); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Glossary.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Glossary.spec.ts index a8c9e8a8a474..096a56ce1e77 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Glossary.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Glossary.spec.ts @@ -43,10 +43,12 @@ import { approveTagsTask, assignTagToGlossaryTerm, changeTermHierarchyFromModal, + checkGlossaryTermDetails, clickSaveButton, confirmationDragAndDropGlossary, createDescriptionTaskForGlossary, createGlossary, + createGlossaryTerm, createGlossaryTerms, createTagTaskForGlossary, deleteGlossaryOrGlossaryTerm, @@ -259,7 +261,7 @@ test.describe('Glossary tests', () => { type: 'Users', }); - await assignTag(page, 'PersonalData.Personal', 'Add', 'tabs'); + await assignTag(page, 'PersonalData.Personal'); }); await test.step('Update Glossary Term', async () => { @@ -288,7 +290,7 @@ test.describe('Glossary tests', () => { page, 'PersonalData.Personal', 'Add', - 'glossary-term' + 'panel-container' ); }); } finally { @@ -302,7 +304,9 @@ test.describe('Glossary tests', () => { } }); - test('Add and Update Glossary Term', async ({ browser }) => { + test('Add, Update and Verify Data Glossary Term', async ({ browser }) => { + test.slow(true); + const { page, afterAction, apiContext } = await performAdminLogin(browser); const glossary1 = new Glossary(); const glossaryTerm1 = new GlossaryTerm(glossary1); @@ -314,7 +318,7 @@ test.describe('Glossary tests', () => { await glossaryTerm1.create(apiContext); await owner1.create(apiContext); await reviewer1.create(apiContext); - await await redirectToHomePage(page); + await redirectToHomePage(page); await sidebarClick(page, SidebarItem.GLOSSARY); await selectActiveGlossary(page, glossary1.data.displayName); @@ -356,6 +360,13 @@ test.describe('Glossary tests', () => { const ownerText = await termRow.locator(ownerSelector).textContent(); expect(ownerText).toBe(`${owner1.data.firstName}${owner1.data.lastName}`); + + await checkGlossaryTermDetails( + page, + glossaryTerm1.data, + owner1, + reviewer1 + ); } finally { await glossaryTerm1.delete(apiContext); await glossary1.delete(apiContext); @@ -511,7 +522,7 @@ test.describe('Glossary tests', () => { await patchRequest2; // Check if the terms are present - const glossaryContainer = await page.locator( + const glossaryContainer = page.locator( '[data-testid="entity-right-panel"] [data-testid="glossary-container"]' ); const glossaryContainerText = await glossaryContainer.innerText(); @@ -521,7 +532,7 @@ test.describe('Glossary tests', () => { // Check if the icons are present - const icons = await page.locator( + const icons = page.locator( '[data-testid="entity-right-panel"] [data-testid="glossary-container"] [data-testid="glossary-icon"]' ); @@ -571,7 +582,7 @@ test.describe('Glossary tests', () => { expect(tagSelectorText).toContain(glossaryTerm3.data.displayName); // Check if the icon is visible - const icon = await page.locator( + const icon = page.locator( '[data-testid="glossary-tags-0"] > [data-testid="tags-wrapper"] > [data-testid="glossary-container"] [data-testid="glossary-icon"]' ); @@ -583,7 +594,7 @@ test.describe('Glossary tests', () => { await goToAssetsTab(page, glossaryTerm3.data.displayName, 2); // Check if the selected asset are present - const assetContainer = await page.locator( + const assetContainer = page.locator( '[data-testid="table-container"] .assets-data-container' ); @@ -805,6 +816,8 @@ test.describe('Glossary tests', () => { test('Assign Glossary Term to entity and check assets', async ({ browser, }) => { + test.slow(true); + const { page, afterAction, apiContext } = await performAdminLogin(browser); const table = new TableClass(); const glossary1 = new Glossary(); @@ -1186,6 +1199,36 @@ test.describe('Glossary tests', () => { } }); + test('Add Glossary Term inside another Term', async ({ browser }) => { + const { page, afterAction, apiContext } = await performAdminLogin(browser); + const glossary1 = new Glossary(); + const glossaryTerm1 = new GlossaryTerm(glossary1); + const glossary2 = new Glossary(); + glossary2.data.terms = [new GlossaryTerm(glossary2)]; + + try { + await glossary1.create(apiContext); + await glossaryTerm1.create(apiContext); + await redirectToHomePage(page); + await sidebarClick(page, SidebarItem.GLOSSARY); + await selectActiveGlossary(page, glossary1.data.displayName); + await selectActiveGlossaryTerm(page, glossaryTerm1.data.displayName); + await page.getByTestId('terms').click(); + + await createGlossaryTerm( + page, + glossary2.data.terms[0].data, + 'Approved', + false, + true + ); + } finally { + await glossaryTerm1.delete(apiContext); + await glossary1.delete(apiContext); + await afterAction(); + } + }); + test.afterAll(async ({ browser }) => { const { afterAction, apiContext } = await performAdminLogin(browser); await user1.delete(apiContext); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Lineage.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Lineage.spec.ts index ce818ae4e9fb..2bc5cc55809f 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Lineage.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Lineage.spec.ts @@ -86,7 +86,8 @@ for (const EntityClass of entities) { test(`Lineage creation from ${defaultEntity.getType()} entity`, async ({ browser, }) => { - test.slow(true); + // 5 minutes to avoid test timeout happening some times in AUTs + test.setTimeout(300_000); const { page } = await createNewPage(browser); const { currentEntity, entities, cleanup } = await setupEntitiesForLineage( diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Login.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Login.spec.ts index 12eb7b164490..b5251478dbef 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Login.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Login.spec.ts @@ -11,26 +11,54 @@ * limitations under the License. */ import { expect, test } from '@playwright/test'; -import { LOGIN_ERROR_MESSAGE } from '../../constant/login'; +import { JWT_EXPIRY_TIME_MAP, LOGIN_ERROR_MESSAGE } from '../../constant/login'; +import { AdminClass } from '../../support/user/AdminClass'; import { UserClass } from '../../support/user/UserClass'; import { performAdminLogin } from '../../utils/admin'; +import { clickOutside, redirectToHomePage } from '../../utils/common'; +import { updateJWTTokenExpiryTime } from '../../utils/login'; +import { visitUserProfilePage } from '../../utils/user'; const user = new UserClass(); const CREDENTIALS = user.data; const invalidEmail = 'userTest@openmetadata.org'; const invalidPassword = 'testUsers@123'; +test.describe.configure({ + // 5 minutes max for refresh token tests + timeout: 5 * 60 * 1000, +}); + test.describe('Login flow should work properly', () => { test.afterAll('Cleanup', async ({ browser }) => { const { apiContext, afterAction, page } = await performAdminLogin(browser); const response = await page.request.get( `/api/v1/users/name/${user.getUserName()}` ); + + // reset token expiry to 4 hours + await updateJWTTokenExpiryTime(apiContext, JWT_EXPIRY_TIME_MAP['4 hours']); + user.responseData = await response.json(); await user.delete(apiContext); await afterAction(); }); + test.beforeAll( + 'Update token timer to be 3 minutes for new token created', + async ({ browser }) => { + const { apiContext, afterAction } = await performAdminLogin(browser); + + // update expiry for 3 mins + await updateJWTTokenExpiryTime( + apiContext, + JWT_EXPIRY_TIME_MAP['3 minutes'] + ); + + await afterAction(); + } + ); + test('Signup and Login with signed up credentials', async ({ page }) => { await page.goto('/'); @@ -111,4 +139,64 @@ test.describe('Login flow should work properly', () => { await page.getByRole('button', { name: 'Submit' }).click(); await page.locator('[data-testid="go-back-button"]').click(); }); + + test('Refresh should work', async ({ browser }) => { + const browserContext = await browser.newContext(); + const { apiContext, afterAction } = await performAdminLogin(browser); + const page1 = await browserContext.newPage(), + page2 = await browserContext.newPage(); + + const testUser = new UserClass(); + await testUser.create(apiContext); + + await test.step('Login and wait for refresh call is made', async () => { + // User login + + await testUser.login(page1); + await redirectToHomePage(page1); + await redirectToHomePage(page2); + + // Need to wait until refresh happens and update the storage + await page1.waitForTimeout(3 * 60 * 1000); + + await redirectToHomePage(page1); + + await visitUserProfilePage(page1, testUser.responseData.name); + await redirectToHomePage(page2); + await visitUserProfilePage(page2, testUser.responseData.name); + }); + + await afterAction(); + }); + + test('accessing app with expired token should do auto renew token', async ({ + browser, + }) => { + const browserContext = await browser.newContext(); + + // Create new page and validate access + const page1 = await browserContext.newPage(); + const page2 = await browserContext.newPage(); + + const admin = new AdminClass(); + await admin.login(page1); + + await redirectToHomePage(page1); + await page1.getByTestId('dropdown-profile').click(); + await page1.waitForLoadState('networkidle'); + await clickOutside(page1); + + await expect(page1.getByTestId('user-name')).toContainText(/admin/i); + + // Wait for token expiry + await page2.waitForTimeout(3 * 60 * 1000); + + await redirectToHomePage(page2); + + await page2.getByTestId('dropdown-profile').click(); + await page2.waitForLoadState('networkidle'); + await clickOutside(page2); + + await expect(page2.getByTestId('user-name')).toContainText(/admin/i); + }); }); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/MyData.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/MyData.spec.ts index a71cf6ad1f52..5d2de7cfc77d 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/MyData.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/MyData.spec.ts @@ -89,7 +89,7 @@ test.describe.serial('My Data page', () => { // Verify entities await verifyEntities( page, - '/api/v1/search/query?q=*&index=all&from=0&size=25', + '/api/v1/search/query?q=*&index=all&from=0&size=25*', TableEntities ); }); @@ -105,7 +105,7 @@ test.describe.serial('My Data page', () => { // Verify entities await verifyEntities( page, - '/api/v1/search/query?q=*followers:*&index=all&from=0&size=25', + '/api/v1/search/query?q=*followers:*&index=all&from=0&size=25*', TableEntities ); }); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/SearchIndexApplication.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/SearchIndexApplication.spec.ts index 9f8d15d1ef53..54b31968698c 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/SearchIndexApplication.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/SearchIndexApplication.spec.ts @@ -90,6 +90,22 @@ test('Search Index Application', async ({ page }) => { await verifyLastExecutionRun(page); }); + await test.step('View App Run Config', async () => { + await page.getByTestId('app-historical-config').click(); + await page.waitForSelector('[role="dialog"].ant-modal'); + + await expect(page.locator('[role="dialog"].ant-modal')).toBeVisible(); + + await expect(page.locator('.ant-modal-title')).toContainText( + 'Search Indexing Configuration' + ); + + await page.click('[data-testid="app-run-config-close"]'); + await page.waitForSelector('[role="dialog"].ant-modal', { + state: 'detached', + }); + }); + await test.step('Edit application', async () => { await page.click('[data-testid="edit-button"]'); await page.waitForSelector('[data-testid="schedular-card-container"]'); @@ -113,8 +129,14 @@ test('Search Index Application', async ({ page }) => { await page.getByTestId('tree-select-widget').click(); + // Bring table option to view in dropdown via searching for it + await page + .getByTestId('tree-select-widget') + .getByRole('combobox') + .fill('Table'); + // uncheck the entity - await page.getByRole('tree').getByTitle('Topic').click(); + await page.getByRole('tree').getByTitle('Table').click(); await page.click( '[data-testid="select-widget"] > .ant-select-selector > .ant-select-selection-item' diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Tag.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Tag.spec.ts index 8ef44d871d39..4cdb8e0f56bc 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Tag.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Tag.spec.ts @@ -18,7 +18,12 @@ import { TagClass } from '../../support/tag/TagClass'; import { TeamClass } from '../../support/team/TeamClass'; import { UserClass } from '../../support/user/UserClass'; import { performAdminLogin } from '../../utils/admin'; -import { getApiContext, redirectToHomePage, uuid } from '../../utils/common'; +import { + descriptionBox, + getApiContext, + redirectToHomePage, + uuid, +} from '../../utils/common'; import { addAssetsToTag, editTagPageDescription, @@ -172,9 +177,9 @@ test.describe('Tag Page with Admin Roles', () => { await expect(adminPage.getByRole('dialog')).toBeVisible(); - await adminPage.locator('.toastui-editor-pseudo-clipboard').clear(); + await adminPage.locator(descriptionBox).clear(); await adminPage - .locator('.toastui-editor-pseudo-clipboard') + .locator(descriptionBox) .fill(`This is updated test description for tag ${tag.data.name}.`); const editDescription = adminPage.waitForResponse(`/api/v1/tags/*`); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Tags.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Tags.spec.ts index 3cfc7619465f..b0395c2f6aba 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Tags.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Tags.spec.ts @@ -225,7 +225,7 @@ test('Classification Page', async ({ page }) => { '[data-testid="displayName"]', NEW_CLASSIFICATION.displayName ); - await page.fill(descriptionBox, NEW_CLASSIFICATION.description); + await page.locator(descriptionBox).fill(NEW_CLASSIFICATION.description); await page.click('[data-testid="mutually-exclusive-button"]'); const createTagCategoryResponse = page.waitForResponse( @@ -261,7 +261,7 @@ test('Classification Page', async ({ page }) => { await page.fill('[data-testid="name"]', NEW_TAG.name); await page.fill('[data-testid="displayName"]', NEW_TAG.displayName); - await page.fill(descriptionBox, NEW_TAG.description); + await page.locator(descriptionBox).fill(NEW_TAG.description); await page.fill('[data-testid="icon-url"]', NEW_TAG.icon); await page.fill('[data-testid="tags_color-color-input"]', NEW_TAG.color); @@ -284,7 +284,7 @@ test('Classification Page', async ({ page }) => { tagDisplayName: displayName, tableId: table.entityResponseData?.['id'], columnNumber: 0, - rowName: 'user_id numeric', + rowName: `${table.entity?.columns[0].name} numeric`, }); }); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Teams.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Teams.spec.ts index 2436b924d16c..868d86e15f5f 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Teams.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Teams.spec.ts @@ -10,15 +10,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import test, { expect } from '@playwright/test'; +import { expect, Page, test as base } from '@playwright/test'; +import { EDIT_USER_FOR_TEAM_RULES } from '../../constant/permission'; import { GlobalSettingOptions } from '../../constant/settings'; +import { PolicyClass } from '../../support/access-control/PoliciesClass'; +import { RolesClass } from '../../support/access-control/RolesClass'; import { EntityTypeEndpoint } from '../../support/entity/Entity.interface'; import { TableClass } from '../../support/entity/TableClass'; import { TeamClass } from '../../support/team/TeamClass'; import { UserClass } from '../../support/user/UserClass'; +import { performAdminLogin } from '../../utils/admin'; import { createNewPage, descriptionBox, + descriptionBoxReadOnly, getApiContext, redirectToHomePage, toastNotification, @@ -28,6 +33,8 @@ import { addMultiOwner } from '../../utils/entity'; import { settingClick } from '../../utils/sidebar'; import { addTeamOwnerToEntity, + addUserInTeam, + checkTeamTabCount, createTeam, hardDeleteTeam, searchTeam, @@ -35,10 +42,15 @@ import { verifyAssetsInTeamsPage, } from '../../utils/team'; -// use the admin user to login -test.use({ storageState: 'playwright/.auth/admin.json' }); - +const id = uuid(); +const dataConsumerUser = new UserClass(); +const editOnlyUser = new UserClass(); // this user will have only editUser permission in team +let team = new TeamClass(); +const team2 = new TeamClass(); +const policy = new PolicyClass(); +const role = new RolesClass(); const user = new UserClass(); +const user2 = new UserClass(); const userName = user.data.email.split('@')[0]; let teamDetails: { @@ -55,7 +67,28 @@ let teamDetails: { updatedEmail: `pwteamUpdated${uuid()}@example.com`, }; +const test = base.extend<{ + editOnlyUserPage: Page; + dataConsumerPage: Page; +}>({ + editOnlyUserPage: async ({ browser }, use) => { + const page = await browser.newPage(); + await editOnlyUser.login(page); + await use(page); + await page.close(); + }, + dataConsumerPage: async ({ browser }, use) => { + const page = await browser.newPage(); + await dataConsumerUser.login(page); + await use(page); + await page.close(); + }, +}); + test.describe('Teams Page', () => { + // use the admin user to login + test.use({ storageState: 'playwright/.auth/admin.json' }); + test.slow(true); test.beforeAll('Setup pre-requests', async ({ browser }) => { @@ -81,7 +114,7 @@ test.describe('Teams Page', () => { test('Teams Page Flow', async ({ page }) => { await test.step('Create a new team', async () => { - await settingClick(page, GlobalSettingOptions.TEAMS); + await checkTeamTabCount(page); await page.waitForLoadState('networkidle'); await page.waitForSelector('[data-testid="add-team"]'); @@ -272,7 +305,9 @@ test.describe('Teams Page', () => { // Validating the updated description await expect( - page.locator('[data-testid="asset-description-container"] p') + page.locator( + `[data-testid="asset-description-container"] ${descriptionBoxReadOnly}` + ) ).toContainText(updatedDescription); }); @@ -636,3 +671,199 @@ test.describe('Teams Page', () => { await afterAction(); }); }); + +test.describe('Teams Page with EditUser Permission', () => { + test.slow(true); + + test.beforeAll('Setup pre-requests', async ({ browser }) => { + const { apiContext, afterAction } = await performAdminLogin(browser); + await editOnlyUser.create(apiContext); + + const id = uuid(); + await policy.create(apiContext, EDIT_USER_FOR_TEAM_RULES); + await role.create(apiContext, [policy.responseData.name]); + + team = new TeamClass({ + name: `PW%edit-user-team-${id}`, + displayName: `PW Edit User Team ${id}`, + description: 'playwright edit user team description', + teamType: 'Group', + users: [editOnlyUser.responseData.id], + defaultRoles: role.responseData.id ? [role.responseData.id] : [], + }); + await team.create(apiContext); + await team2.create(apiContext); + await user.create(apiContext); + await user2.create(apiContext); + await afterAction(); + }); + + test.afterAll('Cleanup', async ({ browser }) => { + const { apiContext, afterAction } = await performAdminLogin(browser); + await user.delete(apiContext); + await user2.delete(apiContext); + await editOnlyUser.delete(apiContext); + await team.delete(apiContext); + await team2.delete(apiContext); + await policy.delete(apiContext); + await role.delete(apiContext); + await afterAction(); + }); + + test.beforeEach('Visit Home Page', async ({ editOnlyUserPage }) => { + await redirectToHomePage(editOnlyUserPage); + await team2.visitTeamPage(editOnlyUserPage); + }); + + test('Add and Remove User for Team', async ({ editOnlyUserPage }) => { + await test.step('Add user in Team from the placeholder', async () => { + await addUserInTeam(editOnlyUserPage, user); + }); + + await test.step('Add user in Team for the header manage area', async () => { + await addUserInTeam(editOnlyUserPage, user2); + }); + + await test.step('Remove user from Team', async () => { + await editOnlyUserPage + .getByRole('row', { + name: `${user.data.firstName.slice(0, 1).toUpperCase()} ${ + user.data.firstName + }.`, + }) + .getByTestId('remove-user-btn') + .click(); + + const userResponse = editOnlyUserPage.waitForResponse( + '/api/v1/users?fields=**' + ); + await editOnlyUserPage.getByRole('button', { name: 'Confirm' }).click(); + await userResponse; + + await expect( + editOnlyUserPage.locator(`[data-testid="${userName.toLowerCase()}"]`) + ).not.toBeVisible(); + }); + }); +}); + +test.describe('Teams Page with Data Consumer User', () => { + test.slow(true); + + test.beforeAll('Setup pre-requests', async ({ browser }) => { + const { apiContext, afterAction } = await performAdminLogin(browser); + await dataConsumerUser.create(apiContext); + await user.create(apiContext); + await policy.create(apiContext, EDIT_USER_FOR_TEAM_RULES); + await role.create(apiContext, [policy.responseData.name]); + + team = new TeamClass({ + name: `PW%-data-consumer-team-${id}`, + displayName: `PW Data Consumer Team ${id}`, + description: 'playwright data consumer team description', + teamType: 'Group', + users: [user.responseData.id], + defaultRoles: role.responseData.id ? [role.responseData.id] : [], + }); + await team.create(apiContext); + await team2.create(apiContext); + await afterAction(); + }); + + test.afterAll('Cleanup', async ({ browser }) => { + const { apiContext, afterAction } = await performAdminLogin(browser); + await dataConsumerUser.delete(apiContext); + await user.delete(apiContext); + await team.delete(apiContext); + await team2.delete(apiContext); + await afterAction(); + }); + + test.beforeEach('Visit Home Page', async ({ dataConsumerPage }) => { + await redirectToHomePage(dataConsumerPage); + }); + + test('Should not have edit access on team page with no data available', async ({ + dataConsumerPage, + }) => { + await team2.visitTeamPage(dataConsumerPage); + + await expect( + dataConsumerPage.getByTestId('edit-team-name') + ).not.toBeVisible(); + await expect(dataConsumerPage.getByTestId('add-domain')).not.toBeVisible(); + await expect(dataConsumerPage.getByTestId('edit-owner')).not.toBeVisible(); + await expect(dataConsumerPage.getByTestId('edit-email')).not.toBeVisible(); + await expect( + dataConsumerPage.getByTestId('edit-team-subscription') + ).not.toBeVisible(); + await expect( + dataConsumerPage.getByTestId('manage-button') + ).not.toBeVisible(); + + await expect(dataConsumerPage.getByTestId('join-teams')).toBeVisible(); + + // User Tab + await expect( + dataConsumerPage.getByTestId('add-new-user') + ).not.toBeVisible(); + await expect( + dataConsumerPage.getByTestId('permission-error-placeholder') + ).toBeVisible(); + + // Asset Tab + const assetResponse = dataConsumerPage.waitForResponse( + '/api/v1/search/query?**' + ); + await dataConsumerPage.getByTestId('assets').click(); + await assetResponse; + + await expect( + dataConsumerPage.getByTestId('add-placeholder-button') + ).not.toBeVisible(); + await expect( + dataConsumerPage.getByTestId('no-data-placeholder') + ).toBeVisible(); + + // Role Tab + await dataConsumerPage.getByTestId('roles').click(); + + await expect( + dataConsumerPage.getByTestId('add-placeholder-button') + ).not.toBeVisible(); + await expect( + dataConsumerPage.getByTestId('permission-error-placeholder') + ).toBeVisible(); + + // Policies Tab + await dataConsumerPage.getByTestId('policies').click(); + + await expect( + dataConsumerPage.getByTestId('add-placeholder-button') + ).not.toBeVisible(); + await expect( + dataConsumerPage.getByTestId('permission-error-placeholder') + ).toBeVisible(); + }); + + test('Should not have edit access on team page with data available', async ({ + dataConsumerPage, + }) => { + await team.visitTeamPage(dataConsumerPage); + + // User Tab + await expect( + dataConsumerPage.getByTestId('add-new-user') + ).not.toBeVisible(); + + // Role Tab + await dataConsumerPage.getByTestId('roles').click(); + + await expect(dataConsumerPage.getByTestId('add-role')).not.toBeVisible(); + + // Policies Tab + await dataConsumerPage.getByTestId('policies').click(); + + await expect(dataConsumerPage.getByTestId('add-policy')).not.toBeVisible(); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/TestCases.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/TestCases.spec.ts index 56bef624f085..8e694343c561 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/TestCases.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/TestCases.spec.ts @@ -79,18 +79,27 @@ test('Table difference test case', async ({ page }) => { .locator('div') .click(); - await page.fill(`#tableTestForm_params_keyColumns_0_value`, 'user_id'); - await page.getByTitle('user_id').click(); + await page.fill( + `#tableTestForm_params_keyColumns_0_value`, + table1.entity?.columns[0].name + ); + await page.getByTitle(table1.entity?.columns[0].name).click(); await page.fill('#tableTestForm_params_threshold', testCase.threshold); - await page.fill('#tableTestForm_params_useColumns_0_value', 'user_id'); - - await expect(page.getByTitle('user_id').nth(2)).toHaveClass( - /ant-select-item-option-disabled/ + await page.fill( + '#tableTestForm_params_useColumns_0_value', + table1.entity?.columns[0].name ); + await expect( + page.getByTitle(table1.entity?.columns[0].name).nth(2) + ).toHaveClass(/ant-select-item-option-disabled/); + await page.locator('#tableTestForm_params_useColumns_0_value').clear(); - await page.fill('#tableTestForm_params_useColumns_0_value', 'shop_id'); - await page.getByTitle('shop_id').click(); + await page.fill( + '#tableTestForm_params_useColumns_0_value', + table1.entity?.columns[1].name + ); + await page.getByTitle(table1.entity?.columns[1].name).click(); await page.fill('#tableTestForm_params_where', 'test'); const createTestCaseResponse = page.waitForResponse( @@ -121,16 +130,26 @@ test('Table difference test case', async ({ page }) => { .filter({ hasText: 'Key Columns' }) .getByRole('button') .click(); - await page.fill('#tableTestForm_params_keyColumns_1_value', 'email'); - await page.getByTitle('email', { exact: true }).click(); + await page.fill( + '#tableTestForm_params_keyColumns_1_value', + table1.entity?.columns[3].name + ); + await page + .getByTitle(table1.entity?.columns[3].name, { exact: true }) + .click(); await page .locator('label') .filter({ hasText: 'Use Columns' }) .getByRole('button') .click(); - await page.fill('#tableTestForm_params_useColumns_1_value', 'name'); - await page.getByTitle('name', { exact: true }).click(); + await page.fill( + '#tableTestForm_params_useColumns_1_value', + table1.entity?.columns[2].name + ); + await page + .getByTitle(table1.entity?.columns[2].name, { exact: true }) + .click(); await page.getByRole('button', { name: 'Submit' }).click(); await expect(page.getByRole('alert')).toContainText( @@ -252,17 +271,17 @@ test('Custom SQL Query', async ({ page }) => { test('Column Values To Be Not Null', async ({ page }) => { test.slow(); + await redirectToHomePage(page); + const { afterAction, apiContext } = await getApiContext(page); + const table = new TableClass(); const NEW_COLUMN_TEST_CASE_WITH_NULL_TYPE = { name: 'id_column_values_to_be_not_null', displayName: 'ID Column Values To Be Not Null', - column: 'user_id', + column: table.entity?.columns[0].name, type: 'columnValuesToBeNotNull', label: 'Column Values To Be Not Null', description: 'New table test case for columnValuesToBeNotNull', }; - await redirectToHomePage(page); - const { afterAction, apiContext } = await getApiContext(page); - const table = new TableClass(); await table.create(apiContext); await visitDataQualityTab(page, table); @@ -291,10 +310,9 @@ test('Column Values To Be Not Null', async ({ page }) => { await page.click( `[data-testid="${NEW_COLUMN_TEST_CASE_WITH_NULL_TYPE.type}"]` ); - await page.fill( - descriptionBox, - NEW_COLUMN_TEST_CASE_WITH_NULL_TYPE.description - ); + await page + .locator(descriptionBox) + .fill(NEW_COLUMN_TEST_CASE_WITH_NULL_TYPE.description); await page.click('[data-testid="submit-test"]'); await page.waitForSelector('[data-testid="success-line"]'); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/TestSuite.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/TestSuite.spec.ts index 45addd132221..1d0540775d21 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/TestSuite.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/TestSuite.spec.ts @@ -73,11 +73,11 @@ test('Logical TestSuite', async ({ page }) => { await test.step('Create', async () => { await page.click('[data-testid="add-test-suite-btn"]'); await page.fill('[data-testid="test-suite-name"]', NEW_TEST_SUITE.name); - await page.fill(descriptionBox, NEW_TEST_SUITE.description); + await page.locator(descriptionBox).fill(NEW_TEST_SUITE.description); await page.click('[data-testid="submit-button"]'); const getTestCase = page.waitForResponse( - '/api/v1/search/query?q=*&index=test_case_search_index*' + `/api/v1/dataQuality/testCases/search/list?*${testCaseName1}*` ); await page.fill('[data-testid="searchbar"]', testCaseName1); await getTestCase; @@ -127,13 +127,13 @@ test('Logical TestSuite', async ({ page }) => { await test.step('Add test case to logical test suite', async () => { const testCaseResponse = page.waitForResponse( - '/api/v1/search/query?q=*&index=test_case_search_index*' + '/api/v1/dataQuality/testCases/search/list*' ); await page.click('[data-testid="add-test-case-btn"]'); await testCaseResponse; const getTestCase = page.waitForResponse( - `/api/v1/search/query?q=*${testCaseName2}*&index=test_case_search_index*` + `/api/v1/dataQuality/testCases/search/list?*${testCaseName2}*` ); await page.fill('[data-testid="searchbar"]', testCaseName2); await getTestCase; @@ -149,6 +149,31 @@ test('Logical TestSuite', async ({ page }) => { }); }); + await test.step('Add test suite pipeline', async () => { + await page.getByRole('tab', { name: 'Pipeline' }).click(); + + await expect(page.getByTestId('add-placeholder-button')).toBeVisible(); + + await page.getByTestId('add-placeholder-button').click(); + await page.getByTestId('select-all-test-cases').click(); + + await expect(page.getByTestId('cron-type').getByText('Day')).toBeAttached(); + + await page.getByTestId('deploy-button').click(); + + await expect(page.getByTestId('view-service-button')).toBeVisible(); + + await page.waitForSelector('[data-testid="body-text"]', { + state: 'detached', + }); + + await expect(page.getByTestId('success-line')).toContainText( + /has been created and deployed successfully/ + ); + + await page.getByTestId('view-service-button').click(); + }); + await test.step('Remove test case from logical test suite', async () => { await page.click(`[data-testid="remove-${testCaseName1}"]`); const removeTestCase1 = page.waitForResponse( @@ -176,7 +201,9 @@ test('Logical TestSuite', async ({ page }) => { await page.waitForSelector("[data-testid='select-owner-tabs']", { state: 'visible', }); - const getOwnerList = page.waitForResponse('/api/v1/users?*isBot=false*'); + const getOwnerList = page.waitForResponse( + '/api/v1/search/query?q=*isBot:false*index=user_search_index*' + ); await page.click('.ant-tabs [id*=tab-users]'); await getOwnerList; await page.waitForSelector(`[data-testid="loader"]`, { diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/UserDetails.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/UserDetails.spec.ts index ad32bf9abf85..0fbef8c43c56 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/UserDetails.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/UserDetails.spec.ts @@ -14,15 +14,23 @@ import { expect, Page, test as base } from '@playwright/test'; import { GlobalSettingOptions } from '../../constant/settings'; import { USER_DESCRIPTION } from '../../constant/user'; +import { TeamClass } from '../../support/team/TeamClass'; import { AdminClass } from '../../support/user/AdminClass'; import { UserClass } from '../../support/user/UserClass'; import { performAdminLogin } from '../../utils/admin'; -import { descriptionBox, redirectToHomePage } from '../../utils/common'; +import { descriptionBox, redirectToHomePage, uuid } from '../../utils/common'; import { settingClick } from '../../utils/sidebar'; +import { redirectToUserPage } from '../../utils/userDetails'; const user1 = new UserClass(); const user2 = new UserClass(); const admin = new AdminClass(); +const team = new TeamClass({ + name: `a-new-team-${uuid()}`, + displayName: `A New Team ${uuid()}`, + description: 'playwright team description', + teamType: 'Group', +}); // Create 2 page and authenticate 1 with admin and another with normal user const test = base.extend<{ @@ -50,6 +58,8 @@ test.describe('User with different Roles', () => { await user1.create(apiContext); await user2.create(apiContext); + await team.create(apiContext); + await afterAction(); }); @@ -59,32 +69,57 @@ test.describe('User with different Roles', () => { await user1.delete(apiContext); await user2.delete(apiContext); + await team.delete(apiContext); + await afterAction(); }); - test('Non admin user should be able to edit display name and description on own profile', async ({ - userPage, + test('Admin user can get all the teams hierarchy which editing teams', async ({ + adminPage, }) => { - await redirectToHomePage(userPage); + await redirectToUserPage(adminPage); + + // Check if the avatar is visible + await expect( + adminPage + .getByTestId('user-profile-details') + .getByTestId('profile-avatar') + ).toBeVisible(); + + await adminPage + .locator('.user-profile-container [data-icon="right"]') + .click(); - await userPage.getByTestId('dropdown-profile').click(); + await expect( + adminPage.getByTestId('user-team-card-container') + ).toBeVisible(); - // Hover on the profile avatar to close the name tooltip - await userPage.getByTestId('profile-avatar').hover(); + const teamListResponse = adminPage.waitForResponse( + '/api/v1/teams/hierarchy?isJoinable=false' + ); - await userPage.waitForSelector('.profile-dropdown', { state: 'visible' }); + await adminPage.getByTestId('edit-teams-button').click(); - const getUserDetails = userPage.waitForResponse(`/api/v1/users/name/*`); + await teamListResponse; - await userPage - .locator('.profile-dropdown') - .getByTestId('user-name') - .click(); + await expect(adminPage.getByTestId('team-select')).toBeVisible(); - await getUserDetails; + await adminPage.getByTestId('team-select').click(); - // Close the profile dropdown - await userPage.getByTestId('dropdown-profile').click(); + await adminPage.waitForSelector('.ant-tree-select-dropdown', { + state: 'visible', + }); + + // Check if newly added team is there or not + await expect(adminPage.locator('.ant-tree-select-dropdown')).toContainText( + team.responseData.displayName + ); + }); + + test('Non admin user should be able to edit display name and description on own profile', async ({ + userPage, + }) => { + await redirectToUserPage(userPage); // Check if the display name is present await expect( @@ -130,6 +165,10 @@ test.describe('User with different Roles', () => { .getByTestId('edit-description') .click(); + await userPage.waitForSelector('[role="dialog"].ant-modal', { + state: 'visible', + }); + // Add description content await userPage.locator(descriptionBox).fill(USER_DESCRIPTION); @@ -155,7 +194,17 @@ test.describe('User with different Roles', () => { .getByTestId('edit-description') .click(); - await userPage.locator(descriptionBox).clear(); + await userPage.waitForSelector('[role="dialog"].ant-modal', { + state: 'visible', + }); + + await userPage.click(descriptionBox); + await userPage.keyboard.press('ControlOrMeta+A'); + await userPage.keyboard.press('Backspace'); + + await expect(userPage.locator(descriptionBox)).not.toContainText( + 'Name of the User' + ); const removeUserDescription = userPage.waitForResponse( (response) => response.request().method() === 'PATCH' @@ -174,6 +223,32 @@ test.describe('User with different Roles', () => { ).toContainText('No description'); }); + test('Non admin user should not be able to edit the persona or roles', async ({ + userPage, + }) => { + await redirectToUserPage(userPage); + + // Check if the display name is present + await expect( + userPage.getByTestId('user-profile-details').getByTestId('user-name') + ).toHaveText(user1.responseData.displayName); + + await userPage + .locator('.user-profile-container [data-icon="right"]') + .click(); + + // Check for Roles field visibility + await expect(userPage.getByTestId('user-profile-roles')).toBeVisible(); + + // Edit Persona icon shouldn't be visible + await expect( + userPage.getByTestId('persona-list').getByTestId('edit-persona') + ).not.toBeVisible(); + + // Edit Roles icon shouldn't be visible + await expect(userPage.getByTestId('edit-roles-button')).not.toBeVisible(); + }); + test('Non logged in user should not be able to edit display name and description on other users', async ({ userPage, adminPage, diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Users.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Users.spec.ts index 9f139664f1fd..fb74e92382d0 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Users.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Users.spec.ts @@ -189,6 +189,10 @@ test.describe('User with Admin Roles', () => { }) => { await redirectToHomePage(adminPage); await settingClick(adminPage, GlobalSettingOptions.USERS); + await adminPage.waitForLoadState('networkidle'); + await adminPage.waitForSelector('.user-list-table [data-testid="loader"]', { + state: 'detached', + }); await softDeleteUserProfilePage( adminPage, user.responseData.name, diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/VersionPages/EntityVersionPages.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/VersionPages/EntityVersionPages.spec.ts index 6b8ae01172a2..2fc799eadfc2 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/VersionPages/EntityVersionPages.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/VersionPages/EntityVersionPages.spec.ts @@ -22,7 +22,11 @@ import { SearchIndexClass } from '../../support/entity/SearchIndexClass'; import { StoredProcedureClass } from '../../support/entity/StoredProcedureClass'; import { TableClass } from '../../support/entity/TableClass'; import { TopicClass } from '../../support/entity/TopicClass'; -import { createNewPage, redirectToHomePage } from '../../utils/common'; +import { + createNewPage, + descriptionBoxReadOnly, + redirectToHomePage, +} from '../../utils/common'; import { addMultiOwner, assignTier } from '../../utils/entity'; const entities = [ @@ -123,7 +127,7 @@ entities.forEach((EntityClass) => { await expect( page.locator( - '[data-testid="viewer-container"] [data-testid="diff-added"]' + `[data-testid="asset-description-container"] ${descriptionBoxReadOnly} [data-testid="diff-added"]` ) ).toBeVisible(); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/VersionPages/ServiceEntityVersionPage.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/VersionPages/ServiceEntityVersionPage.spec.ts index d083c18c1ecc..c7df9756dd9e 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/VersionPages/ServiceEntityVersionPage.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/VersionPages/ServiceEntityVersionPage.spec.ts @@ -23,7 +23,11 @@ import { MlmodelServiceClass } from '../../support/entity/service/MlmodelService import { PipelineServiceClass } from '../../support/entity/service/PipelineServiceClass'; import { SearchIndexServiceClass } from '../../support/entity/service/SearchIndexServiceClass'; import { StorageServiceClass } from '../../support/entity/service/StorageServiceClass'; -import { createNewPage, redirectToHomePage } from '../../utils/common'; +import { + createNewPage, + descriptionBoxReadOnly, + redirectToHomePage, +} from '../../utils/common'; import { addMultiOwner, assignTier } from '../../utils/entity'; const entities = [ @@ -122,7 +126,7 @@ entities.forEach((EntityClass) => { await expect( page.locator( - '[data-testid="viewer-container"] [data-testid="diff-added"]' + `[data-testid="asset-description-container"] ${descriptionBoxReadOnly} [data-testid="diff-added"]` ) ).toBeVisible(); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/auth.teardown.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/auth.teardown.ts new file mode 100644 index 000000000000..bba61a4ba67e --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/auth.teardown.ts @@ -0,0 +1,22 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { test as teardown } from '@playwright/test'; +import { AdminClass } from '../support/user/AdminClass'; +import { resetPolicyChanges } from '../utils/authTeardown'; + +teardown('restore the organization roles and policies', async ({ page }) => { + const admin = new AdminClass(); + + // Reset the default organization roles and policies + await resetPolicyChanges(page, admin); +}); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/support/domain/DataProduct.ts b/openmetadata-ui/src/main/resources/ui/playwright/support/domain/DataProduct.ts index 6c60b130350e..b932c61edaae 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/support/domain/DataProduct.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/support/domain/DataProduct.ts @@ -15,6 +15,7 @@ import { uuid } from '../../utils/common'; import { EntityTypeEndpoint } from '../entity/Entity.interface'; import { EntityClass } from '../entity/EntityClass'; import { Domain } from './Domain'; +import { SubDomain } from './SubDomain'; type UserTeamRef = { name: string; @@ -45,9 +46,10 @@ export class DataProduct extends EntityClass { responseData: ResponseDataType = {} as ResponseDataType; - constructor(domain: Domain, name?: string) { + constructor(domain: Domain, name?: string, subDomain?: SubDomain) { super(EntityTypeEndpoint.DATA_PRODUCT); - this.data.domain = domain.data.name; + this.data.domain = + domain.data.name + (subDomain ? `.${subDomain?.data.name}` : ''); // fqn this.data.name = name ?? this.data.name; // eslint-disable-next-line no-useless-escape this.data.fullyQualifiedName = `\"${this.data.name}\"`; diff --git a/openmetadata-ui/src/main/resources/ui/playwright/support/domain/Domain.ts b/openmetadata-ui/src/main/resources/ui/playwright/support/domain/Domain.ts index d5e760a47a85..93a982f08abc 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/support/domain/Domain.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/support/domain/Domain.ts @@ -11,11 +11,14 @@ * limitations under the License. */ import { APIRequestContext } from '@playwright/test'; +import { Operation } from 'fast-json-patch'; import { uuid } from '../../utils/common'; type UserTeamRef = { name: string; type: string; + fullyQualifiedName?: string; + id?: string; }; type ResponseDataType = { @@ -26,7 +29,7 @@ type ResponseDataType = { id?: string; fullyQualifiedName?: string; owners?: UserTeamRef[]; - experts?: UserTeamRef[]; + experts?: string[]; }; export class Domain { @@ -70,4 +73,28 @@ export class Domain { return response.body; } + + async patch({ + apiContext, + patchData, + }: { + apiContext: APIRequestContext; + patchData: Operation[]; + }) { + const response = await apiContext.patch( + `/api/v1/domains/${this.responseData?.id}`, + { + data: patchData, + headers: { + 'Content-Type': 'application/json-patch+json', + }, + } + ); + + this.responseData = await response.json(); + + return { + entity: this.responseData, + }; + } } diff --git a/openmetadata-ui/src/main/resources/ui/playwright/support/domain/SubDomain.ts b/openmetadata-ui/src/main/resources/ui/playwright/support/domain/SubDomain.ts index fb7c3c1fbd55..41d5b09c9b32 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/support/domain/SubDomain.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/support/domain/SubDomain.ts @@ -11,12 +11,15 @@ * limitations under the License. */ import { APIRequestContext } from '@playwright/test'; +import { Operation } from 'fast-json-patch'; import { uuid } from '../../utils/common'; import { Domain } from './Domain'; type UserTeamRef = { name: string; type: string; + fullyQualifiedName?: string; + id?: string; }; type ResponseDataType = { @@ -27,7 +30,7 @@ type ResponseDataType = { id?: string; fullyQualifiedName?: string; owners?: UserTeamRef[]; - experts?: UserTeamRef[]; + experts?: string[]; parent?: string; }; @@ -74,4 +77,28 @@ export class SubDomain { return response.body; } + + async patch({ + apiContext, + patchData, + }: { + apiContext: APIRequestContext; + patchData: Operation[]; + }) { + const response = await apiContext.patch( + `/api/v1/domains/${this.responseData?.id}`, + { + data: patchData, + headers: { + 'Content-Type': 'application/json-patch+json', + }, + } + ); + + this.responseData = await response.json(); + + return { + entity: this.responseData, + }; + } } diff --git a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/ApiCollectionClass.ts b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/ApiCollectionClass.ts index f2a281a808c6..381763afe7d4 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/ApiCollectionClass.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/ApiCollectionClass.ts @@ -13,6 +13,7 @@ import { APIRequestContext, Page } from '@playwright/test'; import { Operation } from 'fast-json-patch'; import { SERVICE_TYPE } from '../../constant/service'; +import { ServiceTypes } from '../../constant/settings'; import { uuid } from '../../utils/common'; import { visitEntityPage } from '../../utils/entity'; import { visitServiceDetailsPage } from '../../utils/service'; @@ -43,7 +44,6 @@ export class ApiCollectionClass extends EntityClass { }; private apiEndpointName = `pw-api-endpoint-${uuid()}`; - private fqn = `${this.service.name}.${this.entity.name}.${this.apiEndpointName}`; apiEndpoint = { name: this.apiEndpointName, @@ -53,91 +53,89 @@ export class ApiCollectionClass extends EntityClass { schemaType: 'JSON', schemaFields: [ { - name: 'default', + name: `default${uuid()}`, dataType: 'RECORD', - fullyQualifiedName: `${this.fqn}.default`, tags: [], children: [ { - name: 'name', + name: `name${uuid()}`, dataType: 'RECORD', - fullyQualifiedName: `${this.fqn}.default.name`, tags: [], children: [ { - name: 'first_name', + name: `first_name${uuid()}`, dataType: 'STRING', description: 'Description for schema field first_name', - fullyQualifiedName: `${this.fqn}.default.name.first_name`, tags: [], }, { - name: 'last_name', + name: `last_name${uuid()}`, dataType: 'STRING', - fullyQualifiedName: `${this.fqn}.default.name.last_name`, tags: [], }, ], }, { - name: 'age', + name: `age${uuid()}`, dataType: 'INT', - fullyQualifiedName: `${this.fqn}.default.age`, tags: [], }, { - name: 'club_name', + name: `club_name${uuid()}`, dataType: 'STRING', - fullyQualifiedName: `${this.fqn}.default.club_name`, tags: [], }, ], }, + { + name: `secondary${uuid()}`, + dataType: 'RECORD', + tags: [], + }, ], }, responseSchema: { schemaType: 'JSON', schemaFields: [ { - name: 'default', + name: `default${uuid()}`, dataType: 'RECORD', - fullyQualifiedName: `${this.fqn}.default`, tags: [], children: [ { - name: 'name', + name: `name${uuid()}`, dataType: 'RECORD', - fullyQualifiedName: `${this.fqn}.default.name`, tags: [], children: [ { - name: 'first_name', + name: `first_name${uuid()}`, dataType: 'STRING', - fullyQualifiedName: `${this.fqn}.default.name.first_name`, tags: [], }, { - name: 'last_name', + name: `last_name${uuid()}`, dataType: 'STRING', - fullyQualifiedName: `${this.fqn}.default.name.last_name`, tags: [], }, ], }, { - name: 'age', + name: `age${uuid()}`, dataType: 'INT', - fullyQualifiedName: `${this.fqn}.default.age`, tags: [], }, { - name: 'club_name', + name: `club_name${uuid()}`, dataType: 'STRING', - fullyQualifiedName: `${this.fqn}.default.club_name`, tags: [], }, ], }, + { + name: `secondary${uuid()}`, + dataType: 'RECORD', + tags: [], + }, ], }, }; @@ -149,6 +147,8 @@ export class ApiCollectionClass extends EntityClass { constructor(name?: string) { super(EntityTypeEndpoint.API_COLLECTION); + this.serviceCategory = SERVICE_TYPE.ApiService; + this.serviceType = ServiceTypes.API_SERVICES; this.service.name = name ?? this.service.name; this.type = 'Api Collection'; } diff --git a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/ApiEndpointClass.ts b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/ApiEndpointClass.ts index 0eaa969a7746..9a0b9efa91c8 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/ApiEndpointClass.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/ApiEndpointClass.ts @@ -13,6 +13,7 @@ import { APIRequestContext, Page } from '@playwright/test'; import { Operation } from 'fast-json-patch'; import { SERVICE_TYPE } from '../../constant/service'; +import { ServiceTypes } from '../../constant/settings'; import { uuid } from '../../utils/common'; import { visitEntityPage } from '../../utils/entity'; import { @@ -153,6 +154,7 @@ export class ApiEndpointClass extends EntityClass { super(EntityTypeEndpoint.API_ENDPOINT); this.service.name = name ?? this.service.name; this.serviceCategory = SERVICE_TYPE.ApiService; + this.serviceType = ServiceTypes.API_SERVICES; this.type = 'ApiEndpoint'; this.childrenTabId = 'schema'; this.childrenSelectorId = this.children[0].name; diff --git a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/ContainerClass.ts b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/ContainerClass.ts index e3559452b0b6..8f54d624f4bf 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/ContainerClass.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/ContainerClass.ts @@ -13,6 +13,7 @@ import { APIRequestContext, Page } from '@playwright/test'; import { Operation } from 'fast-json-patch'; import { SERVICE_TYPE } from '../../constant/service'; +import { ServiceTypes } from '../../constant/settings'; import { uuid } from '../../utils/common'; import { visitEntityPage } from '../../utils/entity'; import { @@ -24,6 +25,7 @@ import { EntityClass } from './EntityClass'; export class ContainerClass extends EntityClass { private containerName = `pw-container-${uuid()}`; + private childContainerName = `pw-container-${uuid()}`; service = { name: `pw-storage-service-${uuid()}`, serviceType: 'S3', @@ -44,15 +46,54 @@ export class ContainerClass extends EntityClass { name: this.containerName, displayName: this.containerName, service: this.service.name, + dataModel: { + isPartitioned: true, + columns: [ + { + name: `merchant${uuid()}`, + dataType: 'VARCHAR', + dataLength: 100, + dataTypeDisplay: 'varchar', + description: 'The merchant for this transaction.', + tags: [], + ordinalPosition: 2, + }, + { + name: `columbia${uuid()}`, + dataType: 'NUMERIC', + dataTypeDisplay: 'numeric', + description: + 'The ID of the executed transaction. This column is the primary key for this table.', + tags: [], + constraint: 'PRIMARY_KEY', + ordinalPosition: 1, + }, + { + name: `delivery${uuid()}`, + dataType: 'TIMESTAMP', + dataTypeDisplay: 'timestamp', + description: 'The time the transaction took place.', + tags: [], + ordinalPosition: 3, + }, + ], + }, + }; + childContainer = { + name: this.childContainerName, + displayName: this.childContainerName, + service: this.service.name, }; serviceResponseData: ResponseDataType = {} as ResponseDataType; entityResponseData: ResponseDataWithServiceType = {} as ResponseDataWithServiceType; + childResponseData: ResponseDataType = {} as ResponseDataType; constructor(name?: string) { super(EntityTypeEndpoint.Container); this.service.name = name ?? this.service.name; + this.serviceType = ServiceTypes.STORAGE_SERVICES; this.type = 'Container'; this.serviceCategory = SERVICE_TYPE.Storage; } @@ -71,6 +112,20 @@ export class ContainerClass extends EntityClass { this.serviceResponseData = await serviceResponse.json(); this.entityResponseData = await entityResponse.json(); + const childContainer = { + ...this.childContainer, + parent: { + id: this.entityResponseData.id, + type: 'container', + }, + }; + + const childResponse = await apiContext.post('/api/v1/containers', { + data: childContainer, + }); + + this.childResponseData = await childResponse.json(); + return { service: serviceResponse.body, entity: entityResponse.body, diff --git a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/DashboardClass.ts b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/DashboardClass.ts index bea3c4579a35..b1c8c226be4b 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/DashboardClass.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/DashboardClass.ts @@ -13,6 +13,7 @@ import { APIRequestContext, Page } from '@playwright/test'; import { Operation } from 'fast-json-patch'; import { SERVICE_TYPE } from '../../constant/service'; +import { ServiceTypes } from '../../constant/settings'; import { uuid } from '../../utils/common'; import { visitEntityPage } from '../../utils/entity'; import { @@ -24,6 +25,8 @@ import { EntityClass } from './EntityClass'; export class DashboardClass extends EntityClass { private dashboardName = `pw-dashboard-${uuid()}`; + private dashboardDataModelName = `pw-dashboard-data-model-${uuid()}`; + private projectName = `pw-project-${uuid()}`; service = { name: `pw-dashboard-service-${uuid()}`, serviceType: 'Superset', @@ -49,18 +52,53 @@ export class DashboardClass extends EntityClass { name: this.dashboardName, displayName: this.dashboardName, service: this.service.name, + project: this.projectName, + }; + children = [ + { + name: 'merchant', + dataType: 'VARCHAR', + dataLength: 256, + dataTypeDisplay: 'varchar', + description: 'merchant', + }, + { + name: 'notes', + dataType: 'VARCHAR', + dataLength: 256, + dataTypeDisplay: 'varchar', + description: 'merchant', + }, + { + name: 'country_name', + dataType: 'VARCHAR', + dataLength: 256, + dataTypeDisplay: 'varchar', + description: 'Name of the country.', + }, + ]; + dataModel = { + name: this.dashboardDataModelName, + displayName: this.dashboardDataModelName, + service: this.service.name, + columns: this.children, + dataModelType: 'SupersetDataModel', }; serviceResponseData: ResponseDataType = {} as ResponseDataType; entityResponseData: ResponseDataWithServiceType = {} as ResponseDataWithServiceType; + dataModelResponseData: ResponseDataWithServiceType = + {} as ResponseDataWithServiceType; chartsResponseData: ResponseDataType = {} as ResponseDataType; - constructor(name?: string) { + constructor(name?: string, dataModelType = 'SupersetDataModel') { super(EntityTypeEndpoint.Dashboard); this.service.name = name ?? this.service.name; this.type = 'Dashboard'; this.serviceCategory = SERVICE_TYPE.Dashboard; + this.serviceType = ServiceTypes.DASHBOARD_SERVICES; + this.dataModel.dataModelType = dataModelType; } async create(apiContext: APIRequestContext) { @@ -80,9 +118,16 @@ export class DashboardClass extends EntityClass { charts: [`${this.service.name}.${this.charts.name}`], }, }); + const dataModelResponse = await apiContext.post( + '/api/v1/dashboard/datamodels', + { + data: this.dataModel, + } + ); this.serviceResponseData = await serviceResponse.json(); this.chartsResponseData = await chartsResponse.json(); + this.dataModelResponseData = await dataModelResponse.json(); this.entityResponseData = await entityResponse.json(); return { diff --git a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/DashboardDataModelClass.ts b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/DashboardDataModelClass.ts index 24d824720cb6..d1b32af7d797 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/DashboardDataModelClass.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/DashboardDataModelClass.ts @@ -13,6 +13,7 @@ import { APIRequestContext, Page } from '@playwright/test'; import { Operation } from 'fast-json-patch'; import { SERVICE_TYPE } from '../../constant/service'; +import { ServiceTypes } from '../../constant/settings'; import { uuid } from '../../utils/common'; import { visitEntityPage } from '../../utils/entity'; import { @@ -24,6 +25,7 @@ import { EntityClass } from './EntityClass'; export class DashboardDataModelClass extends EntityClass { private dashboardDataModelName = `pw-dashboard-data-model-${uuid()}`; + private projectName = `pw-project-${uuid()}`; service = { name: `pw-dashboard-service-${uuid()}`, serviceType: 'Superset', @@ -57,6 +59,7 @@ export class DashboardDataModelClass extends EntityClass { service: this.service.name, columns: this.children, dataModelType: 'SupersetDataModel', + project: this.projectName, }; serviceResponseData: ResponseDataType = {} as ResponseDataType; @@ -70,6 +73,7 @@ export class DashboardDataModelClass extends EntityClass { this.childrenTabId = 'model'; this.childrenSelectorId = this.children[0].name; this.serviceCategory = SERVICE_TYPE.Dashboard; + this.serviceType = ServiceTypes.DASHBOARD_SERVICES; } async create(apiContext: APIRequestContext) { diff --git a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/DatabaseClass.ts b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/DatabaseClass.ts index b5d2a634ad7f..d800d4fe3da5 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/DatabaseClass.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/DatabaseClass.ts @@ -13,6 +13,7 @@ import { APIRequestContext, expect, Page } from '@playwright/test'; import { Operation } from 'fast-json-patch'; import { SERVICE_TYPE } from '../../constant/service'; +import { ServiceTypes } from '../../constant/settings'; import { assignDomain, removeDomain, @@ -127,6 +128,7 @@ export class DatabaseClass extends EntityClass { super(EntityTypeEndpoint.Database); this.service.name = name ?? this.service.name; this.type = 'Database'; + this.serviceType = ServiceTypes.DATABASE_SERVICES; } async create(apiContext: APIRequestContext) { @@ -346,6 +348,6 @@ export class DatabaseClass extends EntityClass { await assignDomain(page, domain1); await this.verifyDomainPropagation(page, domain1); await updateDomain(page, domain2); - await removeDomain(page); + await removeDomain(page, domain2); } } diff --git a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/DatabaseSchemaClass.ts b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/DatabaseSchemaClass.ts index 45909c1e466b..38aa8de0b954 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/DatabaseSchemaClass.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/DatabaseSchemaClass.ts @@ -13,6 +13,7 @@ import { APIRequestContext, Page } from '@playwright/test'; import { Operation } from 'fast-json-patch'; import { SERVICE_TYPE } from '../../constant/service'; +import { ServiceTypes } from '../../constant/settings'; import { uuid } from '../../utils/common'; import { visitServiceDetailsPage } from '../../utils/service'; import { @@ -61,6 +62,7 @@ export class DatabaseSchemaClass extends EntityClass { super(EntityTypeEndpoint.DatabaseSchema); this.service.name = name ?? this.service.name; this.type = 'Database Schema'; + this.serviceType = ServiceTypes.DATABASE_SERVICES; } async create(apiContext: APIRequestContext) { diff --git a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/Entity.interface.ts b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/Entity.interface.ts index 51eece88b0c7..03d8e94bcb2e 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/Entity.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/Entity.interface.ts @@ -88,7 +88,7 @@ export type TestCaseData = { export type TestSuiteData = { name?: string; - executableEntityReference?: string; + basicEntityReference?: string; description?: string; }; @@ -102,3 +102,15 @@ export interface UserResponseDataType extends ResponseDataType { isBot: boolean; href?: string; } + +export interface EntityReference { + id: string; + type: string; + name: string; + displayName?: string; + deleted?: boolean; + description?: string; + fullyQualifiedName?: string; + href?: string; + inherited?: boolean; +} diff --git a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/EntityClass.ts b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/EntityClass.ts index 6df7ae49e18b..e79eb7edf669 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/EntityClass.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/EntityClass.ts @@ -12,7 +12,7 @@ */ import { APIRequestContext, Page } from '@playwright/test'; import { CustomPropertySupportedEntityList } from '../../constant/customProperty'; -import { GlobalSettingOptions } from '../../constant/settings'; +import { GlobalSettingOptions, ServiceTypes } from '../../constant/settings'; import { assignDomain, removeDomain, updateDomain } from '../../utils/common'; import { createCustomPropertyForEntity, @@ -56,6 +56,7 @@ import { EntityTypeEndpoint, ENTITY_PATH } from './Entity.interface'; export class EntityClass { type = ''; serviceCategory?: GlobalSettingOptions; + serviceType?: ServiceTypes; childrenTabId?: string; childrenSelectorId?: string; endpoint: EntityTypeEndpoint; @@ -123,7 +124,7 @@ export class EntityClass { ) { await assignDomain(page, domain1); await updateDomain(page, domain2); - await removeDomain(page); + await removeDomain(page, domain2); } async owner( diff --git a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/EntityDataClass.ts b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/EntityDataClass.ts index d66920a060b2..44e0eb9036fb 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/EntityDataClass.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/EntityDataClass.ts @@ -17,6 +17,15 @@ import { GlossaryTerm } from '../glossary/GlossaryTerm'; import { TagClass } from '../tag/TagClass'; import { TeamClass } from '../team/TeamClass'; import { UserClass } from '../user/UserClass'; +import { ApiCollectionClass } from './ApiCollectionClass'; +import { ContainerClass } from './ContainerClass'; +import { DashboardClass } from './DashboardClass'; +import { DashboardDataModelClass } from './DashboardDataModelClass'; +import { MlModelClass } from './MlModelClass'; +import { PipelineClass } from './PipelineClass'; +import { SearchIndexClass } from './SearchIndexClass'; +import { TableClass } from './TableClass'; +import { TopicClass } from './TopicClass'; export class EntityDataClass { static readonly domain1 = new Domain(); @@ -31,6 +40,25 @@ export class EntityDataClass { static readonly team1 = new TeamClass(); static readonly team2 = new TeamClass(); static readonly tierTag1 = new TagClass({ classification: 'Tier' }); + static readonly tierTag2 = new TagClass({ classification: 'Tier' }); + static readonly table1 = new TableClass(); + static readonly table2 = new TableClass(undefined, 'MaterializedView'); + static readonly topic1 = new TopicClass(); + static readonly topic2 = new TopicClass(); + static readonly dashboard1 = new DashboardClass(); + static readonly dashboard2 = new DashboardClass(undefined, 'LookMlExplore'); + static readonly mlModel1 = new MlModelClass(); + static readonly mlModel2 = new MlModelClass(); + static readonly pipeline1 = new PipelineClass(); + static readonly pipeline2 = new PipelineClass(); + static readonly dashboardDataModel1 = new DashboardDataModelClass(); + static readonly dashboardDataModel2 = new DashboardDataModelClass(); + static readonly apiCollection1 = new ApiCollectionClass(); + static readonly apiCollection2 = new ApiCollectionClass(); + static readonly searchIndex1 = new SearchIndexClass(); + static readonly searchIndex2 = new SearchIndexClass(); + static readonly container1 = new ContainerClass(); + static readonly container2 = new ContainerClass(); static async preRequisitesForTests(apiContext: APIRequestContext) { // Add pre-requisites for tests @@ -46,6 +74,25 @@ export class EntityDataClass { await this.team1.create(apiContext); await this.team2.create(apiContext); await this.tierTag1.create(apiContext); + await this.tierTag2.create(apiContext); + await this.table1.create(apiContext); + await this.table2.create(apiContext); + await this.topic1.create(apiContext); + await this.topic2.create(apiContext); + await this.dashboard1.create(apiContext); + await this.dashboard2.create(apiContext); + await this.mlModel1.create(apiContext); + await this.mlModel2.create(apiContext); + await this.pipeline1.create(apiContext); + await this.pipeline2.create(apiContext); + await this.dashboardDataModel1.create(apiContext); + await this.dashboardDataModel2.create(apiContext); + await this.apiCollection1.create(apiContext); + await this.apiCollection2.create(apiContext); + await this.searchIndex1.create(apiContext); + await this.searchIndex2.create(apiContext); + await this.container1.create(apiContext); + await this.container2.create(apiContext); } static async postRequisitesForTests(apiContext: APIRequestContext) { @@ -61,5 +108,24 @@ export class EntityDataClass { await this.team1.delete(apiContext); await this.team2.delete(apiContext); await this.tierTag1.delete(apiContext); + await this.tierTag2.delete(apiContext); + await this.table1.delete(apiContext); + await this.table2.delete(apiContext); + await this.topic1.delete(apiContext); + await this.topic2.delete(apiContext); + await this.dashboard1.delete(apiContext); + await this.dashboard2.delete(apiContext); + await this.mlModel1.delete(apiContext); + await this.mlModel2.delete(apiContext); + await this.pipeline1.delete(apiContext); + await this.pipeline2.delete(apiContext); + await this.dashboardDataModel1.delete(apiContext); + await this.dashboardDataModel2.delete(apiContext); + await this.apiCollection1.delete(apiContext); + await this.apiCollection2.delete(apiContext); + await this.searchIndex1.delete(apiContext); + await this.searchIndex2.delete(apiContext); + await this.container1.delete(apiContext); + await this.container2.delete(apiContext); } } diff --git a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/MlModelClass.ts b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/MlModelClass.ts index fe415769f519..119ee9401dae 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/MlModelClass.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/MlModelClass.ts @@ -13,6 +13,7 @@ import { APIRequestContext, Page } from '@playwright/test'; import { Operation } from 'fast-json-patch'; import { SERVICE_TYPE } from '../../constant/service'; +import { ServiceTypes } from '../../constant/settings'; import { uuid } from '../../utils/common'; import { visitEntityPage } from '../../utils/entity'; import { @@ -43,6 +44,11 @@ export class MlModelClass extends EntityClass { dataType: 'numerical', description: 'Sales amount', }, + { + name: 'persona', + dataType: 'categorical', + description: 'type of buyer', + }, ]; entity = { @@ -64,6 +70,7 @@ export class MlModelClass extends EntityClass { this.childrenTabId = 'features'; this.childrenSelectorId = `feature-card-${this.children[0].name}`; this.serviceCategory = SERVICE_TYPE.MLModels; + this.serviceType = ServiceTypes.ML_MODEL_SERVICES; } async create(apiContext: APIRequestContext) { diff --git a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/PipelineClass.ts b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/PipelineClass.ts index 30a7c6adc108..e7c461ae2afc 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/PipelineClass.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/PipelineClass.ts @@ -13,6 +13,7 @@ import { APIRequestContext, Page } from '@playwright/test'; import { Operation } from 'fast-json-patch'; import { SERVICE_TYPE } from '../../constant/service'; +import { ServiceTypes } from '../../constant/settings'; import { uuid } from '../../utils/common'; import { visitEntityPage } from '../../utils/entity'; import { @@ -38,7 +39,10 @@ export class PipelineClass extends EntityClass { }, }; - children = [{ name: 'snowflake_task' }]; + children = [ + { name: 'snowflake_task', displayName: 'Snowflake Task' }, + { name: 'presto_task', displayName: 'Presto Task' }, + ]; entity = { name: this.pipelineName, @@ -59,6 +63,7 @@ export class PipelineClass extends EntityClass { this.childrenTabId = 'tasks'; this.childrenSelectorId = this.children[0].name; this.serviceCategory = SERVICE_TYPE.Pipeline; + this.serviceType = ServiceTypes.PIPELINE_SERVICES; } async create(apiContext: APIRequestContext) { diff --git a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/SearchIndexClass.ts b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/SearchIndexClass.ts index 9f75df65cb19..41efd732e915 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/SearchIndexClass.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/SearchIndexClass.ts @@ -13,6 +13,7 @@ import { APIRequestContext, Page } from '@playwright/test'; import { Operation } from 'fast-json-patch'; import { SERVICE_TYPE } from '../../constant/service'; +import { ServiceTypes } from '../../constant/settings'; import { uuid } from '../../utils/common'; import { visitEntityPage } from '../../utils/entity'; import { @@ -44,43 +45,45 @@ export class SearchIndexClass extends EntityClass { children = [ { - name: 'name', + name: `name${uuid()}`, dataType: 'TEXT', dataTypeDisplay: 'text', description: 'Table Entity Name.', - fullyQualifiedName: `${this.fqn}.name`, tags: [], }, { - name: 'description', + name: `databaseSchema${uuid()}`, + dataType: 'TEXT', + dataTypeDisplay: 'text', + description: 'Table Entity Database Schema.', + tags: [], + }, + { + name: `description${uuid()}`, dataType: 'TEXT', dataTypeDisplay: 'text', description: 'Table Entity Description.', - fullyQualifiedName: `${this.fqn}.description`, tags: [], }, { - name: 'columns', + name: `columns${uuid()}`, dataType: 'NESTED', dataTypeDisplay: 'nested', description: 'Table Columns.', - fullyQualifiedName: `${this.fqn}.columns`, tags: [], children: [ { - name: 'name', + name: `name${uuid()}`, dataType: 'TEXT', dataTypeDisplay: 'text', description: 'Column Name.', - fullyQualifiedName: `${this.fqn}.columns.name`, tags: [], }, { - name: 'description', + name: `description${uuid()}`, dataType: 'TEXT', dataTypeDisplay: 'text', description: 'Column Description.', - fullyQualifiedName: `${this.fqn}.columns.description`, tags: [], }, ], @@ -103,8 +106,9 @@ export class SearchIndexClass extends EntityClass { this.service.name = name ?? this.service.name; this.type = 'SearchIndex'; this.childrenTabId = 'fields'; - this.childrenSelectorId = this.children[0].fullyQualifiedName; + this.childrenSelectorId = `${this.fqn}.${this.children[0].name}`; this.serviceCategory = SERVICE_TYPE.Search; + this.serviceType = ServiceTypes.SEARCH_SERVICES; } async create(apiContext: APIRequestContext) { diff --git a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/StoredProcedureClass.ts b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/StoredProcedureClass.ts index b0b41f3d4e0c..1c7c23ebeaf4 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/StoredProcedureClass.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/StoredProcedureClass.ts @@ -13,6 +13,7 @@ import { APIRequestContext, Page } from '@playwright/test'; import { Operation } from 'fast-json-patch'; import { SERVICE_TYPE } from '../../constant/service'; +import { ServiceTypes } from '../../constant/settings'; import { uuid } from '../../utils/common'; import { visitEntityPage } from '../../utils/entity'; import { @@ -71,6 +72,7 @@ export class StoredProcedureClass extends EntityClass { this.service.name = name ?? this.service.name; this.serviceCategory = SERVICE_TYPE.Database; this.type = 'Store Procedure'; + this.serviceType = ServiceTypes.DATABASE_SERVICES; } async create(apiContext: APIRequestContext) { diff --git a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/TableClass.ts b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/TableClass.ts index 29ac8d6490cb..6b8abfd6bd77 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/TableClass.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/TableClass.ts @@ -14,6 +14,7 @@ import { APIRequestContext, Page } from '@playwright/test'; import { Operation } from 'fast-json-patch'; import { isEmpty } from 'lodash'; import { SERVICE_TYPE } from '../../constant/service'; +import { ServiceTypes } from '../../constant/settings'; import { uuid } from '../../utils/common'; import { visitEntityPage } from '../../utils/entity'; import { @@ -55,43 +56,44 @@ export class TableClass extends EntityClass { }; children = [ { - name: 'user_id', + name: `user_id${uuid()}`, dataType: 'NUMERIC', dataTypeDisplay: 'numeric', description: 'Unique identifier for the user of your Shopify POS or your Shopify admin.', }, { - name: 'shop_id', + name: `shop_id${uuid()}`, dataType: 'NUMERIC', dataTypeDisplay: 'numeric', description: 'The ID of the store. This column is a foreign key reference to the shop_id column in the dim.shop table.', }, { - name: 'name', + name: `name${uuid()}`, dataType: 'VARCHAR', dataLength: 100, dataTypeDisplay: 'varchar', description: 'Name of the staff member.', children: [ { - name: 'first_name', - dataType: 'VARCHAR', + name: `first_name${uuid()}`, + dataType: 'STRUCT', dataLength: 100, - dataTypeDisplay: 'varchar', + dataTypeDisplay: + 'struct', description: 'First name of the staff member.', }, { - name: 'last_name', - dataType: 'VARCHAR', + name: `last_name${uuid()}`, + dataType: 'ARRAY', dataLength: 100, - dataTypeDisplay: 'varchar', + dataTypeDisplay: 'array>>', }, ], }, { - name: 'email', + name: `email${uuid()}`, dataType: 'VARCHAR', dataLength: 100, dataTypeDisplay: 'varchar', @@ -104,6 +106,7 @@ export class TableClass extends EntityClass { displayName: `pw table ${uuid()}`, description: 'description', columns: this.children, + tableType: 'SecureView', databaseSchema: `${this.service.name}.${this.database.name}.${this.schema.name}`, }; @@ -120,12 +123,14 @@ export class TableClass extends EntityClass { queryResponseData: ResponseDataType[] = []; additionalEntityTableResponseData: ResponseDataType[] = []; - constructor(name?: string) { + constructor(name?: string, tableType?: string) { super(EntityTypeEndpoint.Table); this.service.name = name ?? this.service.name; this.serviceCategory = SERVICE_TYPE.Database; + this.serviceType = ServiceTypes.DATABASE_SERVICES; this.type = 'Table'; this.childrenTabId = 'schema'; + this.entity.tableType = tableType ?? this.entity.tableType; this.childrenSelectorId = `${this.entity.databaseSchema}.${this.entity.name}.${this.children[0].name}`; } @@ -234,11 +239,10 @@ export class TableClass extends EntityClass { } const testSuiteData = await apiContext - .post('/api/v1/dataQuality/testSuites/executable', { + .post('/api/v1/dataQuality/testSuites/basic', { data: { name: `pw-test-suite-${uuid()}`, - executableEntityReference: - this.entityResponseData?.['fullyQualifiedName'], + basicEntityReference: this.entityResponseData?.['fullyQualifiedName'], description: 'Playwright test suite for table', ...testSuite, }, diff --git a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/TopicClass.ts b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/TopicClass.ts index 82ce4e87f5bd..aa7c6458874b 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/TopicClass.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/TopicClass.ts @@ -13,6 +13,7 @@ import { APIRequestContext, Page } from '@playwright/test'; import { Operation } from 'fast-json-patch'; import { SERVICE_TYPE } from '../../constant/service'; +import { ServiceTypes } from '../../constant/settings'; import { uuid } from '../../utils/common'; import { visitEntityPage } from '../../utils/entity'; import { @@ -38,32 +39,27 @@ export class TopicClass extends EntityClass { }, }; private topicName = `pw-topic-${uuid()}`; - private fqn = `${this.service.name}.${this.topicName}`; children = [ { - name: 'default', + name: `default${uuid()}`, dataType: 'RECORD', - fullyQualifiedName: `${this.fqn}.default`, tags: [], children: [ { - name: 'name', + name: `name${uuid()}`, dataType: 'RECORD', - fullyQualifiedName: `${this.fqn}.default.name`, tags: [], children: [ { name: 'first_name', dataType: 'STRING', description: 'Description for schema field first_name', - fullyQualifiedName: `${this.fqn}.default.name.first_name`, tags: [], }, { name: 'last_name', dataType: 'STRING', - fullyQualifiedName: `${this.fqn}.default.name.last_name`, tags: [], }, ], @@ -71,17 +67,21 @@ export class TopicClass extends EntityClass { { name: 'age', dataType: 'INT', - fullyQualifiedName: `${this.fqn}.default.age`, tags: [], }, { name: 'club_name', dataType: 'STRING', - fullyQualifiedName: `${this.fqn}.default.club_name`, tags: [], }, ], }, + { + name: `secondary${uuid()}`, + dataType: 'RECORD', + tags: [], + children: [], + }, ]; entity = { @@ -107,6 +107,7 @@ export class TopicClass extends EntityClass { this.childrenTabId = 'schema'; this.childrenSelectorId = this.children[0].name; this.serviceCategory = SERVICE_TYPE.Messaging; + this.serviceType = ServiceTypes.MESSAGING_SERVICES; } async create(apiContext: APIRequestContext) { diff --git a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/ingestion/ApiIngestionClass.ts b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/ingestion/ApiIngestionClass.ts index 36565e388f17..a21a8106cfff 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/ingestion/ApiIngestionClass.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/ingestion/ApiIngestionClass.ts @@ -21,7 +21,7 @@ import ServiceBaseClass from './ServiceBaseClass'; class ApiIngestionClass extends ServiceBaseClass { constructor() { - super(Services.API, `pw-api-with-%-${uuid()}`, 'Rest', 'Containers'); + super(Services.API, `pw-api-with-%-${uuid()}`, 'Rest', 'store'); } async createService(page: Page) { @@ -33,7 +33,7 @@ class ApiIngestionClass extends ServiceBaseClass { } async fillConnectionDetails(page: Page) { - const openAPISchemaURL = 'https://docs.open-metadata.org/swagger.json'; + const openAPISchemaURL = 'https://petstore3.swagger.io/api/v3/openapi.json'; await page.locator('#root\\/openAPISchemaURL').fill(openAPISchemaURL); await checkServiceFieldSectionHighlighting(page, 'openAPISchemaURL'); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/ingestion/MySqlIngestionClass.ts b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/ingestion/MySqlIngestionClass.ts index d71358f60588..2aeb33b059e4 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/ingestion/MySqlIngestionClass.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/ingestion/MySqlIngestionClass.ts @@ -18,6 +18,7 @@ import { TestType, } from '@playwright/test'; import { env } from 'process'; +import { resetTokenFromBotPage } from '../../../utils/bot'; import { getApiContext, redirectToHomePage, @@ -84,6 +85,13 @@ class MysqlIngestionClass extends ServiceBaseClass { await test.step('Add Profiler ingestion', async () => { const { apiContext } = await getApiContext(page); await redirectToHomePage(page); + + // Todo: Remove this patch once the issue is fixed #19140 + await resetTokenFromBotPage(page, { + name: 'profiler', + testId: 'bot-link-ProfilerBot', + }); + await visitServiceDetailsPage( page, { diff --git a/openmetadata-ui/src/main/resources/ui/playwright/support/glossary/Glossary.ts b/openmetadata-ui/src/main/resources/ui/playwright/support/glossary/Glossary.ts index 1c5db09f8e6c..9ac173401452 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/support/glossary/Glossary.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/support/glossary/Glossary.ts @@ -17,6 +17,7 @@ import { uuid, visitGlossaryPage, } from '../../utils/common'; +import { EntityReference } from '../entity/Entity.interface'; import { GlossaryData, GlossaryResponseDataType } from './Glossary.interface'; export class Glossary { @@ -38,8 +39,9 @@ export class Glossary { responseData: GlossaryResponseDataType = {} as GlossaryResponseDataType; - constructor(name?: string) { + constructor(name?: string, reviewers?: EntityReference[]) { this.data.name = name ?? this.data.name; + this.data.reviewers = reviewers ?? this.data.reviewers; } async visitPage(page: Page) { diff --git a/openmetadata-ui/src/main/resources/ui/playwright/support/interfaces/ConditionalPermissions.interface.ts b/openmetadata-ui/src/main/resources/ui/playwright/support/interfaces/ConditionalPermissions.interface.ts new file mode 100644 index 000000000000..15bca0e688d8 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/playwright/support/interfaces/ConditionalPermissions.interface.ts @@ -0,0 +1,61 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { PolicyClass } from '../access-control/PoliciesClass'; +import { RolesClass } from '../access-control/RolesClass'; +import { ApiCollectionClass } from '../entity/ApiCollectionClass'; +import { ContainerClass } from '../entity/ContainerClass'; +import { DashboardClass } from '../entity/DashboardClass'; +import { MlModelClass } from '../entity/MlModelClass'; +import { PipelineClass } from '../entity/PipelineClass'; +import { SearchIndexClass } from '../entity/SearchIndexClass'; +import { TableClass } from '../entity/TableClass'; +import { TopicClass } from '../entity/TopicClass'; +import { UserClass } from '../user/UserClass'; + +export interface EntityData { + isOwnerPolicy: PolicyClass; + matchAnyTagPolicy: PolicyClass; + isOwnerRole: RolesClass; + matchAnyTagRole: RolesClass; + userWithOwnerPermission: UserClass; + userWithTagPermission: UserClass; + withOwner: { + apiCollectionWithOwner: ApiCollectionClass; + containerWithOwner: ContainerClass; + dashboardWithOwner: DashboardClass; + mlModelWithOwner: MlModelClass; + pipelineWithOwner: PipelineClass; + searchIndexWithOwner: SearchIndexClass; + tableWithOwner: TableClass; + topicWithOwner: TopicClass; + }; + withTag: { + apiCollectionWithTag: ApiCollectionClass; + containerWithTag: ContainerClass; + dashboardWithTag: DashboardClass; + mlModelWithTag: MlModelClass; + pipelineWithTag: PipelineClass; + searchIndexWithTag: SearchIndexClass; + tableWithTag: TableClass; + topicWithTag: TopicClass; + }; +} + +export type AssetTypes = + | ApiCollectionClass + | ContainerClass + | DashboardClass + | MlModelClass + | PipelineClass + | TableClass + | TopicClass; diff --git a/openmetadata-ui/src/main/resources/ui/playwright/support/tag/TagClass.ts b/openmetadata-ui/src/main/resources/ui/playwright/support/tag/TagClass.ts index 4d15a077f808..b0c86b708052 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/support/tag/TagClass.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/support/tag/TagClass.ts @@ -66,6 +66,7 @@ export class TagClass { async visitPage(page: Page) { await visitClassificationPage( page, + this.responseData.classification.name, this.responseData.classification.displayName ); await page.getByTestId(this.data.name).click(); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/support/team/TeamClass.ts b/openmetadata-ui/src/main/resources/ui/playwright/support/team/TeamClass.ts index d580c59c8b7d..a6553bc4eda0 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/support/team/TeamClass.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/support/team/TeamClass.ts @@ -10,8 +10,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { APIRequestContext } from '@playwright/test'; +import { APIRequestContext, expect, Page } from '@playwright/test'; +import { GlobalSettingOptions } from '../../constant/settings'; import { uuid } from '../../utils/common'; +import { settingClick } from '../../utils/sidebar'; +import { searchTeam } from '../../utils/team'; type ResponseDataType = { name: string; displayName: string; @@ -46,6 +49,28 @@ export class TeamClass { return this.responseData; } + async visitTeamPage(page: Page) { + // complete url since we are making basic and advance call to get the details of the team + const fetchOrganizationResponse = page.waitForResponse( + `/api/v1/teams/name/Organization?fields=users%2CdefaultRoles%2Cpolicies%2CchildrenCount%2Cdomains&include=all` + ); + await settingClick(page, GlobalSettingOptions.TEAMS); + await fetchOrganizationResponse; + + await searchTeam(page, this.responseData?.['displayName']); + + await page + .locator(`[data-row-key="${this.data.name}"]`) + .getByRole('link') + .click(); + + await page.waitForLoadState('networkidle'); + + await expect(page.getByTestId('team-heading')).toHaveText( + this.data.displayName + ); + } + async create(apiContext: APIRequestContext) { const response = await apiContext.post('/api/v1/teams', { data: this.data, diff --git a/openmetadata-ui/src/main/resources/ui/playwright/utils/advancedSearch.ts b/openmetadata-ui/src/main/resources/ui/playwright/utils/advancedSearch.ts index b8d911fa594e..9c7a1f0242f6 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/utils/advancedSearch.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/utils/advancedSearch.ts @@ -63,6 +63,82 @@ export const FIELDS: EntityFields[] = [ localSearch: false, skipConditions: ['isNull', 'isNotNull'], // Null and isNotNull conditions are not present for display name }, + { + id: 'Service Type', + name: 'serviceType', + localSearch: false, + }, + { + id: 'Schema Field', + name: 'messageSchema.schemaFields.name.keyword', + localSearch: false, + }, + { + id: 'Container Column', + name: 'dataModel.columns.name.keyword', + localSearch: false, + }, + { + id: 'Data Model Type', + name: 'dataModelType', + localSearch: false, + }, + { + id: 'Field', + name: 'fields.name.keyword', + localSearch: false, + }, + { + id: 'Task', + name: 'tasks.displayName.keyword', + localSearch: false, + }, + { + id: 'Domain', + name: 'domain.displayName.keyword', + localSearch: false, + }, + { + id: 'Name', + name: 'name.keyword', + localSearch: false, + skipConditions: ['isNull', 'isNotNull'], // Null and isNotNull conditions are not present for name + }, + { + id: 'Project', + name: 'project.keyword', + localSearch: false, + }, + { + id: 'Status', + name: 'status', + localSearch: false, + }, + { + id: 'Table Type', + name: 'tableType', + localSearch: false, + }, + { + id: 'Entity Type', + name: 'entityType', + localSearch: false, + }, + { + id: 'Chart', + name: 'charts.displayName.keyword', + localSearch: false, + }, + { + id: 'Response Schema Field', + name: 'responseSchema.schemaFields.name.keyword', + localSearch: false, + }, + { + id: 'Request Schema Field', + name: 'requestSchema.schemaFields.name.keyword', + localSearch: false, + }, ]; export const OPERATOR = { diff --git a/openmetadata-ui/src/main/resources/ui/playwright/utils/advancedSearchCustomProperty.ts b/openmetadata-ui/src/main/resources/ui/playwright/utils/advancedSearchCustomProperty.ts new file mode 100644 index 000000000000..2540d71065b7 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/playwright/utils/advancedSearchCustomProperty.ts @@ -0,0 +1,28 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { expect, Page } from '@playwright/test'; + +export const advanceSearchSaveFilter = async ( + page: Page, + propertyValue: string +) => { + const searchResponse = page.waitForResponse( + '/api/v1/search/query?*index=dataAsset&from=0&size=10*' + ); + await page.getByTestId('apply-btn').click(); + + const res = await searchResponse; + const json = await res.json(); + + expect(JSON.stringify(json)).toContain(propertyValue); +}; diff --git a/openmetadata-ui/src/main/resources/ui/playwright/utils/alert.ts b/openmetadata-ui/src/main/resources/ui/playwright/utils/alert.ts index e424bc5f885c..37aae920a894 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/utils/alert.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/utils/alert.ts @@ -37,7 +37,7 @@ import { toastNotification, uuid, } from './common'; -import { getEntityDisplayName } from './entity'; +import { getEntityDisplayName, getTextFromHtmlString } from './entity'; import { validateFormNameFieldInput } from './form'; import { addFilterWithUsersListInput, @@ -502,9 +502,9 @@ export const verifyAlertDetails = async ({ ); if (description) { - // Check alert name - await expect(page.getByTestId('alert-description')).toContainText( - description + // Check alert description + await expect(page.getByTestId('markdown-parser')).toContainText( + getTextFromHtmlString(description) ); } diff --git a/openmetadata-ui/src/main/resources/ui/playwright/utils/authTeardown.ts b/openmetadata-ui/src/main/resources/ui/playwright/utils/authTeardown.ts new file mode 100644 index 000000000000..633a9f0bae11 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/playwright/utils/authTeardown.ts @@ -0,0 +1,88 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { APIRequestContext, Page } from '@playwright/test'; +import { + ORGANIZATION_POLICY_RULES, + VIEW_ALL_RULE, +} from '../constant/permission'; +import { AdminClass } from '../support/user/AdminClass'; +import { getApiContext } from './common'; + +export const restoreOrganizationDefaultRole = async ( + apiContext: APIRequestContext +) => { + const organizationTeamResponse = await apiContext + .get(`/api/v1/teams/name/Organization`) + .then((res) => res.json()); + + const dataConsumerRoleResponse = await apiContext + .get('/api/v1/roles/name/DataConsumer') + .then((res) => res.json()); + + await apiContext.patch(`/api/v1/teams/${organizationTeamResponse.id}`, { + data: [ + { + op: 'replace', + path: '/defaultRoles', + value: [ + { + id: dataConsumerRoleResponse.id, + type: 'role', + }, + ], + }, + ], + headers: { + 'Content-Type': 'application/json-patch+json', + }, + }); +}; + +export const updateDefaultOrganizationPolicy = async ( + apiContext: APIRequestContext +) => { + const orgPolicyResponse = await apiContext + .get('/api/v1/policies/name/OrganizationPolicy') + .then((response) => response.json()); + + await apiContext.patch(`/api/v1/policies/${orgPolicyResponse.id}`, { + data: [ + { + op: 'replace', + path: '/rules', + value: [...ORGANIZATION_POLICY_RULES, ...VIEW_ALL_RULE], + }, + ], + headers: { + 'Content-Type': 'application/json-patch+json', + }, + }); +}; + +const restoreRolesAndPolicies = async (page: Page) => { + const { apiContext, afterAction } = await getApiContext(page); + // Remove organization policy and role + await restoreOrganizationDefaultRole(apiContext); + // update default Organization policy + await updateDefaultOrganizationPolicy(apiContext); + + await afterAction(); +}; + +export const resetPolicyChanges = async (page: Page, admin: AdminClass) => { + await admin.login(page); + await page.waitForURL('**/my-data'); + await restoreRolesAndPolicies(page); + await admin.logout(page); + await page.waitForURL('**/signin'); +}; diff --git a/openmetadata-ui/src/main/resources/ui/playwright/utils/bot.ts b/openmetadata-ui/src/main/resources/ui/playwright/utils/bot.ts index 3e0c01b48083..0467fb53e188 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/utils/bot.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/utils/bot.ts @@ -221,3 +221,39 @@ export const redirectToBotPage = async (page: Page) => { await settingClick(page, GlobalSettingOptions.BOTS); await fetchResponse; }; + +export const resetTokenFromBotPage = async ( + page: Page, + bot: { + name: string; + testId: string; + } +) => { + await settingClick(page, GlobalSettingOptions.BOTS); + + await page.getByTestId('searchbar').click(); + await page.getByTestId('searchbar').fill(bot.name); + + await expect(page.getByTestId(bot.testId)).toBeVisible(); + + await page.getByTestId(bot.testId).click(); + + await expect(page.getByTestId('revoke-button')).toBeVisible(); + + await page.getByTestId('revoke-button').click(); + + await expect(page.getByTestId('save-button')).toBeVisible(); + + await page.getByTestId('save-button').click(); + + await expect(page.getByTestId('token-expiry').locator('div')).toBeVisible(); + + await page.getByText('hr').click(); + await page.getByText('Unlimited days').click(); + + await expect(page.getByTestId('save-edit')).toBeVisible(); + + await page.getByTestId('save-edit').click(); + + await redirectToHomePage(page); +}; diff --git a/openmetadata-ui/src/main/resources/ui/playwright/utils/common.ts b/openmetadata-ui/src/main/resources/ui/playwright/utils/common.ts index 446e4220cdec..01a7a17f01c7 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/utils/common.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/utils/common.ts @@ -19,8 +19,10 @@ import { sidebarClick } from './sidebar'; export const uuid = () => randomUUID().split('-')[0]; -export const descriptionBox = - '.toastui-editor-md-container > .toastui-editor > .ProseMirror'; +export const descriptionBox = '.om-block-editor[contenteditable="true"]'; +export const descriptionBoxReadOnly = + '.om-block-editor[contenteditable="false"]'; + export const INVALID_NAMES = { MAX_LENGTH: 'a87439625b1c2d3e4f5061728394a5b6c7d8e90a1b2c3d4e5f67890aba87439625b1c2d3e4f5061728394a5b6c7d8e90a1b2c3d4e5f67890abName can be a maximum of 128 characters', @@ -39,8 +41,7 @@ export const NAME_MAX_LENGTH_VALIDATION_ERROR = export const getToken = async (page: Page) => { return page.evaluate( () => - JSON.parse(localStorage.getItem('om-session') ?? '{}')?.state - ?.oidcIdToken ?? '' + JSON.parse(localStorage.getItem('om-session') ?? '{}')?.oidcIdToken ?? '' ); }; @@ -151,7 +152,7 @@ export const visitOwnProfilePage = async (page: Page) => { export const assignDomain = async ( page: Page, - domain: { name: string; displayName: string } + domain: { name: string; displayName: string; fullyQualifiedName?: string } ) => { await page.getByTestId('add-domain').click(); await page.waitForSelector('[data-testid="loader"]', { state: 'detached' }); @@ -159,11 +160,13 @@ export const assignDomain = async ( `/api/v1/search/query?q=*${encodeURIComponent(domain.name)}*` ); await page - .getByTestId('selectable-list') + .getByTestId('domain-selectable-tree') .getByTestId('searchbar') .fill(domain.name); await searchDomain; - await page.getByRole('listitem', { name: domain.displayName }).click(); + + await page.getByTestId(`tag-${domain.fullyQualifiedName}`).click(); + await page.getByTestId('saveAssociatedTag').click(); await expect(page.getByTestId('domain-link')).toContainText( domain.displayName @@ -172,33 +175,42 @@ export const assignDomain = async ( export const updateDomain = async ( page: Page, - domain: { name: string; displayName: string } + domain: { name: string; displayName: string; fullyQualifiedName?: string } ) => { await page.getByTestId('add-domain').click(); await page.waitForSelector('[data-testid="loader"]', { state: 'detached' }); - await page.getByTestId('selectable-list').getByTestId('searchbar').clear(); + + await page + .getByTestId('domain-selectable-tree') + .getByTestId('searchbar') + .clear(); + const searchDomain = page.waitForResponse( `/api/v1/search/query?q=*${encodeURIComponent(domain.name)}*` ); await page - .getByTestId('selectable-list') + .getByTestId('domain-selectable-tree') .getByTestId('searchbar') .fill(domain.name); await searchDomain; - await page.getByRole('listitem', { name: domain.displayName }).click(); + + await page.getByTestId(`tag-${domain.fullyQualifiedName}`).click(); + await page.getByTestId('saveAssociatedTag').click(); await expect(page.getByTestId('domain-link')).toContainText( domain.displayName ); }; -export const removeDomain = async (page: Page) => { +export const removeDomain = async ( + page: Page, + domain: { name: string; displayName: string; fullyQualifiedName?: string } +) => { await page.getByTestId('add-domain').click(); await page.waitForSelector('[data-testid="loader"]', { state: 'detached' }); - await expect(page.getByTestId('remove-owner').locator('path')).toBeVisible(); - - await page.getByTestId('remove-owner').locator('svg').click(); + await page.getByTestId(`tag-${domain.fullyQualifiedName}`).click(); + await page.getByTestId('saveAssociatedTag').click(); await expect(page.getByTestId('no-domain-text')).toContainText('No Domain'); }; diff --git a/openmetadata-ui/src/main/resources/ui/playwright/utils/conditionalPermissions.ts b/openmetadata-ui/src/main/resources/ui/playwright/utils/conditionalPermissions.ts new file mode 100644 index 000000000000..82cadb5365f8 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/playwright/utils/conditionalPermissions.ts @@ -0,0 +1,304 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { APIRequestContext, expect, Page } from '@playwright/test'; +import { conditionalPermissionsEntityData } from '../constant/conditionalPermissions'; +import { + VIEW_ALL_WITH_IS_OWNER, + VIEW_ALL_WITH_MATCH_TAG_CONDITION, +} from '../constant/permission'; +import { TableClass } from '../support/entity/TableClass'; +import { AssetTypes } from '../support/interfaces/ConditionalPermissions.interface'; +import { redirectToHomePage } from './common'; + +export const conditionalPermissionsPrerequisites = async ( + apiContext: APIRequestContext +) => { + const { + isOwnerPolicy, + matchAnyTagPolicy, + isOwnerRole, + matchAnyTagRole, + userWithOwnerPermission, + userWithTagPermission, + withOwner, + withTag, + } = conditionalPermissionsEntityData; + await userWithOwnerPermission.create(apiContext); + await userWithTagPermission.create(apiContext); + await withOwner.apiCollectionWithOwner.create(apiContext); + await withTag.apiCollectionWithTag.create(apiContext); + await withOwner.containerWithOwner.create(apiContext); + await withTag.containerWithTag.create(apiContext); + await withOwner.dashboardWithOwner.create(apiContext); + await withTag.dashboardWithTag.create(apiContext); + await withOwner.mlModelWithOwner.create(apiContext); + await withTag.mlModelWithTag.create(apiContext); + await withOwner.pipelineWithOwner.create(apiContext); + await withTag.pipelineWithTag.create(apiContext); + await withOwner.searchIndexWithOwner.create(apiContext); + await withTag.searchIndexWithTag.create(apiContext); + await withOwner.tableWithOwner.create(apiContext); + await withTag.tableWithTag.create(apiContext); + await withOwner.topicWithOwner.create(apiContext); + await withTag.topicWithTag.create(apiContext); + + const isOwnerPolicyResponse = await isOwnerPolicy.create( + apiContext, + VIEW_ALL_WITH_IS_OWNER + ); + const matchAnyTagPolicyResponse = await matchAnyTagPolicy.create( + apiContext, + VIEW_ALL_WITH_MATCH_TAG_CONDITION + ); + + const isOwnerRoleResponse = await isOwnerRole.create(apiContext, [ + isOwnerPolicyResponse.fullyQualifiedName, + ]); + const matchAnyTagRoleResponse = await matchAnyTagRole.create(apiContext, [ + matchAnyTagPolicyResponse.fullyQualifiedName, + ]); + + await userWithOwnerPermission.patch({ + apiContext, + patchData: [ + { + op: 'replace', + path: '/roles', + value: [ + { + id: isOwnerRoleResponse.id, + type: 'role', + name: isOwnerRoleResponse.name, + }, + ], + }, + ], + }); + await userWithTagPermission.patch({ + apiContext, + patchData: [ + { + op: 'replace', + path: '/roles', + value: [ + { + id: matchAnyTagRoleResponse.id, + type: 'role', + name: matchAnyTagRoleResponse.name, + }, + ], + }, + ], + }); + + const ownerPatchData = { + data: [ + { + op: 'replace', + path: '/owners', + value: [ + { + id: userWithOwnerPermission.responseData.id, + type: 'user', + }, + ], + }, + ], + headers: { + 'Content-Type': 'application/json-patch+json', + }, + }; + + const tagPatchData = { + data: [ + { + op: 'replace', + path: '/tags', + value: [ + { + tagFQN: 'PersonalData.Personal', + source: 'Classification', + labelType: 'Manual', + name: 'Personal', + state: 'Confirmed', + }, + ], + }, + ], + headers: { + 'Content-Type': 'application/json-patch+json', + }, + }; + + for (const entity of Object.values(withOwner)) { + await apiContext.patch( + `/api/v1/services/${entity.serviceType}/${entity.serviceResponseData.id}`, + ownerPatchData + ); + } + + await apiContext.patch( + `/api/v1/databases/${withOwner.tableWithOwner.databaseResponseData.id}`, + ownerPatchData + ); + + await apiContext.patch( + `/api/v1/databaseSchemas/${withOwner.tableWithOwner.schemaResponseData.id}`, + ownerPatchData + ); + + await apiContext.patch( + `/api/v1/containers/${withOwner.containerWithOwner.entityResponseData.id}`, + ownerPatchData + ); + + for (const entity of Object.values(withTag)) { + await apiContext.patch( + `/api/v1/services/${entity.serviceType}/${entity.serviceResponseData.id}`, + tagPatchData + ); + } + + await apiContext.patch( + `/api/v1/databases/${withTag.tableWithTag.databaseResponseData.id}`, + tagPatchData + ); + + await apiContext.patch( + `/api/v1/databaseSchemas/${withTag.tableWithTag.schemaResponseData.id}`, + tagPatchData + ); + + await apiContext.patch( + `/api/v1/containers/${withTag.containerWithTag.entityResponseData.id}`, + tagPatchData + ); +}; + +export const conditionalPermissionsCleanup = async ( + apiContext: APIRequestContext +) => { + const { + isOwnerPolicy, + matchAnyTagPolicy, + isOwnerRole, + matchAnyTagRole, + userWithOwnerPermission, + userWithTagPermission, + withOwner: { + apiCollectionWithOwner, + containerWithOwner, + dashboardWithOwner, + mlModelWithOwner, + pipelineWithOwner, + searchIndexWithOwner, + tableWithOwner, + topicWithOwner, + }, + withTag: { + apiCollectionWithTag, + containerWithTag, + dashboardWithTag, + mlModelWithTag, + pipelineWithTag, + searchIndexWithTag, + tableWithTag, + topicWithTag, + }, + } = conditionalPermissionsEntityData; + await isOwnerRole.delete(apiContext); + await matchAnyTagRole.delete(apiContext); + await isOwnerPolicy.delete(apiContext); + await matchAnyTagPolicy.delete(apiContext); + await userWithOwnerPermission.delete(apiContext); + await userWithTagPermission.delete(apiContext); + await apiCollectionWithOwner.delete(apiContext); + await apiCollectionWithTag.delete(apiContext); + await containerWithOwner.delete(apiContext); + await containerWithTag.delete(apiContext); + await dashboardWithOwner.delete(apiContext); + await dashboardWithTag.delete(apiContext); + await mlModelWithOwner.delete(apiContext); + await mlModelWithTag.delete(apiContext); + await pipelineWithOwner.delete(apiContext); + await pipelineWithTag.delete(apiContext); + await searchIndexWithOwner.delete(apiContext); + await searchIndexWithTag.delete(apiContext); + await tableWithOwner.delete(apiContext); + await tableWithTag.delete(apiContext); + await topicWithOwner.delete(apiContext); + await topicWithTag.delete(apiContext); +}; + +export const getEntityFQN = (assetName: string, asset: AssetTypes) => { + if (assetName === 'database') { + return (asset as TableClass).databaseResponseData.fullyQualifiedName; + } + if (assetName === 'databaseSchema') { + return (asset as TableClass).schemaResponseData.fullyQualifiedName; + } + if (assetName === 'container') { + return asset.entityResponseData.fullyQualifiedName; + } + + return asset.serviceResponseData.fullyQualifiedName; +}; + +export const checkViewAllPermission = async ({ + page, + url1, + url2, + childTabId, + childTabId2, + childTableId2, +}: { + page: Page; + url1: string; + url2: string; + childTabId: string; + childTabId2?: string; + childTableId2?: string; +}) => { + await redirectToHomePage(page); + + // visit the page of the asset with permission + await page.goto(url1); + + // Check if the details are shown properly + await expect(page.getByTestId('data-assets-header')).toBeAttached(); + + await page.waitForSelector(`[data-testid="${childTabId}"]`); + + await page.click(`[data-testid="${childTabId}"]`); + + await expect( + page + .getByTestId('service-children-table') + .getByTestId('no-data-placeholder') + ).not.toBeAttached(); + + if (childTabId2) { + await page.click(`[data-testid="${childTabId2}"]`); + + await expect( + page.getByTestId(childTableId2 ?? '').getByTestId('no-data-placeholder') + ).not.toBeAttached(); + } + + // visit the page of the asset without permission + await page.goto(url2); + + // Check if the no permissions placeholder is shown + await expect(page.getByTestId('permission-error-placeholder')).toBeAttached(); +}; diff --git a/openmetadata-ui/src/main/resources/ui/playwright/utils/customProperty.ts b/openmetadata-ui/src/main/resources/ui/playwright/utils/customProperty.ts index 994b61e70e10..64461e0f47be 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/utils/customProperty.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/utils/customProperty.ts @@ -21,7 +21,12 @@ import { ENTITY_PATH, } from '../support/entity/Entity.interface'; import { UserClass } from '../support/user/UserClass'; -import { clickOutside, descriptionBox, uuid } from './common'; +import { + clickOutside, + descriptionBox, + descriptionBoxReadOnly, + uuid, +} from './common'; export enum CustomPropertyType { STRING = 'String', @@ -107,16 +112,9 @@ export const setValueForProperty = async (data: { const patchRequest = page.waitForResponse(`/api/v1/${endpoint}/*`); switch (propertyType) { case 'markdown': - await page - .locator( - '.toastui-editor-md-container > .toastui-editor > .ProseMirror' - ) - .isVisible(); - await page - .locator( - '.toastui-editor-md-container > .toastui-editor > .ProseMirror' - ) - .fill(value); + await page.locator(descriptionBox).isVisible(); + await page.click(descriptionBox); + await page.keyboard.type(value); await page.locator('[data-testid="save"]').click(); break; @@ -282,6 +280,10 @@ export const validateValueForProperty = async (data: { await expect( page.getByRole('row', { name: `${values[0]} ${values[1]}` }) ).toBeVisible(); + } else if (propertyType === 'markdown') { + await expect( + container.locator(descriptionBoxReadOnly).last() + ).toContainText(value.replace(/\*|_/gi, '')); } else if ( ![ 'entityReference', @@ -681,7 +683,7 @@ export const addCustomPropertiesForEntity = async ({ // Description - await page.fill(descriptionBox, customPropertyData.description); + await page.locator(descriptionBox).fill(customPropertyData.description); const createPropertyPromise = page.waitForResponse( '/api/v1/metadata/types/name/*?fields=customProperties' diff --git a/openmetadata-ui/src/main/resources/ui/playwright/utils/customizeLandingPage.ts b/openmetadata-ui/src/main/resources/ui/playwright/utils/customizeLandingPage.ts index a4c2ddf4d4e9..c1fa70f2f5ac 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/utils/customizeLandingPage.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/utils/customizeLandingPage.ts @@ -24,20 +24,20 @@ export const navigateToCustomizeLandingPage = async ( ) => { const getPersonas = page.waitForResponse('/api/v1/personas*'); - await settingClick(page, GlobalSettingOptions.PERSONA); + await settingClick(page, GlobalSettingOptions.CUSTOMIZE_LANDING_PAGE); await getPersonas; const getCustomPageDataResponse = page.waitForResponse( - `/api/v1/docStore/name/persona.${encodeURIComponent(personaName)}` + `/api/v1/docStore/name/persona.${encodeURIComponent( + personaName + )}.Page.LandingPage` ); // Navigate to the customize landing page - await page.getByTestId(`persona-details-card-${personaName}`).click(); - - await page.getByRole('tab', { name: 'Customize UI' }).click(); - - await page.getByTestId('LandingPage').click(); + await page.click( + `[data-testid="persona-details-card-${personaName}"] [data-testid="customize-page-button"]` + ); expect((await getCustomPageDataResponse).status()).toBe( customPageDataResponse diff --git a/openmetadata-ui/src/main/resources/ui/playwright/utils/domain.ts b/openmetadata-ui/src/main/resources/ui/playwright/utils/domain.ts index 3704966267e1..0df4ccaf35dd 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/utils/domain.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/utils/domain.ts @@ -10,8 +10,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { expect, Page } from '@playwright/test'; +import test, { expect, Page } from '@playwright/test'; import { get, isEmpty, isUndefined } from 'lodash'; +import { SidebarItem } from '../constant/sidebar'; import { DataProduct } from '../support/domain/DataProduct'; import { Domain } from '../support/domain/Domain'; import { SubDomain } from '../support/domain/SubDomain'; @@ -27,8 +28,10 @@ import { INVALID_NAMES, NAME_MAX_LENGTH_VALIDATION_ERROR, NAME_VALIDATION_ERROR, + redirectToHomePage, } from './common'; import { addOwner } from './entity'; +import { sidebarClick } from './sidebar'; export const assignDomain = async (page: Page, domain: Domain['data']) => { await page.getByTestId('add-domain').click(); @@ -113,13 +116,45 @@ export const selectSubDomain = async ( domain: Domain['data'], subDomain: SubDomain['data'] ) => { - await page - .getByRole('menuitem', { name: domain.displayName }) - .locator('span') - .click(); + const menuItem = page.getByRole('menuitem', { name: domain.displayName }); + const isSelected = await menuItem.evaluate((element) => { + return element.classList.contains('ant-menu-item-selected'); + }); + + if (!isSelected) { + const subDomainRes = page.waitForResponse( + '/api/v1/search/query?*&from=0&size=50&index=domain_search_index' + ); + await menuItem.click(); + await subDomainRes; + } await page.getByTestId('subdomains').getByText('Sub Domains').click(); + const res = page.waitForResponse( + '/api/v1/search/query?*&index=data_product_search_index' + ); await page.getByTestId(subDomain.name).click(); + await res; +}; + +export const selectDataProductFromTab = async ( + page: Page, + dataProduct: DataProduct['data'] +) => { + const dpRes = page.waitForResponse( + '/api/v1/search/query?*&from=0&size=50&index=data_product_search_index' + ); + await page.getByText('Data Products').click(); + + await dpRes; + + const dpDataRes = page.waitForResponse('/api/v1/dataProducts/name/*'); + + await page + .getByTestId(`explore-card-${dataProduct.name}`) + .getByTestId('entity-link') + .click(); + await dpDataRes; }; export const selectDataProduct = async ( @@ -132,11 +167,7 @@ export const selectDataProduct = async ( .locator('span') .click(); - await page.getByText('Data Products').click(); - await page - .getByTestId(`explore-card-${dataProduct.name}`) - .getByTestId('entity-link') - .click(); + await selectDataProductFromTab(page, dataProduct); }; const goToAssetsTab = async (page: Page, domain: Domain['data']) => { @@ -151,7 +182,7 @@ const fillCommonFormItems = async ( ) => { await page.locator('[data-testid="name"]').fill(entity.name); await page.locator('[data-testid="display-name"]').fill(entity.displayName); - await page.fill(descriptionBox, entity.description); + await page.locator(descriptionBox).fill(entity.description); if (!isEmpty(entity.owners) && !isUndefined(entity.owners)) { await addOwner({ page, @@ -280,10 +311,13 @@ export const createSubDomain = async ( export const addAssetsToDomain = async ( page: Page, - domain: Domain['data'], - assets: EntityClass[] + domain: Domain, + assets: EntityClass[], + navigateToAssetsTab = true ) => { - await goToAssetsTab(page, domain); + if (navigateToAssetsTab) { + await goToAssetsTab(page, domain.data); + } await checkAssetsCount(page, 0); await expect(page.getByTestId('no-data-placeholder')).toContainText( @@ -309,20 +343,34 @@ export const addAssetsToDomain = async ( await page.locator(`[data-testid="table-data-card_${fqn}"] input`).check(); } - const assetsAddRes = page.waitForResponse( - `/api/v1/domains/${encodeURIComponent( - domain.fullyQualifiedName ?? '' - )}/assets/add` - ); + const assetsAddRes = page.waitForResponse(`/api/v1/domains/*/assets/add`); + const searchRes = page.waitForResponse((response) => { + const url = new URL(response.url()); + const queryParams = new URLSearchParams(url.search); + const queryFilter = queryParams.get('query_filter'); + + return ( + response + .url() + .includes('/api/v1/search/query?q=**&index=all&from=0&size=15') && + queryFilter !== null && + queryFilter !== '' + ); + }); await page.getByTestId('save-btn').click(); await assetsAddRes; + await searchRes; + + await page.reload(); + await page.waitForLoadState('networkidle'); + await checkAssetsCount(page, assets.length); }; export const addAssetsToDataProduct = async ( page: Page, - dataProduct: DataProduct['data'], + dataProductFqn: string, assets: EntityClass[] ) => { await page.getByTestId('assets').click(); @@ -348,9 +396,7 @@ export const addAssetsToDataProduct = async ( } const assetsAddRes = page.waitForResponse( - `/api/v1/dataProducts/${encodeURIComponent( - dataProduct.fullyQualifiedName ?? '' - )}/assets/add` + `/api/v1/dataProducts/*/assets/add` ); await page.getByTestId('save-btn').click(); await assetsAddRes; @@ -423,3 +469,89 @@ export const createDataProduct = async ( await page.getByTestId('save-data-product').click(); await saveRes; }; + +export const verifyDataProductAssetsAfterDelete = async ( + page: Page, + { + domain, + dataProduct1, + dataProduct2, + assets, + subDomain, + }: { + domain: Domain; + dataProduct1: DataProduct; + dataProduct2: DataProduct; + assets: EntityClass[]; + subDomain?: SubDomain; + } +) => { + const { apiContext } = await getApiContext(page); + const newDataProduct1 = new DataProduct(domain, 'PW_DataProduct_Sales'); + + await test.step('Add assets to DataProduct Sales', async () => { + await redirectToHomePage(page); + await sidebarClick(page, SidebarItem.DOMAIN); + if (subDomain) { + await selectSubDomain(page, domain.data, subDomain.data); + await selectDataProductFromTab(page, dataProduct1.data); + } else { + await selectDataProduct(page, domain.data, dataProduct1.data); + } + await addAssetsToDataProduct( + page, + dataProduct1.responseData.fullyQualifiedName ?? '', + assets.slice(0, 2) + ); + }); + + await test.step('Add assets to DataProduct Finance', async () => { + await redirectToHomePage(page); + await sidebarClick(page, SidebarItem.DOMAIN); + if (subDomain) { + await selectSubDomain(page, domain.data, subDomain.data); + await selectDataProductFromTab(page, dataProduct2.data); + } else { + await selectDataProduct(page, domain.data, dataProduct2.data); + } + await addAssetsToDataProduct( + page, + dataProduct2.responseData.fullyQualifiedName ?? '', + [assets[2]] + ); + }); + + await test.step( + 'Remove Data Product Sales and Create the same again', + async () => { + // Remove sales data product + await dataProduct1.delete(apiContext); + + // Create sales data product again + await redirectToHomePage(page); + await sidebarClick(page, SidebarItem.DOMAIN); + if (subDomain) { + await selectSubDomain(page, domain.data, subDomain.data); + } else { + await selectDomain(page, domain.data); + } + + await createDataProduct(page, newDataProduct1.data); + } + ); + + await test.step( + 'Verify assets are not present in the newly created data product', + async () => { + await redirectToHomePage(page); + await sidebarClick(page, SidebarItem.DOMAIN); + if (subDomain) { + await selectSubDomain(page, domain.data, subDomain.data); + await selectDataProductFromTab(page, newDataProduct1.data); + } else { + await selectDataProduct(page, domain.data, newDataProduct1.data); + } + await checkAssetsCount(page, 0); + } + ); +}; diff --git a/openmetadata-ui/src/main/resources/ui/playwright/utils/entity.ts b/openmetadata-ui/src/main/resources/ui/playwright/utils/entity.ts index b78e65eaa595..3ac93c5339c6 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/utils/entity.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/utils/entity.ts @@ -24,7 +24,7 @@ import { } from '../constant/delete'; import { ES_RESERVED_CHARACTERS } from '../constant/entity'; import { EntityTypeEndpoint } from '../support/entity/Entity.interface'; -import { clickOutside, redirectToHomePage } from './common'; +import { clickOutside, descriptionBox, redirectToHomePage } from './common'; export const visitEntityPage = async (data: { page: Page; @@ -59,7 +59,7 @@ export const addOwner = async ({ await page.getByTestId(initiatorId).click(); if (type === 'Users') { const userListResponse = page.waitForResponse( - '/api/v1/users?limit=*&isBot=false*' + '/api/v1/search/query?q=*isBot:false*index=user_search_index*' ); await page.getByRole('tab', { name: type }).click(); await userListResponse; @@ -330,9 +330,9 @@ export const updateDescription = async ( isModal = false ) => { await page.getByTestId('edit-description').click(); - await page.locator('.ProseMirror').first().click(); - await page.locator('.ProseMirror').first().clear(); - await page.locator('.ProseMirror').first().fill(description); + await page.locator(descriptionBox).first().click(); + await page.locator(descriptionBox).first().clear(); + await page.locator(descriptionBox).first().fill(description); await page.getByTestId('save').click(); if (isModal) { @@ -353,11 +353,10 @@ export const updateDescription = async ( export const assignTag = async ( page: Page, tag: string, - action: 'Add' | 'Edit' = 'Add', - parentId = 'entity-right-panel' + action: 'Add' | 'Edit' = 'Add' ) => { await page - .getByTestId(parentId) + .getByTestId('entity-right-panel') .getByTestId('tags-container') .getByTestId(action === 'Add' ? 'add-tag' : 'edit-button') .click(); @@ -380,7 +379,7 @@ export const assignTag = async ( await expect( page - .getByTestId(parentId) + .getByTestId('entity-right-panel') .getByTestId('tags-container') .getByTestId(`tag-${tag}`) ).toBeVisible(); @@ -784,10 +783,7 @@ const announcementForm = async ( await page.fill('#endTime', `${data.endDate}`); await page.press('#startTime', 'Enter'); - await page.fill( - '.toastui-editor-md-container > .toastui-editor > .ProseMirror', - data.description - ); + await page.locator(descriptionBox).fill(data.description); await page.locator('#announcement-submit').scrollIntoViewIfNeeded(); const announcementSubmit = page.waitForResponse( @@ -1304,6 +1300,7 @@ export const hardDeleteEntity = async ( }; export const checkDataAssetWidget = async (page: Page, serviceType: string) => { + await clickOutside(page); const quickFilterResponse = page.waitForResponse( `/api/v1/search/query?q=&index=dataAsset*${serviceType}*` ); @@ -1356,3 +1353,16 @@ export const getEntityDisplayName = (entity?: { }) => { return entity?.displayName || entity?.name || ''; }; + +/** + * + * @param description HTML string + * @returns Text from HTML string + */ +export const getTextFromHtmlString = (description?: string): string => { + if (!description) { + return ''; + } + + return description.replace(/<[^>]*>/g, '').trim(); +}; diff --git a/openmetadata-ui/src/main/resources/ui/playwright/utils/explore.ts b/openmetadata-ui/src/main/resources/ui/playwright/utils/explore.ts index fbe65243af9d..7cbbc55f46ab 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/utils/explore.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/utils/explore.ts @@ -38,7 +38,8 @@ export const searchAndClickOnOption = async ( export const selectNullOption = async ( page: Page, - filter: { key: string; label: string; value?: string } + filter: { key: string; label: string; value?: string }, + clearFilter = true ) => { const queryFilter = JSON.stringify({ query: { @@ -94,7 +95,9 @@ export const selectNullOption = async ( expect(isQueryFilterPresent).toBeTruthy(); - await page.click(`[data-testid="clear-filters"]`); + if (clearFilter) { + await page.click(`[data-testid="clear-filters"]`); + } }; export const checkCheckboxStatus = async ( diff --git a/openmetadata-ui/src/main/resources/ui/playwright/utils/glossary.ts b/openmetadata-ui/src/main/resources/ui/playwright/utils/glossary.ts index d694d659f3db..2f195d73b43b 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/utils/glossary.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/utils/glossary.ts @@ -25,9 +25,11 @@ import { UserTeamRef, } from '../support/glossary/Glossary.interface'; import { GlossaryTerm } from '../support/glossary/GlossaryTerm'; +import { UserClass } from '../support/user/UserClass'; import { clickOutside, closeFirstPopupAlert, + descriptionBox, getApiContext, INVALID_NAMES, NAME_MAX_LENGTH_VALIDATION_ERROR, @@ -47,9 +49,6 @@ type TaskEntity = { const GLOSSARY_NAME_VALIDATION_ERROR = 'Name size must be between 1 and 128'; -export const descriptionBox = - '.toastui-editor-md-container > .toastui-editor > .ProseMirror'; - export const checkDisplayName = async (page: Page, displayName: string) => { await expect(page.getByTestId('entity-header-display-name')).toHaveText( displayName @@ -64,11 +63,14 @@ export const selectActiveGlossary = async ( const isSelected = await menuItem.evaluate((element) => { return element.classList.contains('ant-menu-item-selected'); }); - if (!isSelected) { const glossaryResponse = page.waitForResponse('/api/v1/glossaryTerms*'); await menuItem.click(); await glossaryResponse; + } else { + await page.waitForSelector('[data-testid="loader"]', { + state: 'detached', + }); } }; @@ -244,7 +246,7 @@ export const createGlossary = async ( await page.fill('[data-testid="name"]', glossaryData.name); - await page.fill(descriptionBox, glossaryData.description); + await page.locator(descriptionBox).fill(glossaryData.description); await expect( page.locator('[data-testid="form-item-alert"]') @@ -390,12 +392,18 @@ export const deleteGlossary = async (page: Page, glossary: GlossaryData) => { export const fillGlossaryTermDetails = async ( page: Page, term: GlossaryTermData, - validateCreateForm = true + validateCreateForm = true, + isGlossaryTerm = false ) => { // Safety check to close potential glossary not found alert // Arrived due to parallel testing await closeFirstPopupAlert(page); - await page.click('[data-testid="add-new-tag-button-header"]'); + + if (isGlossaryTerm) { + await page.click('[data-testid="add-placeholder-button"]'); + } else { + await page.click('[data-testid="add-new-tag-button-header"]'); + } await page.waitForSelector('[role="dialog"].edit-glossary-modal'); @@ -588,28 +596,39 @@ export const updateGlossaryTermDataFromTree = async ( export const validateGlossaryTerm = async ( page: Page, term: GlossaryTermData, - status: 'Draft' | 'Approved' + status: 'Draft' | 'Approved', + isGlossaryTermPage = false ) => { // eslint-disable-next-line no-useless-escape const escapedFqn = term.fullyQualifiedName.replace(/\"/g, '\\"'); const termSelector = `[data-row-key="${escapedFqn}"]`; const statusSelector = `[data-testid="${escapedFqn}-status"]`; - await expect(page.locator(termSelector)).toContainText(term.name); - await expect(page.locator(statusSelector)).toContainText(status); + if (isGlossaryTermPage) { + await expect(page.getByTestId(term.name)).toBeVisible(); + } else { + await expect(page.locator(termSelector)).toContainText(term.name); + await expect(page.locator(statusSelector)).toContainText(status); + } }; export const createGlossaryTerm = async ( page: Page, term: GlossaryTermData, status: 'Draft' | 'Approved', - validateCreateForm = true + validateCreateForm = true, + isGlossaryTermPage = false ) => { - await fillGlossaryTermDetails(page, term, validateCreateForm); + await fillGlossaryTermDetails( + page, + term, + validateCreateForm, + isGlossaryTermPage + ); const glossaryTermResponse = page.waitForResponse('/api/v1/glossaryTerms'); await page.click('[data-testid="save-glossary-term"]'); await glossaryTermResponse; - await validateGlossaryTerm(page, term, status); + await validateGlossaryTerm(page, term, status, isGlossaryTermPage); }; export const createGlossaryTerms = async ( @@ -1071,8 +1090,11 @@ export const approveTagsTask = async ( await taskResolve; await redirectToHomePage(page); + const glossaryTermsResponse = page.waitForResponse('/api/v1/glossaryTerms*'); await sidebarClick(page, SidebarItem.GLOSSARY); + await glossaryTermsResponse; await selectActiveGlossary(page, entity.data.displayName); + await page.waitForLoadState('networkidle'); const tagVisibility = await page.isVisible( `[data-testid="tag-${value.tag}"]` @@ -1221,7 +1243,11 @@ export const filterStatus = async ( await saveButton.click(); const glossaryTermsTable = page.getByTestId('glossary-terms-table'); - const rows = glossaryTermsTable.locator('tbody tr'); + // will select all elements inside the but exclude those with aria-hidden="true" + // since we have added re-sizeable columns, that one entry is present in the tbody + const rows = glossaryTermsTable.locator( + 'tbody.ant-table-tbody > tr:not([aria-hidden="true"])' + ); const statusColumnIndex = 3; for (let i = 0; i < (await rows.count()); i++) { @@ -1411,3 +1437,28 @@ export const updateGlossaryTermReviewers = async ( await page.getByTestId('save-glossary-term').click(); await glossaryTermResponse; }; + +export const checkGlossaryTermDetails = async ( + page: Page, + term: GlossaryTermData, + owner: UserClass, + reviewer: UserClass +) => { + await openEditGlossaryTermModal(page, term); + + await expect(page.locator('[data-testid="name"]')).toHaveValue(term.name); + await expect(page.locator('[data-testid="display-name"]')).toHaveValue( + term.displayName + ); + await expect(page.getByTestId('editor')).toContainText(term.description); + + await expect( + page.locator('[data-testid="owner-container"] [data-testid="owner-link"]') + ).toContainText(owner.responseData.displayName); + + await expect( + page.locator( + '[data-testid="reviewers-container"] [data-testid="owner-link"]' + ) + ).toContainText(reviewer.responseData.displayName); +}; diff --git a/openmetadata-ui/src/main/resources/ui/playwright/utils/importUtils.ts b/openmetadata-ui/src/main/resources/ui/playwright/utils/importUtils.ts index ced6575cb1c1..e5906f554469 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/utils/importUtils.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/utils/importUtils.ts @@ -46,17 +46,15 @@ export const fillDescriptionDetails = async ( description: string ) => { await page.locator('.InovuaReactDataGrid__cell--cell-active').press('Enter'); - await page.click( - '.toastui-editor-md-container > .toastui-editor > .ProseMirror' - ); + await page.click(descriptionBox); - await page.fill( - '.toastui-editor-md-container > .toastui-editor > .ProseMirror', - description - ); + await page.fill(descriptionBox, description); await page.click('[data-testid="save"]'); - await page.click('.InovuaReactDataGrid__cell--cell-active'); + + await expect( + page.locator('.InovuaReactDataGrid__cell--cell-active') + ).not.toContainText('

'); }; export const fillOwnerDetails = async (page: Page, owners: string[]) => { @@ -65,7 +63,7 @@ export const fillOwnerDetails = async (page: Page, owners: string[]) => { .press('Enter', { delay: 100 }); const userListResponse = page.waitForResponse( - '/api/v1/users?limit=*&isBot=false*' + '/api/v1/search/query?q=*isBot:false*index=user_search_index*' ); await page.getByRole('tab', { name: 'Users' }).click(); await userListResponse; @@ -77,7 +75,7 @@ export const fillOwnerDetails = async (page: Page, owners: string[]) => { await page.locator('[data-testid="owner-select-users-search-bar"]').clear(); await page.keyboard.type(owner); await page.waitForResponse( - `/api/v1/search/query?q=*${owner}*%20AND%20isBot:false&from=0&size=25&index=user_search_index` + `/api/v1/search/query?q=*${owner}*%20AND%20isBot:false*index=user_search_index*` ); await page.getByRole('listitem', { name: owner }).click(); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/utils/lineage.ts b/openmetadata-ui/src/main/resources/ui/playwright/utils/lineage.ts index 1c8f2cf5aa69..a7ee6506a4ea 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/utils/lineage.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/utils/lineage.ts @@ -133,10 +133,9 @@ export const dragAndDropNode = async ( await page.hover(originSelector); await page.mouse.down(); const box = (await destinationElement.boundingBox())!; - const x = (box.x + box.width / 2) * 0.25; // 0.25 as zoom factor - const y = (box.y + box.height / 2) * 0.25; // 0.25 as zoom factor - await page.mouse.move(x, y); - await destinationElement.hover(); + const x = box.x + 250; + const y = box.y + box.height / 2; + await page.mouse.move(x, y, { steps: 20 }); await page.mouse.up(); }; @@ -370,6 +369,10 @@ export const applyPipelineFromModal = async ( const saveRes = page.waitForResponse('/api/v1/lineage'); await page.click('[data-testid="save-button"]'); await saveRes; + + await page.waitForSelector('[data-testid="add-edge-modal"]', { + state: 'detached', + }); }; export const deleteNode = async (page: Page, node: EntityClass) => { diff --git a/openmetadata-ui/src/main/resources/ui/playwright/utils/serviceIngestion.ts b/openmetadata-ui/src/main/resources/ui/playwright/utils/serviceIngestion.ts index 314411d1ff10..56fe71a9f66b 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/utils/serviceIngestion.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/utils/serviceIngestion.ts @@ -15,6 +15,7 @@ import { expect, Page } from '@playwright/test'; import { GlobalSettingOptions } from '../constant/settings'; import { EntityTypeEndpoint } from '../support/entity/Entity.interface'; import { toastNotification } from './common'; +import { escapeESReservedCharacters } from './entity'; export enum Services { Database = GlobalSettingOptions.DATABASES, @@ -78,8 +79,16 @@ export const deleteService = async ( serviceName: string, page: Page ) => { + const serviceResponse = page.waitForResponse( + `/api/v1/search/query?q=*${encodeURIComponent( + escapeESReservedCharacters(serviceName) + )}*` + ); + await page.fill('[data-testid="searchbar"]', serviceName); + await serviceResponse; + // click on created service await page.click(`[data-testid="service-name-${serviceName}"]`); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/utils/tag.ts b/openmetadata-ui/src/main/resources/ui/playwright/utils/tag.ts index f5539666bc0f..cffea876f52b 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/utils/tag.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/utils/tag.ts @@ -22,6 +22,7 @@ import { TableClass } from '../support/entity/TableClass'; import { TopicClass } from '../support/entity/TopicClass'; import { TagClass } from '../support/tag/TagClass'; import { + descriptionBox, getApiContext, NAME_MIN_MAX_LENGTH_VALIDATION_ERROR, NAME_VALIDATION_ERROR, @@ -38,23 +39,26 @@ export const TAG_INVALID_NAMES = { export const visitClassificationPage = async ( page: Page, - classificationName: string + classificationName: string, + classificationDisplayName: string ) => { await redirectToHomePage(page); const classificationResponse = page.waitForResponse( '/api/v1/classifications?**' ); + const fetchTags = page.waitForResponse( + `/api/v1/tags?*parent=${classificationName}**` + ); await sidebarClick(page, SidebarItem.TAGS); await classificationResponse; - const fetchTags = page.waitForResponse('/api/v1/tags?*parent=*'); await page - .locator(`[data-testid="side-panel-classification"]`) - .filter({ hasText: classificationName }) + .getByTestId('data-summary-container') + .getByText(classificationDisplayName) .click(); await expect(page.locator('.activeCategory')).toContainText( - classificationName + classificationDisplayName ); await fetchTags; @@ -335,9 +339,9 @@ export const editTagPageDescription = async (page: Page, tag: TagClass) => { await expect(page.getByRole('dialog')).toBeVisible(); - await page.locator('.toastui-editor-pseudo-clipboard').clear(); + await page.locator(descriptionBox).clear(); await page - .locator('.toastui-editor-pseudo-clipboard') + .locator(descriptionBox) .fill(`This is updated test description for tag ${tag.data.name}.`); const editDescription = page.waitForResponse(`/api/v1/tags/*`); @@ -350,7 +354,7 @@ export const editTagPageDescription = async (page: Page, tag: TagClass) => { }; export const verifyCertificationTagPageUI = async (page: Page) => { - await visitClassificationPage(page, 'Certification'); + await visitClassificationPage(page, 'Certification', 'Certification'); const res = page.waitForResponse(`/api/v1/tags/name/*`); await page.getByTestId('Gold').click(); await res; diff --git a/openmetadata-ui/src/main/resources/ui/playwright/utils/team.ts b/openmetadata-ui/src/main/resources/ui/playwright/utils/team.ts index 382f5c2f0146..8cb2c836978b 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/utils/team.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/utils/team.ts @@ -11,11 +11,14 @@ * limitations under the License. */ import { APIRequestContext, expect, Page } from '@playwright/test'; +import { GlobalSettingOptions } from '../constant/settings'; import { TableClass } from '../support/entity/TableClass'; import { TeamClass } from '../support/team/TeamClass'; +import { UserClass } from '../support/user/UserClass'; import { descriptionBox, toastNotification, uuid } from './common'; import { addOwner } from './entity'; import { validateFormNameFieldInput } from './form'; +import { settingClick } from './sidebar'; const TEAM_TYPES = ['Department', 'Division', 'Group']; @@ -39,12 +42,8 @@ export const createTeam = async (page: Page, isPublic?: boolean) => { await page.getByTestId('isJoinable-switch-button').click(); } - await page - .locator('.toastui-editor-md-container > .toastui-editor > .ProseMirror') - .isVisible(); - await page - .locator('.toastui-editor-md-container > .toastui-editor > .ProseMirror') - .fill(teamData.description); + await page.locator(descriptionBox).isVisible(); + await page.locator(descriptionBox).fill(teamData.description); const createTeamResponse = page.waitForResponse('/api/v1/teams'); @@ -225,7 +224,7 @@ export const addTeamHierarchy = async ( await page.click(`.ant-select-dropdown [title="${teamDetails.teamType}"]`); } - await page.fill(descriptionBox, teamDetails.description); + await page.locator(descriptionBox).fill(teamDetails.description); // Saving the created team const saveTeamResponse = page.waitForResponse('/api/v1/teams'); @@ -304,7 +303,7 @@ export const verifyAssetsInTeamsPage = async ( .locator(`a:has-text("${team.data.displayName}")`) .click(); - const res = page.waitForResponse('/api/v1/search/query?*size=15'); + const res = page.waitForResponse('/api/v1/search/query?*size=15*'); await page.getByTestId('assets').click(); await res; @@ -316,3 +315,56 @@ export const verifyAssetsInTeamsPage = async ( page.getByTestId('assets').getByTestId('filter-count') ).toContainText(assetCount.toString()); }; + +export const addUserInTeam = async (page: Page, user: UserClass) => { + const userName = user.data.email.split('@')[0]; + const fetchUsersResponse = page.waitForResponse( + '/api/v1/users?limit=25&isBot=false' + ); + await page.locator('[data-testid="add-new-user"]').click(); + await fetchUsersResponse; + + // Search and select the user + await page + .locator('[data-testid="selectable-list"] [data-testid="searchbar"]') + .fill(user.getUserName()); + + await page + .locator(`[data-testid="selectable-list"] [title="${user.getUserName()}"]`) + .click(); + + await expect( + page.locator( + `[data-testid="selectable-list"] [title="${user.getUserName()}"]` + ) + ).toHaveClass(/active/); + + const updateTeamResponse = page.waitForResponse('/api/v1/users*'); + + // Update the team with the new user + await page.locator('[data-testid="selectable-list-update-btn"]').click(); + await updateTeamResponse; + + // Verify the user is added to the team + + await expect( + page.locator(`[data-testid="${userName.toLowerCase()}"]`) + ).toBeVisible(); +}; + +export const checkTeamTabCount = async (page: Page) => { + const fetchResponse = page.waitForResponse( + '/api/v1/teams/name/*?fields=*childrenCount*include=all' + ); + + await settingClick(page, GlobalSettingOptions.TEAMS); + + const response = await fetchResponse; + const jsonRes = await response.json(); + + await expect( + page.locator( + '[data-testid="teams"] [data-testid="count"] [data-testid="filter-count"]' + ) + ).toContainText(jsonRes.childrenCount.toString()); +}; diff --git a/openmetadata-ui/src/main/resources/ui/playwright/utils/user.ts b/openmetadata-ui/src/main/resources/ui/playwright/utils/user.ts index eaee1add4451..8a6d184398a3 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/utils/user.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/utils/user.ts @@ -30,6 +30,7 @@ import { SidebarItem } from '../constant/sidebar'; import { UserClass } from '../support/user/UserClass'; import { descriptionBox, + descriptionBoxReadOnly, getAuthContext, getToken, redirectToHomePage, @@ -97,6 +98,12 @@ export const deletedUserChecks = async (page: Page) => { export const visitUserProfilePage = async (page: Page, userName: string) => { await settingClick(page, GlobalSettingOptions.USERS); + await page.waitForSelector( + '[data-testid="user-list-v1-component"] [data-testid="loader"]', + { + state: 'detached', + } + ); const userResponse = page.waitForResponse( '/api/v1/search/query?q=**&from=0&size=*&index=*' ); @@ -259,7 +266,7 @@ export const editDescription = async ( // Verify the updated description const description = page.locator( - '[data-testid="asset-description-container"] .toastui-editor-contents > p' + `[data-testid="asset-description-container"] ${descriptionBoxReadOnly}` ); await expect(description).toContainText(updatedDescription); @@ -470,9 +477,9 @@ export const permanentDeleteUser = async ( ); await page.click('[data-testid="confirm-button"]'); await hardDeleteUserResponse; - await reFetchUsers; await toastNotification(page, `"${displayName}" deleted successfully!`); + await reFetchUsers; // Wait for the loader to disappear await page.waitForSelector('[data-testid="loader"]', { state: 'hidden' }); @@ -701,7 +708,7 @@ export const addUser = async ( await page.fill('[data-testid="displayName"]', name); - await page.fill(descriptionBox, 'Adding new user'); + await page.locator(descriptionBox).fill('Adding new user'); await page.click(':nth-child(2) > .ant-radio > .ant-radio-input'); await page.fill('#password', password); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/utils/userDetails.ts b/openmetadata-ui/src/main/resources/ui/playwright/utils/userDetails.ts new file mode 100644 index 000000000000..cf8caa7b0fd0 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/playwright/utils/userDetails.ts @@ -0,0 +1,34 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Page } from '@playwright/test'; +import { clickOutside, redirectToHomePage } from './common'; + +export const redirectToUserPage = async (page: Page) => { + await redirectToHomePage(page); + + await page.getByTestId('dropdown-profile').click(); + + // Hover on the profile avatar to close the name tooltip + await page.getByTestId('profile-avatar').hover(); + + await page.waitForSelector('.profile-dropdown', { state: 'visible' }); + + const getUserDetails = page.waitForResponse(`/api/v1/users/name/*`); + + await page.locator('.profile-dropdown').getByTestId('user-name').click(); + + await getUserDetails; + + // Close the profile dropdown + await clickOutside(page); +}; diff --git a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/workflows/dbt.md b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/workflows/dbt.md index 80094c8a134a..c03a86d9049f 100644 --- a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/workflows/dbt.md +++ b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/workflows/dbt.md @@ -336,6 +336,16 @@ $$section Set the `Enable Debug Log` toggle to set the logging level of the process to debug. You can check these logs in the Ingestion tab of the service and dig deeper into any errors you might find. $$ + +$$section +### Search Tables Across Databases $(id="searchAcrossDatabases") + +Option to search across database services for tables or not for processing dbt metadata ingestion. +If this option is enabled, OpenMetadata will first search for tables within the same database service if tables are not found it will search across all database services. + +If the option is disabled, the search will be limited to the tables within the same database service. +$$ + $$section ### Update Descriptions $(id="dbtUpdateDescriptions") @@ -346,6 +356,16 @@ If the option is disabled, only tables and columns without any existing descript However, if the option is enabled, descriptions for all tables and columns in the dbt manifest will be updated in OpenMetadata. $$ +$$section +### Update Owners $(id="dbtUpdateOwners") + +This options updates the table owner in OpenMetadata with owners from dbt. + +If the option is disabled, only tables without any existing owners will have their owners updated based on the dbt manifest. + +However, if the option is enabled, owners for all tables and columns in the dbt manifest will be updated in OpenMetadata. +$$ + $$section ### Include dbt Tags $(id="includeTags") diff --git a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Pipeline/Airbyte.md b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Pipeline/Airbyte.md index 930ce605284d..75b314b0d7ca 100644 --- a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Pipeline/Airbyte.md +++ b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Pipeline/Airbyte.md @@ -25,3 +25,9 @@ $$section ### Password $(id="password") Password to connect to Airbyte. $$ + +$$section +### Api Version $(id="apiVersion") + +Version of the Airbyte REST API by default `api/v1`. +$$ \ No newline at end of file diff --git a/openmetadata-ui/src/main/resources/ui/src/App.tsx b/openmetadata-ui/src/main/resources/ui/src/App.tsx index cbcc1eaef496..a038ba039a38 100644 --- a/openmetadata-ui/src/main/resources/ui/src/App.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/App.tsx @@ -63,10 +63,12 @@ const App: FC = () => { ? '/favicon.png' : applicationConfig?.customLogoConfig?.customFaviconUrlPath ?? '/favicon.png'; - const link = document.querySelector('link[rel~="icon"]'); + const link = document.querySelectorAll('link[rel~="icon"]'); - if (link) { - link.setAttribute('href', faviconHref); + if (!isEmpty(link)) { + link.forEach((item) => { + item.setAttribute('href', faviconHref); + }); } }, [applicationConfig]); diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/img/appScreenshots/DataRetentionApplication.png b/openmetadata-ui/src/main/resources/ui/src/assets/img/appScreenshots/DataRetentionApplication.png new file mode 100644 index 000000000000..aff9b000643f Binary files /dev/null and b/openmetadata-ui/src/main/resources/ui/src/assets/img/appScreenshots/DataRetentionApplication.png differ diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/DataRetentionApplication.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/DataRetentionApplication.svg new file mode 100644 index 000000000000..e8e59f7b0d71 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/DataRetentionApplication.svg @@ -0,0 +1,56 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/data-assets.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/data-assets.svg index ded89c071eb3..475eb3222e12 100644 --- a/openmetadata-ui/src/main/resources/ui/src/assets/svg/data-assets.svg +++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/data-assets.svg @@ -1,27 +1 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - + \ No newline at end of file diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/governance.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/governance.svg deleted file mode 100644 index da1ed7dcc114..000000000000 --- a/openmetadata-ui/src/main/resources/ui/src/assets/svg/governance.svg +++ /dev/null @@ -1,21 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/homepage.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/homepage.svg deleted file mode 100644 index 4ceab3740731..000000000000 --- a/openmetadata-ui/src/main/resources/ui/src/assets/svg/homepage.svg +++ /dev/null @@ -1,15 +0,0 @@ - - - - - - - - - - - - - - - diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-edit-circle.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-edit-circle.svg new file mode 100644 index 000000000000..416e5fbbf474 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-edit-circle.svg @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-format-block-quote.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-format-block-quote.svg new file mode 100644 index 000000000000..a626260650fd --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-format-block-quote.svg @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-format-highlight.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-format-highlight.svg new file mode 100644 index 000000000000..efeb1f6fce77 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-format-highlight.svg @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-format-horizontal-line.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-format-horizontal-line.svg new file mode 100644 index 000000000000..aeb9db3e5ad4 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-format-horizontal-line.svg @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-format-image-inline.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-format-image-inline.svg new file mode 100644 index 000000000000..c67825ead370 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-format-image-inline.svg @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-subdomain.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-subdomain.svg new file mode 100644 index 000000000000..544403812a74 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-subdomain.svg @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/navigation.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/navigation.svg deleted file mode 100644 index daa0c13adb5f..000000000000 --- a/openmetadata-ui/src/main/resources/ui/src/assets/svg/navigation.svg +++ /dev/null @@ -1,5 +0,0 @@ - - - - - diff --git a/openmetadata-ui/src/main/resources/ui/src/components/APIEndpoint/APIEndpointSchema/APIEndpointSchema.tsx b/openmetadata-ui/src/main/resources/ui/src/components/APIEndpoint/APIEndpointSchema/APIEndpointSchema.tsx index 0ebbf5a8e45e..f766cc1e34a1 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/APIEndpoint/APIEndpointSchema/APIEndpointSchema.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/APIEndpoint/APIEndpointSchema/APIEndpointSchema.tsx @@ -51,7 +51,7 @@ import { updateFieldDescription, updateFieldTags, } from '../../../utils/TableUtils'; -import RichTextEditorPreviewer from '../../common/RichTextEditor/RichTextEditorPreviewer'; +import RichTextEditorPreviewerV1 from '../../common/RichTextEditor/RichTextEditorPreviewerV1'; import ToggleExpandButton from '../../common/ToggleExpandButton/ToggleExpandButton'; import { ColumnFilter } from '../../Database/ColumnFilter/ColumnFilter.component'; import TableDescription from '../../Database/TableDescription/TableDescription.component'; @@ -201,7 +201,7 @@ const APIEndpointSchema: FC = ({ {isVersionView ? ( - + ) : ( getEntityName(record) )} @@ -216,7 +216,7 @@ const APIEndpointSchema: FC = ({ (dataType: DataType, record: Field) => ( {isVersionView ? ( - ) : ( diff --git a/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedCard/FeedCardBody/FeedCardBody.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedCard/FeedCardBody/FeedCardBody.test.tsx index f02d1777f7d9..586d18b08c1f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedCard/FeedCardBody/FeedCardBody.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedCard/FeedCardBody/FeedCardBody.test.tsx @@ -37,7 +37,7 @@ jest.mock('../../../../utils/FeedUtils', () => ({ }, })); -jest.mock('../../../common/RichTextEditor/RichTextEditorPreviewer', () => { +jest.mock('../../../common/RichTextEditor/RichTextEditorPreviewerV1', () => { return jest.fn().mockReturnValue(

RichText Preview

); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedCard/FeedCardBody/FeedCardBody.tsx b/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedCard/FeedCardBody/FeedCardBody.tsx index 796470193d8e..3d8b1ca3c905 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedCard/FeedCardBody/FeedCardBody.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedCard/FeedCardBody/FeedCardBody.tsx @@ -21,7 +21,7 @@ import { getFrontEndFormat, MarkdownToHTMLConverter, } from '../../../../utils/FeedUtils'; -import RichTextEditorPreviewer from '../../../common/RichTextEditor/RichTextEditorPreviewer'; +import RichTextEditorPreviewerV1 from '../../../common/RichTextEditor/RichTextEditorPreviewerV1'; import ActivityFeedEditor from '../../ActivityFeedEditor/ActivityFeedEditor'; import Reactions from '../../Reactions/Reactions'; import { FeedBodyProp } from '../ActivityFeedCard.interface'; @@ -88,7 +88,7 @@ const FeedCardBody: FC = ({ onTextChange={handleMessageUpdate} /> ) : ( - @@ -114,7 +114,7 @@ const FeedCardBody: FC = ({ {postMessage} - diff --git a/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedCard/FeedCardBody/FeedCardBodyV1.tsx b/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedCard/FeedCardBody/FeedCardBodyV1.tsx index 499cc9830ca2..843f3c0c569f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedCard/FeedCardBody/FeedCardBodyV1.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedCard/FeedCardBody/FeedCardBodyV1.tsx @@ -18,7 +18,6 @@ import React, { useCallback, useMemo, useState } from 'react'; import { useTranslation } from 'react-i18next'; import { Link } from 'react-router-dom'; import ActivityFeedEditor from '../../../../components/ActivityFeed/ActivityFeedEditor/ActivityFeedEditor'; -import RichTextEditorPreviewer from '../../../../components/common/RichTextEditor/RichTextEditorPreviewer'; import { ASSET_CARD_STYLES } from '../../../../constants/Feeds.constants'; import { EntityType } from '../../../../enums/entity.enum'; import { CardStyle } from '../../../../generated/entity/feed/thread'; @@ -34,6 +33,7 @@ import { getFrontEndFormat, MarkdownToHTMLConverter, } from '../../../../utils/FeedUtils'; +import RichTextEditorPreviewerV1 from '../../../common/RichTextEditor/RichTextEditorPreviewerV1'; import ExploreSearchCard from '../../../ExploreV1/ExploreSearchCard/ExploreSearchCard'; import DescriptionFeed from '../../ActivityFeedCardV2/FeedCardBody/DescriptionFeed/DescriptionFeed'; import TagsFeed from '../../ActivityFeedCardV2/FeedCardBody/TagsFeed/TagsFeed'; @@ -83,7 +83,7 @@ const FeedCardBodyV1 = ({ if (ASSET_CARD_STYLES.includes(cardStyle as CardStyle)) { const entityInfo = feed.feedInfo?.entitySpecificInfo?.entity; const isExecutableTestSuite = - entityType === EntityType.TEST_SUITE && entityInfo.executable; + entityType === EntityType.TEST_SUITE && entityInfo.basic; const isObservabilityAlert = entityType === EntityType.EVENT_SUBSCRIPTION && (entityInfo as EventSubscription).alertType === @@ -118,7 +118,7 @@ const FeedCardBodyV1 = ({ } return ( - @@ -192,7 +192,7 @@ const FeedCardBodyV1 = ({ - diff --git a/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedCardV2/FeedCardBody/CustomPropertyFeed/CustomPropertyFeed.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedCardV2/FeedCardBody/CustomPropertyFeed/CustomPropertyFeed.component.tsx index bba62c4d7b94..a65cf2546161 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedCardV2/FeedCardBody/CustomPropertyFeed/CustomPropertyFeed.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedCardV2/FeedCardBody/CustomPropertyFeed/CustomPropertyFeed.component.tsx @@ -13,7 +13,7 @@ import React, { useMemo } from 'react'; import { getTextDiffCustomProperty } from '../../../../../utils/EntityVersionUtils'; -import RichTextEditorPreviewer from '../../../../common/RichTextEditor/RichTextEditorPreviewer'; +import RichTextEditorPreviewerV1 from '../../../../common/RichTextEditor/RichTextEditorPreviewerV1'; import { CustomPropertyFeedProps } from './CustomPropertyFeed.interface'; function CustomPropertyFeed({ feed }: Readonly) { @@ -28,7 +28,7 @@ function CustomPropertyFeed({ feed }: Readonly) { ); return ( - diff --git a/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedCardV2/FeedCardBody/DescriptionFeed/DescriptionFeed.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedCardV2/FeedCardBody/DescriptionFeed/DescriptionFeed.test.tsx index 8bebd16ed3cb..d860f59b5aa1 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedCardV2/FeedCardBody/DescriptionFeed/DescriptionFeed.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedCardV2/FeedCardBody/DescriptionFeed/DescriptionFeed.test.tsx @@ -20,7 +20,7 @@ import { } from '../../../../../mocks/DescriptionFeed.mock'; import DescriptionFeed from './DescriptionFeed'; -jest.mock('../../../../common/RichTextEditor/RichTextEditorPreviewer', () => { +jest.mock('../../../../common/RichTextEditor/RichTextEditorPreviewerV1', () => { return jest.fn().mockReturnValue(

RichTextEditorPreviewer

); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedCardV2/FeedCardBody/DescriptionFeed/DescriptionFeed.tsx b/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedCardV2/FeedCardBody/DescriptionFeed/DescriptionFeed.tsx index f6f72abbc1e7..16a004167aaf 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedCardV2/FeedCardBody/DescriptionFeed/DescriptionFeed.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedCardV2/FeedCardBody/DescriptionFeed/DescriptionFeed.tsx @@ -19,7 +19,7 @@ import { getFieldOperationIcon, getFrontEndFormat, } from '../../../../../utils/FeedUtils'; -import RichTextEditorPreviewer from '../../../../common/RichTextEditor/RichTextEditorPreviewer'; +import RichTextEditorPreviewerV1 from '../../../../common/RichTextEditor/RichTextEditorPreviewerV1'; import { DescriptionFeedProps } from './DescriptionFeed.interface'; function DescriptionFeed({ feed }: Readonly) { @@ -45,7 +45,7 @@ function DescriptionFeed({ feed }: Readonly) { {operationIcon} - diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AppBar/Appbar.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AppBar/Appbar.tsx index cd95f7c754ba..fae6f2ad736e 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AppBar/Appbar.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AppBar/Appbar.tsx @@ -26,6 +26,7 @@ import { isTourRoute, } from '../../utils/AuthProvider.util'; import { addToRecentSearched } from '../../utils/CommonUtils'; +import { getOidcToken } from '../../utils/LocalStorageUtils'; import searchClassBase from '../../utils/SearchClassBase'; import NavBar from '../NavBar/NavBar'; import './app-bar.style.less'; @@ -37,7 +38,7 @@ const Appbar: React.FC = (): JSX.Element => { const { isTourOpen, updateTourPage, updateTourSearch, tourSearchValue } = useTourProvider(); - const { isAuthenticated, searchCriteria, getOidcToken, trySilentSignIn } = + const { isAuthenticated, searchCriteria, trySilentSignIn } = useApplicationStore(); const parsedQueryString = Qs.parse( @@ -75,7 +76,7 @@ const Appbar: React.FC = (): JSX.Element => { getExplorePath({ tab: defaultTab, search: value, - isPersistFilters: false, + isPersistFilters: true, extraParameters: { sort: '_score', }, diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AppBar/Suggestions.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AppBar/Suggestions.tsx index d3d15043eb5e..81ac9ef0cf12 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AppBar/Suggestions.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AppBar/Suggestions.tsx @@ -13,8 +13,15 @@ import { Typography } from 'antd'; import { AxiosError } from 'axios'; -import { isEmpty } from 'lodash'; -import React, { useCallback, useEffect, useRef, useState } from 'react'; +import { isEmpty, isString } from 'lodash'; +import Qs from 'qs'; +import React, { + useCallback, + useEffect, + useMemo, + useRef, + useState, +} from 'react'; import { useTranslation } from 'react-i18next'; import { PAGE_SIZE_BASE } from '../../constants/constants'; import { @@ -43,7 +50,7 @@ import { DashboardDataModelSearchSource, StoredProcedureSearchSource, } from '../../interface/search.interface'; -import { searchData } from '../../rest/miscAPI'; +import { searchQuery } from '../../rest/searchAPI'; import { Transi18next } from '../../utils/CommonUtils'; import searchClassBase from '../../utils/SearchClassBase'; import { @@ -171,6 +178,18 @@ const Suggestions = ({ ); }; + const quickFilter = useMemo(() => { + const parsedSearch = Qs.parse( + location.search.startsWith('?') + ? location.search.substring(1) + : location.search + ); + + return !isString(parsedSearch.quickFilter) + ? {} + : JSON.parse(parsedSearch.quickFilter); + }, [location.search]); + const getSuggestionsForIndex = ( suggestions: SearchSuggestions, searchIndex: SearchIndex @@ -264,23 +283,16 @@ const Suggestions = ({ const fetchSearchData = useCallback(async () => { try { setIsLoading(true); - const res = await searchData( - searchText, - 1, - PAGE_SIZE_BASE, - '', - '', - '', - searchCriteria ?? SearchIndex.DATA_ASSET, - false, - false, - false - ); - if (res.data) { - setOptions(res.data.hits.hits as unknown as Option[]); - updateSuggestions(res.data.hits.hits as unknown as Option[]); - } + const res = await searchQuery({ + query: searchText, + searchIndex: searchCriteria ?? SearchIndex.DATA_ASSET, + queryFilter: quickFilter, + pageSize: PAGE_SIZE_BASE, + }); + + setOptions(res.hits.hits as unknown as Option[]); + updateSuggestions(res.hits.hits as unknown as Option[]); } catch (err) { showErrorToast( err as AxiosError, diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AppRouter/SettingsRouter.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AppRouter/SettingsRouter.test.tsx index cb47cdff5486..50caeeb07e86 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AppRouter/SettingsRouter.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AppRouter/SettingsRouter.test.tsx @@ -412,7 +412,7 @@ describe('SettingsRouter', () => { it('should render PersonaPage component for persona list route', async () => { render( - + ); @@ -422,7 +422,7 @@ describe('SettingsRouter', () => { it('should render PersonaDetailsPage component for persona details route', async () => { render( - + ); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AppRouter/SettingsRouter.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AppRouter/SettingsRouter.tsx index 3a235ac62067..4e16ace5458e 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AppRouter/SettingsRouter.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AppRouter/SettingsRouter.tsx @@ -172,12 +172,6 @@ const SettingsRouter = () => { {/* Setting Page Routes with categories */} - - { )}> + {/* Roles route start * Do not change the order of these route diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AppTour/Tour.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AppTour/Tour.tsx index 27239dfbc370..d062a08981c4 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AppTour/Tour.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AppTour/Tour.tsx @@ -67,7 +67,7 @@ const Tour = ({ steps }: { steps: TourSteps[] }) => { } maskColor="#302E36" playTour={isTourOpen} - stepWaitTimer={300} + stepWaitTimer={900} steps={steps} onRequestClose={handleRequestClose} onRequestSkip={handleModalSubmit} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/Auth0Authenticator.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/Auth0Authenticator.tsx index d7a876634081..44d14efad402 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/Auth0Authenticator.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/Auth0Authenticator.tsx @@ -22,6 +22,7 @@ import { useTranslation } from 'react-i18next'; import { AuthProvider } from '../../../generated/settings/settings'; import { useApplicationStore } from '../../../hooks/useApplicationStore'; +import { setOidcToken } from '../../../utils/LocalStorageUtils'; import { AuthenticatorRef } from '../AuthProviders/AuthProvider.interface'; interface Props { @@ -31,8 +32,7 @@ interface Props { const Auth0Authenticator = forwardRef( ({ children, onLogoutSuccess }: Props, ref) => { - const { setIsAuthenticated, authConfig, setOidcToken } = - useApplicationStore(); + const { setIsAuthenticated, authConfig } = useApplicationStore(); const { t } = useTranslation(); const { loginWithRedirect, getAccessTokenSilently, getIdTokenClaims } = useAuth0(); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/BasicAuthAuthenticator.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/BasicAuthAuthenticator.tsx index 2faba2a762ef..0f2583f8b224 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/BasicAuthAuthenticator.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/BasicAuthAuthenticator.tsx @@ -26,6 +26,11 @@ import { } from '../../../rest/auth-API'; import { useApplicationStore } from '../../../hooks/useApplicationStore'; +import { + getRefreshToken, + setOidcToken, + setRefreshToken, +} from '../../../utils/LocalStorageUtils'; import Loader from '../../common/Loader/Loader'; import { useBasicAuth } from '../AuthProviders/BasicAuthProvider'; @@ -40,9 +45,7 @@ const BasicAuthenticator = forwardRef( const { setIsAuthenticated, authConfig, - getRefreshToken, - setRefreshToken, - setOidcToken, + isApplicationLoading, } = useApplicationStore(); @@ -54,7 +57,13 @@ const BasicAuthenticator = forwardRef( authConfig?.provider !== AuthProvider.Basic && authConfig?.provider !== AuthProvider.LDAP ) { - Promise.reject(t('message.authProvider-is-not-basic')); + return Promise.reject( + new Error(t('message.authProvider-is-not-basic')) + ); + } + + if (!refreshToken) { + return Promise.reject(new Error(t('message.no-token-available'))); } const response = await getAccessTokenOnExpiry({ diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/GenericAuthenticator.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/GenericAuthenticator.tsx index f70c1757f732..f732fa3c2d30 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/GenericAuthenticator.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/GenericAuthenticator.tsx @@ -20,15 +20,11 @@ import { useHistory } from 'react-router-dom'; import { ROUTES } from '../../../constants/constants'; import { useApplicationStore } from '../../../hooks/useApplicationStore'; import { logoutUser, renewToken } from '../../../rest/LoginAPI'; +import { setOidcToken } from '../../../utils/LocalStorageUtils'; export const GenericAuthenticator = forwardRef( ({ children }: { children: ReactNode }, ref) => { - const { - setIsAuthenticated, - setIsSigningUp, - removeOidcToken, - setOidcToken, - } = useApplicationStore(); + const { setIsAuthenticated, setIsSigningUp } = useApplicationStore(); const history = useHistory(); const handleLogin = () => { @@ -42,7 +38,7 @@ export const GenericAuthenticator = forwardRef( await logoutUser(); history.push(ROUTES.SIGNIN); - removeOidcToken(); + setOidcToken(''); setIsAuthenticated(false); }; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/MsalAuthenticator.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/MsalAuthenticator.tsx index dfc9efb7579e..b6722bb5a372 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/MsalAuthenticator.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/MsalAuthenticator.tsx @@ -124,7 +124,7 @@ const MsalAuthenticator = forwardRef( }; const renewIdToken = async () => { - const user = await fetchIdToken(true); + const user = await fetchIdToken(); return user.id_token; }; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/OidcAuthenticator.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/OidcAuthenticator.tsx index 290e89cdd2c1..74eb1f3597a5 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/OidcAuthenticator.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/OidcAuthenticator.tsx @@ -27,6 +27,8 @@ import { ROUTES } from '../../../constants/constants'; import { useApplicationStore } from '../../../hooks/useApplicationStore'; import useCustomLocation from '../../../hooks/useCustomLocation/useCustomLocation'; import SignInPage from '../../../pages/LoginPage/SignInPage'; +import TokenService from '../../../utils/Auth/TokenService/TokenServiceUtil'; +import { setOidcToken } from '../../../utils/LocalStorageUtils'; import { showErrorToast } from '../../../utils/ToastUtils'; import Loader from '../../common/Loader/Loader'; import { @@ -71,7 +73,6 @@ const OidcAuthenticator = forwardRef( updateAxiosInterceptors, currentUser, newUser, - setOidcToken, isApplicationLoading, } = useApplicationStore(); const history = useHistory(); @@ -105,6 +106,9 @@ const OidcAuthenticator = forwardRef( // On success update token in store and update axios interceptors setOidcToken(user.id_token); updateAxiosInterceptors(); + // Clear the refresh token in progress flag + // Since refresh token request completes with a callback + TokenService.getInstance().clearRefreshInProgress(); }; const handleSilentSignInFailure = (error: unknown) => { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/OktaAuthenticator.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/OktaAuthenticator.tsx index 759677906a84..8ada4d6e1275 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/OktaAuthenticator.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/OktaAuthenticator.tsx @@ -20,6 +20,7 @@ import React, { } from 'react'; import { useApplicationStore } from '../../../hooks/useApplicationStore'; +import { setOidcToken } from '../../../utils/LocalStorageUtils'; import { AuthenticatorRef } from '../AuthProviders/AuthProvider.interface'; interface Props { @@ -30,7 +31,7 @@ interface Props { const OktaAuthenticator = forwardRef( ({ children, onLogoutSuccess }: Props, ref) => { const { oktaAuth } = useOktaAuth(); - const { setIsAuthenticated, setOidcToken } = useApplicationStore(); + const { setIsAuthenticated } = useApplicationStore(); const login = async () => { oktaAuth.signInWithRedirect(); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/SamlAuthenticator.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/SamlAuthenticator.tsx index 37a1ffd4dc59..7aa037356a8e 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/SamlAuthenticator.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/SamlAuthenticator.tsx @@ -36,6 +36,12 @@ import { showErrorToast } from '../../../utils/ToastUtils'; import { ROUTES } from '../../../constants/constants'; import { useApplicationStore } from '../../../hooks/useApplicationStore'; import { AccessTokenResponse, refreshSAMLToken } from '../../../rest/auth-API'; +import { + getOidcToken, + getRefreshToken, + setOidcToken, + setRefreshToken, +} from '../../../utils/LocalStorageUtils'; import { AuthenticatorRef } from '../AuthProviders/AuthProvider.interface'; interface Props { @@ -45,14 +51,7 @@ interface Props { const SamlAuthenticator = forwardRef( ({ children, onLogoutSuccess }: Props, ref) => { - const { - setIsAuthenticated, - authConfig, - getOidcToken, - getRefreshToken, - setRefreshToken, - setOidcToken, - } = useApplicationStore(); + const { setIsAuthenticated, authConfig } = useApplicationStore(); const config = authConfig?.samlConfiguration as SamlSSOClientConfig; const handleSilentSignIn = async (): Promise => { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppCallbacks/Auth0Callback/Auth0Callback.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppCallbacks/Auth0Callback/Auth0Callback.test.tsx index 935c9f1a50d1..a95a10a38434 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppCallbacks/Auth0Callback/Auth0Callback.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppCallbacks/Auth0Callback/Auth0Callback.test.tsx @@ -53,11 +53,14 @@ jest.mock('../../../../hooks/useApplicationStore', () => { useApplicationStore: jest.fn(() => ({ authConfig: {}, handleSuccessfulLogin: mockHandleSuccessfulLogin, - setOidcToken: jest.fn(), })), }; }); +jest.mock('../../../../utils/LocalStorageUtils', () => ({ + setOidcToken: jest.fn(), +})); + describe('Test Auth0Callback component', () => { afterEach(() => { jest.clearAllMocks(); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppCallbacks/Auth0Callback/Auth0Callback.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppCallbacks/Auth0Callback/Auth0Callback.tsx index 5bbff633223e..d5752ef8ed7e 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppCallbacks/Auth0Callback/Auth0Callback.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppCallbacks/Auth0Callback/Auth0Callback.tsx @@ -16,11 +16,12 @@ import { t } from 'i18next'; import React, { VFC } from 'react'; import { useApplicationStore } from '../../../../hooks/useApplicationStore'; +import { setOidcToken } from '../../../../utils/LocalStorageUtils'; import { OidcUser } from '../../AuthProviders/AuthProvider.interface'; const Auth0Callback: VFC = () => { const { isAuthenticated, user, getIdTokenClaims, error } = useAuth0(); - const { handleSuccessfulLogin, setOidcToken } = useApplicationStore(); + const { handleSuccessfulLogin } = useApplicationStore(); if (isAuthenticated) { getIdTokenClaims() .then((token) => { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AuthProviders/AuthProvider.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AuthProviders/AuthProvider.tsx index 207ac9c71ae2..67f2c38587f6 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AuthProviders/AuthProvider.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AuthProviders/AuthProvider.tsx @@ -25,7 +25,7 @@ import { InternalAxiosRequestConfig, } from 'axios'; import { CookieStorage } from 'cookie-storage'; -import { debounce, isEmpty, isNil, isNumber } from 'lodash'; +import { isEmpty, isNil, isNumber } from 'lodash'; import Qs from 'qs'; import React, { ComponentType, @@ -72,7 +72,12 @@ import { isProtectedRoute, prepareUserProfileFromClaims, } from '../../../utils/AuthProvider.util'; -import { getOidcToken } from '../../../utils/LocalStorageUtils'; +import { + getOidcToken, + getRefreshToken, + setOidcToken, + setRefreshToken, +} from '../../../utils/LocalStorageUtils'; import { getPathNameFromWindowLocation } from '../../../utils/RouterUtils'; import { escapeESReservedCharacters } from '../../../utils/StringsUtils'; import { showErrorToast, showInfoToast } from '../../../utils/ToastUtils'; @@ -110,7 +115,9 @@ const isEmailVerifyField = 'isEmailVerified'; let requestInterceptor: number | null = null; let responseInterceptor: number | null = null; -let failedLoggedInUserRequest: boolean | null; + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +let pendingRequests: any[] = []; export const AuthProvider = ({ childComponentType, @@ -130,14 +137,11 @@ export const AuthProvider = ({ jwtPrincipalClaimsMapping, setJwtPrincipalClaims, setJwtPrincipalClaimsMapping, - removeRefreshToken, - removeOidcToken, - getRefreshToken, isApplicationLoading, setApplicationLoading, } = useApplicationStore(); const { updateDomains, updateDomainLoading } = useDomainStore(); - const tokenService = useRef(); + const tokenService = useRef(TokenService.getInstance()); const location = useCustomLocation(); const history = useHistory(); @@ -176,7 +180,7 @@ export const AuthProvider = ({ removeSession(); // remove the refresh token on logout - removeRefreshToken(); + setRefreshToken(''); setApplicationLoading(false); @@ -184,14 +188,6 @@ export const AuthProvider = ({ history.push(ROUTES.SIGNIN); }, [timeoutId]); - useEffect(() => { - if (authenticatorRef.current?.renewIdToken) { - tokenService.current = new TokenService( - authenticatorRef.current?.renewIdToken - ); - } - }, [authenticatorRef.current?.renewIdToken]); - const fetchDomainList = useCallback(async () => { try { updateDomainLoading(true); @@ -228,7 +224,7 @@ export const AuthProvider = ({ const resetUserDetails = (forceLogout = false) => { setCurrentUser({} as User); - removeOidcToken(); + setOidcToken(''); setIsAuthenticated(false); setApplicationLoading(false); clearTimeout(timeoutId); @@ -268,39 +264,6 @@ export const AuthProvider = ({ } }; - /** - * This method will try to signIn silently when token is about to expire - * if it's not succeed then it will proceed for logout - */ - const trySilentSignIn = async (forceLogout?: boolean) => { - const pathName = getPathNameFromWindowLocation(); - // Do not try silent sign in for SignIn or SignUp route - if ( - [ROUTES.SIGNIN, ROUTES.SIGNUP, ROUTES.SILENT_CALLBACK].includes(pathName) - ) { - return; - } - - if (!tokenService.current?.isTokenUpdateInProgress()) { - // For OIDC we won't be getting newToken immediately hence not updating token here - const newToken = await tokenService.current?.refreshToken(); - // Start expiry timer on successful silent signIn - if (newToken) { - // Start expiry timer on successful silent signIn - // eslint-disable-next-line @typescript-eslint/no-use-before-define - startTokenExpiryTimer(); - - // Retry the failed request after successful silent signIn - if (failedLoggedInUserRequest) { - await getLoggedInUserDetails(); - failedLoggedInUserRequest = null; - } - } else if (forceLogout) { - resetUserDetails(true); - } - } - }; - /** * It will set an timer for 5 mins before Token will expire * If time if less then 5 mins then it will try to SilentSignIn @@ -326,13 +289,25 @@ export const AuthProvider = ({ // If token is about to expire then start silentSignIn // else just set timer to try for silentSignIn before token expires clearTimeout(timeoutId); - const timerId = setTimeout(() => { - trySilentSignIn(); - }, timeoutExpiry); + + const timerId = setTimeout( + tokenService.current?.refreshToken, + timeoutExpiry + ); setTimeoutId(Number(timerId)); } }; + useEffect(() => { + if (authenticatorRef.current?.renewIdToken) { + tokenService.current.updateRenewToken( + authenticatorRef.current?.renewIdToken + ); + // After every refresh success, start timer again + tokenService.current.updateRefreshSuccessCallback(startTokenExpiryTimer); + } + }, [authenticatorRef.current?.renewIdToken]); + /** * Performs cleanup around timers * Clean silentSignIn activities if going on @@ -542,13 +517,52 @@ export const AuthProvider = ({ if (error.response) { const { status } = error.response; if (status === ClientErrors.UNAUTHORIZED) { - // store the failed request for retry after successful silent signIn - if (error.config.url === '/users/loggedInUser') { - failedLoggedInUserRequest = true; + if (error.config.url === '/users/refresh') { + return Promise.reject(error as Error); } handleStoreProtectedRedirectPath(); - // try silent signIn if token is about to expire - debounce(() => trySilentSignIn(true), 100); + + // If 401 error and refresh is not in progress, trigger the refresh + if (!tokenService.current?.isTokenUpdateInProgress()) { + // Start the refresh process + return new Promise((resolve, reject) => { + // Add this request to the pending queue + pendingRequests.push({ + resolve, + reject, + config: error.config, + }); + + // Refresh the token and retry the requests in the queue + tokenService.current.refreshToken().then((token) => { + if (token) { + // Retry the pending requests + initializeAxiosInterceptors(); + pendingRequests.forEach(({ resolve, reject, config }) => { + axiosClient(config).then(resolve).catch(reject); + }); + + // Clear the queue after retrying + pendingRequests = []; + } else { + resetUserDetails(true); + } + }); + }).catch((err) => { + resetUserDetails(true); + + return Promise.reject(err); + }); + } else { + // If refresh is in progress, queue the request + return new Promise((resolve, reject) => { + pendingRequests.push({ + resolve, + reject, + config: error.config, + }); + }); + } } } @@ -721,7 +735,6 @@ export const AuthProvider = ({ onLoginHandler, onLogoutHandler, handleSuccessfulLogin, - trySilentSignIn, handleFailedLogin, updateAxiosInterceptors: initializeAxiosInterceptors, }); @@ -734,7 +747,6 @@ export const AuthProvider = ({ onLoginHandler, onLogoutHandler, handleSuccessfulLogin, - trySilentSignIn, handleFailedLogin, updateAxiosInterceptors: initializeAxiosInterceptors, }); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AuthProviders/BasicAuthProvider.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AuthProviders/BasicAuthProvider.tsx index 649b37fc2697..11b972671450 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AuthProviders/BasicAuthProvider.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AuthProviders/BasicAuthProvider.tsx @@ -40,7 +40,13 @@ import { import { resetWebAnalyticSession } from '../../../utils/WebAnalyticsUtils'; import { toLower } from 'lodash'; -import { useApplicationStore } from '../../../hooks/useApplicationStore'; +import { extractDetailsFromToken } from '../../../utils/AuthProvider.util'; +import { + getOidcToken, + getRefreshToken, + setOidcToken, + setRefreshToken, +} from '../../../utils/LocalStorageUtils'; import { OidcUser } from './AuthProvider.interface'; export interface BasicAuthJWTPayload extends JwtPayload { @@ -90,13 +96,7 @@ const BasicAuthProvider = ({ onLoginFailure, }: BasicAuthProps) => { const { t } = useTranslation(); - const { - setRefreshToken, - setOidcToken, - getOidcToken, - removeOidcToken, - getRefreshToken, - } = useApplicationStore(); + const [loginError, setLoginError] = useState(null); const history = useHistory(); @@ -169,18 +169,7 @@ const BasicAuthProvider = ({ }; const handleForgotPassword = async (email: string) => { - try { - await generatePasswordResetLink(email); - } catch (err) { - if ( - (err as AxiosError).response?.status === - HTTP_STATUS_CODE.FAILED_DEPENDENCY - ) { - showErrorToast(t('server.forgot-password-email-error')); - } else { - showErrorToast(t('server.email-not-found')); - } - } + await generatePasswordResetLink(email); }; const handleResetPassword = async (payload: PasswordResetRequest) => { @@ -193,10 +182,11 @@ const BasicAuthProvider = ({ const handleLogout = async () => { const token = getOidcToken(); const refreshToken = getRefreshToken(); - if (token) { + const isExpired = extractDetailsFromToken(token).isExpired; + if (token && !isExpired) { try { await logoutUser({ token, refreshToken }); - removeOidcToken(); + setOidcToken(''); history.push(ROUTES.SIGNIN); } catch (error) { showErrorToast(error as AxiosError); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AuthProviders/OktaAuthProvider.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AuthProviders/OktaAuthProvider.tsx index c90e9b4a634c..1944d170d1df 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AuthProviders/OktaAuthProvider.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AuthProviders/OktaAuthProvider.tsx @@ -20,6 +20,7 @@ import React, { useMemo, } from 'react'; import { useApplicationStore } from '../../../hooks/useApplicationStore'; +import { setOidcToken } from '../../../utils/LocalStorageUtils'; import { OidcUser } from './AuthProvider.interface'; interface Props { @@ -31,7 +32,7 @@ export const OktaAuthProvider: FunctionComponent = ({ children, onLoginSuccess, }: Props) => { - const { authConfig, setOidcToken } = useApplicationStore(); + const { authConfig } = useApplicationStore(); const { clientId, issuer, redirectUri, scopes, pkce } = authConfig as OktaAuthOptions; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/BlockEditor/BarMenu/BarMenu.tsx b/openmetadata-ui/src/main/resources/ui/src/components/BlockEditor/BarMenu/BarMenu.tsx new file mode 100644 index 000000000000..8c6ff087e2da --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/BlockEditor/BarMenu/BarMenu.tsx @@ -0,0 +1,146 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import classNames from 'classnames'; +import { uniqueId } from 'lodash'; +import React, { FC, Fragment } from 'react'; +import BlockQuoteIcon from '../../../assets/svg/ic-format-block-quote.svg'; +import BoldIcon from '../../../assets/svg/ic-format-bold.svg'; +import UnorderedListIcon from '../../../assets/svg/ic-format-bullet-list.svg'; +import CodeBlockIcon from '../../../assets/svg/ic-format-code-block.svg'; +import HorizontalLineIcon from '../../../assets/svg/ic-format-horizontal-line.svg'; +import ImageIcon from '../../../assets/svg/ic-format-image-inline.svg'; +import InlineCodeIcon from '../../../assets/svg/ic-format-inline-code.svg'; +import ItalicIcon from '../../../assets/svg/ic-format-italic.svg'; +import LinkIcon from '../../../assets/svg/ic-format-link.svg'; +import OrderedListIcon from '../../../assets/svg/ic-format-numbered-list.svg'; +import StrikeIcon from '../../../assets/svg/ic-format-strike.svg'; +import { BarMenuProps } from '../BlockEditor.interface'; +import './bar-menu.less'; + +const BarMenu: FC = ({ editor, onLinkToggle }) => { + const formats = [ + [ + { + name: 'bold', + icon: BoldIcon, + command: () => editor.chain().focus().toggleBold().run(), + isActive: () => editor.isActive('bold'), + }, + { + name: 'italic', + icon: ItalicIcon, + command: () => editor.chain().focus().toggleItalic().run(), + isActive: () => editor.isActive('italic'), + }, + { + name: 'strike', + icon: StrikeIcon, + command: () => editor.chain().focus().toggleStrike().run(), + isActive: () => editor.isActive('strike'), + }, + ], + [ + { + name: 'inline-code', + icon: InlineCodeIcon, + command: () => editor.chain().focus().toggleCode().run(), + isActive: () => editor.isActive('code'), + }, + ], + [ + { + name: 'unordered-list', + icon: UnorderedListIcon, + command: () => editor.chain().focus().toggleBulletList().run(), + isActive: () => editor.isActive('bulletList'), + }, + { + name: 'ordered-list', + icon: OrderedListIcon, + command: () => editor.chain().focus().toggleOrderedList().run(), + isActive: () => editor.isActive('orderedList'), + }, + ], + [ + { + name: 'link', + icon: LinkIcon, + command: () => { + editor.chain().focus().setLink({ href: '' }).run(); + onLinkToggle?.(); + }, + isActive: () => editor.isActive('link'), + }, + { + name: 'image', + icon: ImageIcon, + command: () => editor.chain().focus().setImage({ src: '' }).run(), + isActive: () => editor.isActive('image'), + }, + { + name: 'code-block', + icon: CodeBlockIcon, + command: () => editor.chain().focus().toggleCodeBlock().run(), + isActive: () => editor.isActive('codeBlock'), + }, + { + name: 'block-quote', + icon: BlockQuoteIcon, + command: () => editor.chain().focus().toggleBlockquote().run(), + isActive: () => editor.isActive('blockquote'), + }, + { + name: 'horizontal-line', + icon: HorizontalLineIcon, + command: () => editor.chain().focus().setHorizontalRule().run(), + isActive: () => false, + }, + ], + ]; + + return ( +
+ {formats.map((format, index) => { + return ( + +
+ {format.map((item) => { + return ( + + ); + })} +
+ {index !== formats.length - 1 && ( +
+ )} + + ); + })} +
+ ); +}; + +export default BarMenu; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/BlockEditor/BarMenu/bar-menu.less b/openmetadata-ui/src/main/resources/ui/src/components/BlockEditor/BarMenu/bar-menu.less new file mode 100644 index 000000000000..779e072c7014 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/BlockEditor/BarMenu/bar-menu.less @@ -0,0 +1,58 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@button-bg-color-hover: #ecedee; +@button-bg-color: #e8e8e9; +@format-group-separator-color: #e5e7eb; +@menu-bg-color: #f7f9fc; + +.bar-menu-wrapper { + border-top-left-radius: 4px; + border-top-right-radius: 4px; + background-color: @menu-bg-color; + display: flex; + flex-wrap: wrap; + flex-direction: row; + padding: 8px; + border-bottom: 1px solid @format-group-separator-color; + gap: 1rem; + + .bar-menu-wrapper--format-group { + display: flex; + flex-direction: row; + gap: 0.5rem; + + .bar-menu-wrapper--format-group--button { + background-color: transparent; + border: none; + border-radius: 0.375rem; + cursor: pointer; + + &:hover { + background-color: @button-bg-color-hover; + } + + &.active { + background-color: @button-bg-color; + } + .bar-menu-wrapper--format--button--icon { + width: 28px; + height: 28px; + } + } + } + + .bar-menu-wrapper--format-group--separator { + border-left: 1px solid @format-group-separator-color; + } +} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/BlockEditor/BlockEditor.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/BlockEditor/BlockEditor.interface.ts index dba44a126c7c..2ebf3a73696f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/BlockEditor/BlockEditor.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/BlockEditor/BlockEditor.interface.ts @@ -10,8 +10,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import { Editor } from '@tiptap/react'; import { SuggestionKeyDownProps } from '@tiptap/suggestion'; +export type MenuType = 'bubble' | 'bar'; + export interface SuggestionItem { id: string; name: string; @@ -27,4 +30,27 @@ export interface ExtensionRef { export interface EditorSlotsRef { onMouseDown: (e: React.MouseEvent) => void; + onLinkToggle: () => void; +} + +export interface BlockEditorRef { + editor: Editor | null; +} +export interface BlockEditorProps { + content?: string; + editable?: boolean; + onChange?: (htmlContent: string) => void; + menuType?: MenuType; + autoFocus?: boolean; + placeholder?: string; +} + +export interface EditorSlotsProps { + editor: Editor | null; + menuType: MenuType; +} + +export interface BarMenuProps { + editor: Editor; + onLinkToggle?: () => void; } diff --git a/openmetadata-ui/src/main/resources/ui/src/components/BlockEditor/BlockEditor.tsx b/openmetadata-ui/src/main/resources/ui/src/components/BlockEditor/BlockEditor.tsx index 5c45c975c317..3ae7f649474e 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/BlockEditor/BlockEditor.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/BlockEditor/BlockEditor.tsx @@ -10,7 +10,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Editor, EditorContent } from '@tiptap/react'; +import { EditorContent } from '@tiptap/react'; +import classNames from 'classnames'; import { isNil } from 'lodash'; import React, { forwardRef, @@ -21,26 +22,33 @@ import React, { import { useTranslation } from 'react-i18next'; import { EDITOR_OPTIONS } from '../../constants/BlockEditor.constants'; import { formatContent, setEditorContent } from '../../utils/BlockEditorUtils'; +import BarMenu from './BarMenu/BarMenu'; import './block-editor.less'; -import { EditorSlotsRef } from './BlockEditor.interface'; +import { + BlockEditorProps, + BlockEditorRef, + EditorSlotsRef, +} from './BlockEditor.interface'; import EditorSlots from './EditorSlots'; import { extensions } from './Extensions'; import { useCustomEditor } from './hooks/useCustomEditor'; -export interface BlockEditorRef { - editor: Editor | null; -} -export interface BlockEditorProps { - content?: string; - editable?: boolean; - onChange?: (htmlContent: string) => void; -} - const BlockEditor = forwardRef( - ({ content = '', editable = true, onChange }, ref) => { + ( + { + content = '', + editable = true, + menuType = 'bubble', + autoFocus, + placeholder, + onChange, + }, + ref + ) => { const { i18n } = useTranslation(); const editorSlots = useRef(null); + // this hook to initialize the editor const editor = useCustomEditor({ ...EDITOR_OPTIONS, extensions, @@ -54,14 +62,18 @@ const BlockEditor = forwardRef( editorProps: { attributes: { class: 'om-block-editor', + ...(autoFocus ? { autofocus: 'true' } : {}), }, }, + autofocus: autoFocus, }); + // this hook to expose the editor instance useImperativeHandle(ref, () => ({ editor, })); + // this effect to handle the content change useEffect(() => { if (isNil(editor) || editor.isDestroyed || content === undefined) { return; @@ -77,6 +89,7 @@ const BlockEditor = forwardRef( }); }, [content, editor]); + // this effect to handle the editable state useEffect(() => { if ( isNil(editor) || @@ -91,6 +104,7 @@ const BlockEditor = forwardRef( setTimeout(() => editor.setEditable(editable)); }, [editable, editor]); + // this effect to handle the RTL and LTR direction useEffect(() => { const editorWrapper = document.getElementById('block-editor-wrapper'); if (!editorWrapper) { @@ -106,12 +120,24 @@ const BlockEditor = forwardRef( }, [i18n]); return ( -
+
+ {menuType === 'bar' && !isNil(editor) && ( + + )} - +
); } diff --git a/openmetadata-ui/src/main/resources/ui/src/components/BlockEditor/EditorSlots.tsx b/openmetadata-ui/src/main/resources/ui/src/components/BlockEditor/EditorSlots.tsx index e3f56d35d120..cb4a54a61528 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/BlockEditor/EditorSlots.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/BlockEditor/EditorSlots.tsx @@ -14,19 +14,15 @@ import { Editor, ReactRenderer } from '@tiptap/react'; import { isEmpty, isNil } from 'lodash'; import React, { forwardRef, useImperativeHandle, useState } from 'react'; import tippy, { Instance, Props } from 'tippy.js'; -import { EditorSlotsRef } from './BlockEditor.interface'; +import { EditorSlotsProps, EditorSlotsRef } from './BlockEditor.interface'; import BlockMenu from './BlockMenu/BlockMenu'; import BubbleMenu from './BubbleMenu/BubbleMenu'; import LinkModal, { LinkData } from './LinkModal/LinkModal'; import LinkPopup from './LinkPopup/LinkPopup'; import TableMenu from './TableMenu/TableMenu'; -interface EditorSlotsProps { - editor: Editor | null; -} - const EditorSlots = forwardRef( - ({ editor }, ref) => { + ({ editor, menuType }, ref) => { const [isLinkModalOpen, setIsLinkModalOpen] = useState(false); const handleLinkToggle = () => { @@ -142,12 +138,17 @@ const EditorSlots = forwardRef( } }; - const menus = !isNil(editor) && ( + /** + * render the bubble menu only if the editor is available + * and the menu type is bubble + */ + const menus = !isNil(editor) && menuType === 'bubble' && ( ); useImperativeHandle(ref, () => ({ onMouseDown: handleLinkPopup, + onLinkToggle: handleLinkToggle, })); if (isNil(editor)) { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/BlockEditor/Extensions/BlockAndDragDrop/BlockAndDragHandle.ts b/openmetadata-ui/src/main/resources/ui/src/components/BlockEditor/Extensions/BlockAndDragDrop/BlockAndDragHandle.ts index a807211858cc..57bcaee06f20 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/BlockEditor/Extensions/BlockAndDragDrop/BlockAndDragHandle.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/BlockEditor/Extensions/BlockAndDragDrop/BlockAndDragHandle.ts @@ -197,42 +197,63 @@ export const BlockAndDragHandle = (options: BlockAndDragHandleOptions) => { return new Plugin({ view: (view) => { - // drag handle initialization - dragHandleElement = document.createElement('div'); - dragHandleElement.draggable = true; - dragHandleElement.dataset.dragHandle = ''; - dragHandleElement.title = 'Drag to move\nClick to open menu'; - dragHandleElement.classList.add('om-drag-handle'); - dragHandleElement.addEventListener('dragstart', (e) => { - handleDragStart(e, view); - }); - dragHandleElement.addEventListener('click', (e) => { - handleDragClick(e, view); - }); - - hideDragHandle(); - - // block handle initialization - blockHandleElement = document.createElement('div'); - blockHandleElement.draggable = false; - blockHandleElement.dataset.blockHandle = ''; - blockHandleElement.title = 'Add new node'; - blockHandleElement.classList.add('om-block-handle'); - - hideBlockHandle(); - - view?.dom?.parentElement?.appendChild(dragHandleElement); - view?.dom?.parentElement?.appendChild(blockHandleElement); - - return { - destroy: () => { - dragHandleElement?.remove?.(); - dragHandleElement = null; - - blockHandleElement?.remove?.(); - blockHandleElement = null; - }, - }; + try { + const isBarMenu = + view.dom?.parentElement?.parentElement?.classList.contains( + 'block-editor-wrapper--bar-menu' + ); + + if (isBarMenu) { + return { + destroy: () => { + // do nothing + }, + }; + } + + // drag handle initialization + dragHandleElement = document.createElement('div'); + dragHandleElement.draggable = true; + dragHandleElement.dataset.dragHandle = ''; + dragHandleElement.title = 'Drag to move\nClick to open menu'; + dragHandleElement.classList.add('om-drag-handle'); + dragHandleElement.addEventListener('dragstart', (e) => { + handleDragStart(e, view); + }); + dragHandleElement.addEventListener('click', (e) => { + handleDragClick(e, view); + }); + + hideDragHandle(); + + // block handle initialization + blockHandleElement = document.createElement('div'); + blockHandleElement.draggable = false; + blockHandleElement.dataset.blockHandle = ''; + blockHandleElement.title = 'Add new node'; + blockHandleElement.classList.add('om-block-handle'); + + hideBlockHandle(); + + view?.dom?.parentElement?.appendChild(dragHandleElement); + view?.dom?.parentElement?.appendChild(blockHandleElement); + + return { + destroy: () => { + dragHandleElement?.remove?.(); + dragHandleElement = null; + + blockHandleElement?.remove?.(); + blockHandleElement = null; + }, + }; + } catch (error) { + return { + destroy: () => { + // do nothing + }, + }; + } }, props: { handleDOMEvents: { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/BlockEditor/Extensions/diff-view.ts b/openmetadata-ui/src/main/resources/ui/src/components/BlockEditor/Extensions/diff-view.ts index 0aa240200fcb..5533bc669308 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/BlockEditor/Extensions/diff-view.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/BlockEditor/Extensions/diff-view.ts @@ -10,7 +10,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { mergeAttributes, Node } from '@tiptap/core'; +import { Node } from '@tiptap/core'; export default Node.create({ name: 'diffView', @@ -23,18 +23,55 @@ export default Node.create({ class: { default: '', }, + 'data-testid': { + default: '', + parseHTML: (element) => element.getAttribute('data-testid'), + renderHTML: (attributes) => { + if (!attributes['data-testid']) { + return {}; + } + + return { + 'data-testid': attributes['data-testid'], + }; + }, + }, + 'data-diff': { + default: true, + parseHTML: (element) => element.getAttribute('data-diff'), + renderHTML: (attributes) => { + if (!attributes['data-diff']) { + return {}; + } + + return { + 'data-diff': attributes['data-diff'], + }; + }, + }, }; }, parseHTML() { return [ { - tag: 'diff-view', + tag: 'span[data-diff]', }, ]; }, - renderHTML({ HTMLAttributes }) { - return ['diff-view', mergeAttributes(HTMLAttributes), 0]; + renderHTML({ HTMLAttributes, node }) { + const diffNode = document.createElement('span'); + + Object.keys(HTMLAttributes).forEach((key) => { + diffNode.setAttribute(key, HTMLAttributes[key]); + }); + + diffNode.setAttribute('data-diff', 'true'); + diffNode.innerHTML = node.textContent; + + return { + dom: diffNode, + }; }, }); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/BlockEditor/Extensions/index.ts b/openmetadata-ui/src/main/resources/ui/src/components/BlockEditor/Extensions/index.ts index 74c796f93d62..008f1565a9a2 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/BlockEditor/Extensions/index.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/BlockEditor/Extensions/index.ts @@ -34,6 +34,7 @@ import { mentionSuggestion } from './mention/mentionSuggestions'; import slashCommand from './slash-command'; import { getSuggestionItems } from './slash-command/items'; import renderItems from './slash-command/renderItems'; +import TextHighlightView from './text-highlight-view'; import { TrailingNode } from './trailing-node'; export const extensions = [ @@ -112,6 +113,7 @@ export const extensions = [ suggestion: hashtagSuggestion(), }), DiffView, + TextHighlightView, Image.configure({ allowBase64: true, inline: true, diff --git a/openmetadata-ui/src/main/resources/ui/src/components/BlockEditor/Extensions/text-highlight-view.ts b/openmetadata-ui/src/main/resources/ui/src/components/BlockEditor/Extensions/text-highlight-view.ts new file mode 100644 index 000000000000..e66503922771 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/BlockEditor/Extensions/text-highlight-view.ts @@ -0,0 +1,77 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Node } from '@tiptap/core'; + +export default Node.create({ + name: 'textHighLightView', + content: 'inline*', + group: 'inline', + inline: true, + + addAttributes() { + return { + class: { + default: '', + }, + 'data-testid': { + default: '', + parseHTML: (element) => element.getAttribute('data-testid'), + renderHTML: (attributes) => { + if (!attributes['data-testid']) { + return {}; + } + + return { + 'data-testid': attributes['data-testid'], + }; + }, + }, + 'data-highlight': { + default: true, + parseHTML: (element) => element.getAttribute('data-highlight'), + renderHTML: (attributes) => { + if (!attributes['data-highlight']) { + return {}; + } + + return { + 'data-highlight': attributes['data-highlight'], + }; + }, + }, + }; + }, + + parseHTML() { + return [ + { + tag: 'span[data-highlight]', + }, + ]; + }, + + renderHTML({ HTMLAttributes, node }) { + const textHighlightNode = document.createElement('span'); + + Object.keys(HTMLAttributes).forEach((key) => { + textHighlightNode.setAttribute(key, HTMLAttributes[key]); + }); + + textHighlightNode.setAttribute('data-highlight', 'true'); + textHighlightNode.innerHTML = node.textContent; + + return { + dom: textHighlightNode, + }; + }, +}); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/BlockEditor/block-editor.less b/openmetadata-ui/src/main/resources/ui/src/components/BlockEditor/block-editor.less index b3853974b9f4..d9c0dc8b1252 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/BlockEditor/block-editor.less +++ b/openmetadata-ui/src/main/resources/ui/src/components/BlockEditor/block-editor.less @@ -203,6 +203,42 @@ .tippy-box { max-width: 400px !important; } + + &.block-editor-wrapper--bar-menu { + border: 1px solid #dadde6; + border-radius: 4px; + + // do not show drag handle and block handle in bar menu mode + .om-drag-handle, + .om-block-handle { + display: none; + } + .om-block-editor { + max-height: 300px; + overflow: auto; + padding-left: 24px; + padding-right: 24px; + padding-top: 8px; + &:focus-visible { + outline: none; + } + } + } + + &.block-editor-wrapper--bubble-menu { + .om-block-editor { + &:focus-visible { + outline: none; + } + } + } + + // if contenteditable is false, remove padding-bottom from last p tag + .om-block-editor[contenteditable='false'] { + > p:last-child { + padding-bottom: 0; + } + } } .menu-wrapper { @@ -388,7 +424,7 @@ width: 100%; } } -.om-image-node-embed-link-btn-col { +body .om-image-node-embed-link-btn-col { display: flex; justify-content: flex-end; } diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Classifications/ClassificationDetails/ClassificationDetails.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Classifications/ClassificationDetails/ClassificationDetails.tsx index f544520a7546..bb96aed3ec67 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Classifications/ClassificationDetails/ClassificationDetails.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Classifications/ClassificationDetails/ClassificationDetails.tsx @@ -65,6 +65,7 @@ import NextPrevious from '../../common/NextPrevious/NextPrevious'; import { NextPreviousProps } from '../../common/NextPrevious/NextPrevious.interface'; import Table from '../../common/Table/Table'; import EntityHeaderTitle from '../../Entity/EntityHeaderTitle/EntityHeaderTitle.component'; +import { EntityName } from '../../Modals/EntityNameModal/EntityNameModal.interface'; import { ClassificationDetailsProps } from './ClassificationDetails.interface'; const ClassificationDetails = forwardRef( @@ -192,10 +193,7 @@ const ClassificationDetails = forwardRef( [currentClassification?.disabled] ); - const handleUpdateDisplayName = async (data: { - name: string; - displayName?: string; - }) => { + const handleUpdateDisplayName = async (data: EntityName) => { if ( !isUndefined(currentClassification) && !isUndefined(handleUpdateClassification) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Container/ContainerChildren/ContainerChildren.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Container/ContainerChildren/ContainerChildren.test.tsx index d21604ae6abe..6a1914b02a77 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Container/ContainerChildren/ContainerChildren.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Container/ContainerChildren/ContainerChildren.test.tsx @@ -39,6 +39,15 @@ const mockDataProps = { }; describe('ContainerChildren', () => { + beforeEach(() => { + jest.useFakeTimers(); + }); + + afterEach(() => { + jest.runOnlyPendingTimers(); + jest.useRealTimers(); + }); + it('Should call fetch container function on load', () => { render( @@ -88,6 +97,9 @@ describe('ContainerChildren', () => { ); + // Fast-forward until all timers have been executed + jest.runAllTimers(); + const richTextPreviewers = screen.getAllByTestId('viewer-container'); expect(richTextPreviewers).toHaveLength(2); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Container/ContainerChildren/ContainerChildren.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Container/ContainerChildren/ContainerChildren.tsx index 405187c63ea9..30892f78a478 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Container/ContainerChildren/ContainerChildren.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Container/ContainerChildren/ContainerChildren.tsx @@ -21,7 +21,7 @@ import { Container } from '../../../generated/entity/data/container'; import { EntityReference } from '../../../generated/type/entityReference'; import { getColumnSorter, getEntityName } from '../../../utils/EntityUtils'; import ErrorPlaceHolder from '../../common/ErrorWithPlaceholder/ErrorPlaceHolder'; -import RichTextEditorPreviewer from '../../common/RichTextEditor/RichTextEditorPreviewer'; +import RichTextEditorPreviewerV1 from '../../common/RichTextEditor/RichTextEditorPreviewerV1'; import Table from '../../common/Table/Table'; interface ContainerChildrenProps { @@ -66,7 +66,7 @@ const ContainerChildren: FC = ({ render: (description: EntityReference['description']) => ( <> {description ? ( - + ) : ( {t('label.no-entity', { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Container/ContainerDataModel/ContainerDataModel.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Container/ContainerDataModel/ContainerDataModel.test.tsx index 6511b6fec06f..0c1372021156 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Container/ContainerDataModel/ContainerDataModel.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Container/ContainerDataModel/ContainerDataModel.test.tsx @@ -110,7 +110,7 @@ jest.mock('../../../utils/ContainerDetailUtils', () => ({ })); jest.mock( - '../../../components/common/RichTextEditor/RichTextEditorPreviewer', + '../../../components/common/RichTextEditor/RichTextEditorPreviewerV1', () => jest .fn() diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Dashboard/DashboardDetails/DashboardDetails.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Dashboard/DashboardDetails/DashboardDetails.test.tsx index 3b74b8e89cba..a87e3f79ea4a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Dashboard/DashboardDetails/DashboardDetails.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Dashboard/DashboardDetails/DashboardDetails.test.tsx @@ -99,7 +99,7 @@ jest.mock('../../common/TabsLabel/TabsLabel.component', () => { jest.mock('../../common/EntityDescription/DescriptionV1', () => { return jest.fn().mockReturnValue(

Description Component

); }); -jest.mock('../../common/RichTextEditor/RichTextEditorPreviewer', () => { +jest.mock('../../common/RichTextEditor/RichTextEditorPreviewerV1', () => { return jest.fn().mockReturnValue(

RichTextEditorPreviwer

); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Dashboard/DashboardVersion/DashboardVersion.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Dashboard/DashboardVersion/DashboardVersion.component.tsx index 395002a904bb..ef13f05e6b3c 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Dashboard/DashboardVersion/DashboardVersion.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Dashboard/DashboardVersion/DashboardVersion.component.tsx @@ -39,7 +39,7 @@ import { import { CustomPropertyTable } from '../../common/CustomPropertyTable/CustomPropertyTable'; import DescriptionV1 from '../../common/EntityDescription/DescriptionV1'; import Loader from '../../common/Loader/Loader'; -import RichTextEditorPreviewer from '../../common/RichTextEditor/RichTextEditorPreviewer'; +import RichTextEditorPreviewerV1 from '../../common/RichTextEditor/RichTextEditorPreviewerV1'; import TabsLabel from '../../common/TabsLabel/TabsLabel.component'; import DataAssetsVersionHeader from '../../DataAssets/DataAssetsVersionHeader/DataAssetsVersionHeader'; import DataProductsContainer from '../../DataProducts/DataProductsContainer/DataProductsContainer.component'; @@ -133,7 +133,7 @@ const DashboardVersion: FC = ({ key: 'description', render: (text) => text ? ( - + ) : ( {t('label.no-description')} ), diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Dashboard/DashboardVersion/DashboardVersion.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Dashboard/DashboardVersion/DashboardVersion.test.tsx index 2943e2cf4e75..19a9e725fe85 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Dashboard/DashboardVersion/DashboardVersion.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Dashboard/DashboardVersion/DashboardVersion.test.tsx @@ -26,7 +26,7 @@ import { DashboardVersionProp } from './DashboardVersion.interface'; const mockPush = jest.fn(); -jest.mock('../../common/RichTextEditor/RichTextEditorPreviewer', () => { +jest.mock('../../common/RichTextEditor/RichTextEditorPreviewerV1', () => { return jest .fn() .mockImplementation(() =>
RichTextEditorPreviewer.component
); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Dashboard/DataModel/DataModels/DataModelsTable.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Dashboard/DataModel/DataModels/DataModelsTable.tsx index 1ac09ad1089b..fd72c640b8e8 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Dashboard/DataModel/DataModels/DataModelsTable.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Dashboard/DataModel/DataModels/DataModelsTable.tsx @@ -36,7 +36,7 @@ import { showErrorToast } from '../../../../utils/ToastUtils'; import ErrorPlaceHolder from '../../../common/ErrorWithPlaceholder/ErrorPlaceHolder'; import NextPrevious from '../../../common/NextPrevious/NextPrevious'; import { NextPreviousProps } from '../../../common/NextPrevious/NextPrevious.interface'; -import RichTextEditorPreviewer from '../../../common/RichTextEditor/RichTextEditorPreviewer'; +import RichTextEditorPreviewerV1 from '../../../common/RichTextEditor/RichTextEditorPreviewerV1'; import Table from '../../../common/Table/Table'; const DataModelTable = () => { @@ -86,7 +86,7 @@ const DataModelTable = () => { key: 'description', render: (description: ServicePageData['description']) => !isUndefined(description) && description.trim() ? ( - + ) : ( {t('label.no-entity', { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataAssets/AssetsSelectionModal/AssetSelectionModal.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DataAssets/AssetsSelectionModal/AssetSelectionModal.tsx index 003edda57f2f..0f6ec886d9bd 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DataAssets/AssetsSelectionModal/AssetSelectionModal.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DataAssets/AssetsSelectionModal/AssetSelectionModal.tsx @@ -131,14 +131,6 @@ export const AssetSelectionModal = ({ >([]); const [quickFilterQuery, setQuickFilterQuery] = useState(); - const [updatedQueryFilter, setUpdatedQueryFilter] = - useState( - getCombinedQueryFilterObject(queryFilter as QueryFilterInterface, { - query: { - bool: {}, - }, - }) - ); const [selectedFilter, setSelectedFilter] = useState([]); const [filters, setFilters] = useState([]); @@ -237,13 +229,17 @@ export const AssetSelectionModal = ({ useEffect(() => { if (open) { + const combinedQueryFilter = getCombinedQueryFilterObject( + queryFilter as unknown as QueryFilterInterface, + quickFilterQuery as QueryFilterInterface + ); fetchEntities({ index: activeFilter, searchText: search, - updatedQueryFilter, + updatedQueryFilter: combinedQueryFilter, }); } - }, [open, activeFilter, search, type, updatedQueryFilter]); + }, [open, activeFilter, search, type, quickFilterQuery, queryFilter]); useEffect(() => { if (open) { @@ -357,18 +353,6 @@ export const AssetSelectionModal = ({ handleSave(); }, [type, handleSave]); - const mergeFilters = useCallback(() => { - const res = getCombinedQueryFilterObject( - queryFilter as QueryFilterInterface, - quickFilterQuery as QueryFilterInterface - ); - setUpdatedQueryFilter(res); - }, [queryFilter, quickFilterQuery]); - - useEffect(() => { - mergeFilters(); - }, [quickFilterQuery, queryFilter]); - useEffect(() => { const updatedQuickFilters = filters .filter((filter) => selectedFilter.includes(filter.key)) @@ -402,22 +386,28 @@ export const AssetSelectionModal = ({ scrollHeight < 501 && items.length < totalCount ) { + const combinedQueryFilter = getCombinedQueryFilterObject( + queryFilter as unknown as QueryFilterInterface, + quickFilterQuery as QueryFilterInterface + ); + !isLoading && fetchEntities({ searchText: search, page: pageNumber + 1, index: activeFilter, - updatedQueryFilter, + updatedQueryFilter: combinedQueryFilter, }); } }, [ pageNumber, - updatedQueryFilter, activeFilter, search, totalCount, items, + quickFilterQuery, + queryFilter, isLoading, fetchEntities, ] diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataAssets/DataAssetAsyncSelectList/DataAssetAsyncSelectList.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/DataAssets/DataAssetAsyncSelectList/DataAssetAsyncSelectList.interface.ts index d24ea97dbfe1..5d69fbf24120 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DataAssets/DataAssetAsyncSelectList/DataAssetAsyncSelectList.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/DataAssets/DataAssetAsyncSelectList/DataAssetAsyncSelectList.interface.ts @@ -27,6 +27,7 @@ export interface FetchOptionsResponse { export interface DataAssetAsyncSelectListProps { mode?: 'multiple'; + autoFocus?: boolean; id?: string; className?: string; placeholder?: string; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataAssets/DataAssetAsyncSelectList/DataAssetAsyncSelectList.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DataAssets/DataAssetAsyncSelectList/DataAssetAsyncSelectList.tsx index e2872eacdcb7..5b029e749de1 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DataAssets/DataAssetAsyncSelectList/DataAssetAsyncSelectList.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DataAssets/DataAssetAsyncSelectList/DataAssetAsyncSelectList.tsx @@ -36,6 +36,7 @@ import { const DataAssetAsyncSelectList: FC = ({ mode, + autoFocus = true, onChange, debounceTimeout = 800, initialOptions, @@ -242,8 +243,8 @@ const DataAssetAsyncSelectList: FC = ({ return ( - - - form.setFieldValue( - [ - 'testCaseResolutionStatusDetails', - 'testCaseFailureComment', - ], - value - ) - } - /> - + {generateFormFields([descriptionField])} )} {statusType === TestCaseResolutionStatusTypes.Assigned && ( diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestCases/TestCases.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestCases/TestCases.component.tsx index ea92c5a08711..460a08e84d7b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestCases/TestCases.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestCases/TestCases.component.tsx @@ -111,8 +111,8 @@ export const TestCases = () => { return params as TestCaseSearchParams; }, [location.search]); - const { searchValue = '' } = params; + const { searchValue = '' } = params; const [testCase, setTestCase] = useState([]); const [isLoading, setIsLoading] = useState(true); const [selectedFilter, setSelectedFilter] = useState([ @@ -416,7 +416,6 @@ export const TestCases = () => { key: filter, label: startCase(name), value: filter, - onClick: handleMenuClick, })); }, []); @@ -497,6 +496,7 @@ export const TestCases = () => { menu={{ items: filterMenu, selectedKeys: selectedFilter, + onClick: handleMenuClick, }} trigger={['click']}>
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestSuite/AddTestSuiteForm/AddTestSuiteForm.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestSuite/AddTestSuiteForm/AddTestSuiteForm.tsx index bd6a7621f19b..2800555ceb92 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestSuite/AddTestSuiteForm/AddTestSuiteForm.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestSuite/AddTestSuiteForm/AddTestSuiteForm.tsx @@ -12,7 +12,7 @@ */ import { Button, Form, Input, Space } from 'antd'; -import React, { useEffect, useState } from 'react'; +import React, { useEffect, useMemo, useState } from 'react'; import { useTranslation } from 'react-i18next'; import { useHistory } from 'react-router-dom'; import { @@ -22,11 +22,15 @@ import { import { NAME_FIELD_RULES } from '../../../../constants/Form.constants'; import { TabSpecificField } from '../../../../enums/entity.enum'; import { TestSuite } from '../../../../generated/tests/testSuite'; +import { + FieldProp, + FieldTypes, +} from '../../../../interface/FormUtils.interface'; import { DataQualityPageTabs } from '../../../../pages/DataQuality/DataQualityPage.interface'; import { getListTestSuites } from '../../../../rest/testAPI'; +import { getField } from '../../../../utils/formUtils'; import { getDataQualityPagePath } from '../../../../utils/RouterUtils'; import Loader from '../../../common/Loader/Loader'; -import RichTextEditor from '../../../common/RichTextEditor/RichTextEditor'; import { AddTestSuiteFormProps } from '../TestSuiteStepper/TestSuiteStepper.interface'; const AddTestSuiteForm: React.FC = ({ @@ -58,6 +62,21 @@ const AddTestSuiteForm: React.FC = ({ history.push(getDataQualityPagePath(DataQualityPageTabs.TEST_SUITES)); }; + const descriptionField: FieldProp = useMemo( + () => ({ + name: 'description', + required: false, + label: `${t('label.description')}:`, + id: 'root/description', + type: FieldTypes.DESCRIPTION, + props: { + 'data-testid': 'test-suite-description', + initialValue: testSuite?.description ?? '', + }, + }), + [testSuite?.description] + ); + useEffect(() => { fetchTestSuites(); }, []); @@ -101,15 +120,7 @@ const AddTestSuiteForm: React.FC = ({ })} /> - - form.setFieldsValue({ description: value })} - /> - - + {getField(descriptionField)} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestSuite/TestSuiteList/TestSuites.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestSuite/TestSuiteList/TestSuites.test.tsx index 82effefc8043..29ff886a4b21 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestSuite/TestSuiteList/TestSuites.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestSuite/TestSuiteList/TestSuites.test.tsx @@ -40,12 +40,12 @@ const mockList = { name: 'sample_data.ecommerce_db.shopify.dim_address.testSuite', fullyQualifiedName: 'sample_data.ecommerce_db.shopify.dim_address.testSuite', - description: 'This is an executable test suite linked to an entity', + description: 'This is an basic test suite linked to an entity', serviceType: 'TestSuite', href: 'href', deleted: false, - executable: true, - executableEntityReference: { + basic: true, + basicEntityReference: { id: 'id1', type: 'table', name: 'dim_address', @@ -161,7 +161,7 @@ describe('TestSuites component', () => { ).toBeInTheDocument(); }); - it('should send testSuiteType executable in api, if active tab is tables', async () => { + it('should send testSuiteType basic in api, if active tab is tables', async () => { const mockGetListTestSuites = getListTestSuitesBySearch as jest.Mock; render(); @@ -180,7 +180,7 @@ describe('TestSuites component', () => { sortNestedMode: ['max'], sortNestedPath: 'testCaseResultSummary', sortType: 'desc', - testSuiteType: 'executable', + testSuiteType: 'basic', }); }); @@ -202,7 +202,7 @@ describe('TestSuites component', () => { sortNestedMode: ['max'], sortNestedPath: 'testCaseResultSummary', sortType: 'desc', - testSuiteType: 'executable', + testSuiteType: 'basic', }); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestSuite/TestSuitePipelineTab/TestSuitePipelineTab.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestSuite/TestSuitePipelineTab/TestSuitePipelineTab.component.tsx index bbfae7d4cee4..14a32abe86e8 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestSuite/TestSuitePipelineTab/TestSuitePipelineTab.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestSuite/TestSuitePipelineTab/TestSuitePipelineTab.component.tsx @@ -15,6 +15,7 @@ import { PlusOutlined } from '@ant-design/icons'; import { Button, Col, Row } from 'antd'; import { AxiosError } from 'axios'; import { sortBy } from 'lodash'; +import QueryString from 'qs'; import React, { useCallback, useEffect, useMemo, useState } from 'react'; import { useTranslation } from 'react-i18next'; import { useHistory } from 'react-router-dom'; @@ -27,6 +28,7 @@ import { PipelineType } from '../../../../generated/api/services/ingestionPipeli import { Table as TableType } from '../../../../generated/entity/data/table'; import { Operation } from '../../../../generated/entity/policies/policy'; import { IngestionPipeline } from '../../../../generated/entity/services/ingestionPipelines/ingestionPipeline'; +import { TestSuite } from '../../../../generated/tests/testCase'; import { useAirflowStatus } from '../../../../hooks/useAirflowStatus'; import { deployIngestionPipelineById, @@ -43,10 +45,14 @@ import ErrorPlaceHolderIngestion from '../../../common/ErrorWithPlaceholder/Erro import IngestionListTable from '../../../Settings/Services/Ingestion/IngestionListTable/IngestionListTable'; interface Props { - testSuite: TableType['testSuite']; + testSuite: TableType['testSuite'] | TestSuite; + isLogicalTestSuite?: boolean; } -const TestSuitePipelineTab = ({ testSuite }: Props) => { +const TestSuitePipelineTab = ({ + testSuite, + isLogicalTestSuite = false, +}: Props) => { const airflowInformation = useAirflowStatus(); const { t } = useTranslation(); const testSuiteFQN = testSuite?.fullyQualifiedName ?? testSuite?.name ?? ''; @@ -102,6 +108,15 @@ const TestSuitePipelineTab = ({ testSuite }: Props) => { } }, [testSuiteFQN]); + const handleAddPipelineRedirection = () => { + history.push({ + pathname: getTestSuiteIngestionPath(testSuiteFQN), + search: isLogicalTestSuite + ? QueryString.stringify({ testSuiteId: testSuite?.id }) + : undefined, + }); + }; + const handleEnableDisableIngestion = useCallback( async (id: string) => { try { @@ -194,9 +209,7 @@ const TestSuitePipelineTab = ({ testSuite }: Props) => { data-testid="add-placeholder-button" icon={} type="primary" - onClick={() => { - history.push(getTestSuiteIngestionPath(testSuiteFQN)); - }}> + onClick={handleAddPipelineRedirection}> {t('label.add')} } @@ -225,9 +238,7 @@ const TestSuitePipelineTab = ({ testSuite }: Props) => { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestSuite/TestSuitePipelineTab/TestSuitePipelineTab.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestSuite/TestSuitePipelineTab/TestSuitePipelineTab.test.tsx index 72c7303512a8..d1cb54014a4b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestSuite/TestSuitePipelineTab/TestSuitePipelineTab.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestSuite/TestSuitePipelineTab/TestSuitePipelineTab.test.tsx @@ -38,8 +38,8 @@ const mockTestSuite = { updatedAt: 1692766701920, updatedBy: 'admin', deleted: false, - executable: true, - executableEntityReference: { + basic: true, + basicEntityReference: { id: 'e926d275-441e-49ee-a073-ad509f625a14', type: 'table', name: 'web_analytic_event', diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestSuite/TestSuiteStepper/TestSuiteStepper.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestSuite/TestSuiteStepper/TestSuiteStepper.tsx index e7c0fc86efce..68c6d66bc1dc 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestSuite/TestSuiteStepper/TestSuiteStepper.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestSuite/TestSuiteStepper/TestSuiteStepper.tsx @@ -38,6 +38,7 @@ import TitleBreadcrumb from '../../../common/TitleBreadcrumb/TitleBreadcrumb.com import IngestionStepper from '../../../Settings/Services/Ingestion/IngestionStepper/IngestionStepper.component'; import RightPanel from '../../AddDataQualityTest/components/RightPanel'; import { getRightPanelForAddTestSuitePage } from '../../AddDataQualityTest/rightPanelData'; +import TestSuiteIngestion from '../../AddDataQualityTest/TestSuiteIngestion'; import { AddTestCaseList } from '../../AddTestCaseList/AddTestCaseList.component'; import AddTestSuiteForm from '../AddTestSuiteForm/AddTestSuiteForm'; @@ -47,6 +48,7 @@ const TestSuiteStepper = () => { const { currentUser } = useApplicationStore(); const [activeServiceStep, setActiveServiceStep] = useState(1); const [testSuiteResponse, setTestSuiteResponse] = useState(); + const [addIngestion, setAddIngestion] = useState(false); const handleViewTestSuiteClick = () => { history.push(getTestSuitePath(testSuiteResponse?.fullyQualifiedName ?? '')); @@ -114,9 +116,10 @@ const TestSuiteStepper = () => { } else if (activeServiceStep === 3) { return ( setAddIngestion(true)} handleViewServiceClick={handleViewTestSuiteClick} name={testSuiteResponse?.name || ''} - showIngestionButton={false} state={FormSubmitType.ADD} viewServiceText="View Test Suite" /> @@ -142,25 +145,33 @@ const TestSuiteStepper = () => { data-testid="test-suite-stepper-container"> - - - - {t('label.add-entity', { - entity: t('label.test-suite'), - })} - - - - - - {RenderSelectedTab()} - + {addIngestion ? ( + setAddIngestion(false)} + onViewServiceClick={handleViewTestSuiteClick} + /> + ) : ( + + + + {t('label.add-entity', { + entity: t('label.test-suite'), + })} + + + + + + {RenderSelectedTab()} + + )} ), diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Database/DatabaseSchema/DatabaseSchemaTable/DatabaseSchemaTable.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Database/DatabaseSchema/DatabaseSchemaTable/DatabaseSchemaTable.tsx index 64148e9108e6..19858045499e 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Database/DatabaseSchema/DatabaseSchemaTable/DatabaseSchemaTable.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Database/DatabaseSchema/DatabaseSchemaTable/DatabaseSchemaTable.tsx @@ -48,7 +48,7 @@ import DisplayName from '../../../common/DisplayName/DisplayName'; import ErrorPlaceHolder from '../../../common/ErrorWithPlaceholder/ErrorPlaceHolder'; import NextPrevious from '../../../common/NextPrevious/NextPrevious'; import { PagingHandlerParams } from '../../../common/NextPrevious/NextPrevious.interface'; -import RichTextEditorPreviewer from '../../../common/RichTextEditor/RichTextEditorPreviewer'; +import RichTextEditorPreviewerV1 from '../../../common/RichTextEditor/RichTextEditorPreviewerV1'; import Searchbar from '../../../common/SearchBarComponent/SearchBar.component'; import Table from '../../../common/Table/Table'; import { EntityName } from '../../../Modals/EntityNameModal/EntityNameModal.interface'; @@ -244,7 +244,7 @@ export const DatabaseSchemaTable = ({ key: 'description', render: (text: string) => text?.trim() ? ( - + ) : ( {t('label.no-entity', { entity: t('label.description') })} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Database/Profiler/DataQualityTab/DataQualityTab.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Database/Profiler/DataQualityTab/DataQualityTab.test.tsx index 18574d0acde7..8e0dc164ecd9 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Database/Profiler/DataQualityTab/DataQualityTab.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Database/Profiler/DataQualityTab/DataQualityTab.test.tsx @@ -18,6 +18,7 @@ import { fireEvent, render, screen, + waitFor, } from '@testing-library/react'; import React from 'react'; import { MOCK_TEST_CASE } from '../../../../mocks/TestSuite.mock'; @@ -68,7 +69,7 @@ jest.mock('../../../common/DeleteWidget/DeleteWidgetModal', () => { return ( visible && (
- DeleteWidgetModal +

DeleteWidgetModal

) @@ -80,7 +81,7 @@ jest.mock('../../../DataQuality/AddDataQualityTest/EditTestCaseModal', () => { return ( visible && (
- EditTestCaseModal +

EditTestCaseModal

@@ -88,6 +89,28 @@ jest.mock('../../../DataQuality/AddDataQualityTest/EditTestCaseModal', () => { ); }); }); +jest.mock('../../../Modals/ConfirmationModal/ConfirmationModal', () => { + return jest + .fn() + .mockImplementation(({ visible, onCancel, onConfirm, isLoading }) => { + return ( + visible && ( +
+

ConfirmationModal

+ + +
+ ) + ); + }); +}); describe('DataQualityTab test', () => { it('Component should render', async () => { @@ -216,6 +239,53 @@ describe('DataQualityTab test', () => { expect(deleteButton).toBeInTheDocument(); }); + it('Remove functionality', async () => { + const firstRowData = MOCK_TEST_CASE[0]; + await act(async () => { + render( + + ); + }); + const tableRows = await screen.findAllByRole('row'); + const firstRow = tableRows[1]; + const closeRemoveModel = screen.queryByText('ConfirmationModal'); + const removeButton = await findByTestId( + firstRow, + `remove-${firstRowData.name}` + ); + + expect(removeButton).toBeInTheDocument(); + expect(closeRemoveModel).not.toBeInTheDocument(); + + await act(async () => { + fireEvent.click(removeButton); + }); + const openRemoveModel = await screen.findByText('ConfirmationModal'); + const submitBtn = await screen.findByText('submit'); + + expect(openRemoveModel).toBeInTheDocument(); + + await act(async () => { + fireEvent.click(submitBtn); + + await waitFor(() => { + const loader = screen.getByTestId('submit-btn-loading'); + + expect(loader).toBeInTheDocument(); + }); + }); + + expect(closeRemoveModel).not.toBeInTheDocument(); + }); + it('Edit functionality', async () => { const firstRowData = MOCK_TEST_CASE[0]; await act(async () => { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Database/Profiler/DataQualityTab/DataQualityTab.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Database/Profiler/DataQualityTab/DataQualityTab.tsx index 08d3dd98f4fe..21ec041b72fc 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Database/Profiler/DataQualityTab/DataQualityTab.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Database/Profiler/DataQualityTab/DataQualityTab.tsx @@ -88,6 +88,8 @@ const DataQualityTab: React.FC = ({ const [testCaseStatus, setTestCaseStatus] = useState< TestCaseResolutionStatus[] >([]); + const [isTestCaseRemovalLoading, setIsTestCaseRemovalLoading] = + useState(false); const isApiSortingEnabled = useRef(false); const testCaseEditPermission = useMemo(() => { @@ -131,6 +133,7 @@ const DataQualityTab: React.FC = ({ }; const handleConfirmClick = async () => { + setIsTestCaseRemovalLoading(true); if (isUndefined(removeFromTestSuite)) { return; } @@ -143,6 +146,8 @@ const DataQualityTab: React.FC = ({ setSelectedTestCase(undefined); } catch (error) { showErrorToast(error as AxiosError); + } finally { + setIsTestCaseRemovalLoading(false); } }; @@ -514,6 +519,7 @@ const DataQualityTab: React.FC = ({ cancelText={t('label.cancel')} confirmText={t('label.remove')} header={t('label.remove-entity', { entity: t('label.test-case') })} + isLoading={isTestCaseRemovalLoading} visible={selectedTestCase?.action === 'DELETE'} onCancel={handleCancel} onConfirm={handleConfirmClick} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Database/Profiler/TableProfiler/ColumnSummary.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Database/Profiler/TableProfiler/ColumnSummary.tsx index 1b7758190ae4..a0967669f6fa 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Database/Profiler/TableProfiler/ColumnSummary.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Database/Profiler/TableProfiler/ColumnSummary.tsx @@ -15,7 +15,7 @@ import { isEmpty } from 'lodash'; import React, { FC } from 'react'; import { Column } from '../../../../generated/entity/data/container'; import { getEntityName } from '../../../../utils/EntityUtils'; -import RichTextEditorPreviewer from '../../../common/RichTextEditor/RichTextEditorPreviewer'; +import RichTextEditorPreviewerV1 from '../../../common/RichTextEditor/RichTextEditorPreviewerV1'; import TagsViewer from '../../../Tag/TagsViewer/TagsViewer'; interface ColumnSummaryProps { @@ -31,7 +31,7 @@ const ColumnSummary: FC = ({ column }) => { {`(${column.dataType})`} - { const { t } = useTranslation(); @@ -26,7 +27,7 @@ const NoProfilerBanner = () => { {t('message.no-profiler-message')} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Database/RetentionPeriod/RetentionPeriod.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Database/RetentionPeriod/RetentionPeriod.component.tsx index 83a1699ef053..b0f8ef72ecb1 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Database/RetentionPeriod/RetentionPeriod.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Database/RetentionPeriod/RetentionPeriod.component.tsx @@ -143,6 +143,7 @@ const RetentionPeriod = ({ - - {displayValue} - - + return ( + + {highlightSearchArrayElement(dataTypeDisplay, searchText)} + ); }; @@ -268,7 +267,7 @@ const SchemaTable = ({ - {name} + {stringToHTML(highlightSearchText(name, searchText))} {!isEmpty(displayName) ? ( @@ -386,7 +385,9 @@ const SchemaTable = ({ - {getEntityName(record)} + {stringToHTML( + highlightSearchText(getEntityName(record), searchText) + )} ) : null} @@ -419,7 +420,6 @@ const SchemaTable = ({ dataIndex: 'dataTypeDisplay', key: 'dataTypeDisplay', accessor: 'dataTypeDisplay', - ellipsis: true, width: 150, render: renderDataTypeDisplay, }, diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Database/SchemaTable/SchemaTable.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Database/SchemaTable/SchemaTable.test.tsx index 1e49687b4a50..a5cafd916810 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Database/SchemaTable/SchemaTable.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Database/SchemaTable/SchemaTable.test.tsx @@ -126,7 +126,7 @@ jest.mock('../../../hooks/authHooks', () => { }; }); -jest.mock('../../common/RichTextEditor/RichTextEditorPreviewer', () => { +jest.mock('../../common/RichTextEditor/RichTextEditorPreviewerV1', () => { return jest.fn().mockReturnValue(

RichTextEditorPreviewer

); }); @@ -167,6 +167,16 @@ jest.mock('../../../constants/Table.constants', () => ({ }, })); +jest.mock('../../../utils/StringsUtils', () => ({ + ...jest.requireActual('../../../utils/StringsUtils'), + stringToHTML: jest.fn((text) => text), +})); + +jest.mock('../../../utils/EntityUtils', () => ({ + ...jest.requireActual('../../../utils/EntityUtils'), + highlightSearchText: jest.fn((text) => text), +})); + describe('Test EntityTable Component', () => { it('Initially, Table should load', async () => { render(, { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Database/TableDataCardBody/TableDataCardBody.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Database/TableDataCardBody/TableDataCardBody.test.tsx index 705dfa4c01ff..307ee62f7b85 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Database/TableDataCardBody/TableDataCardBody.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Database/TableDataCardBody/TableDataCardBody.test.tsx @@ -16,7 +16,7 @@ import React from 'react'; import { TAG_CONSTANT } from '../../../constants/Tag.constants'; import TableDataCardBody from './TableDataCardBody'; -jest.mock('../../common/RichTextEditor/RichTextEditorPreviewer', () => { +jest.mock('../../common/RichTextEditor/RichTextEditorPreviewerV1', () => { return jest.fn().mockReturnValue(

RichTextEditorPreviewer

); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Database/TableDataCardBody/TableDataCardBody.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Database/TableDataCardBody/TableDataCardBody.tsx index a38cdfe00014..1827ec5cc354 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Database/TableDataCardBody/TableDataCardBody.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Database/TableDataCardBody/TableDataCardBody.tsx @@ -18,7 +18,7 @@ import { useTranslation } from 'react-i18next'; import { TagLabel } from '../../../generated/type/tagLabel'; import { getTagValue } from '../../../utils/CommonUtils'; import EntitySummaryDetails from '../../common/EntitySummaryDetails/EntitySummaryDetails'; -import RichTextEditorPreviewer from '../../common/RichTextEditor/RichTextEditorPreviewer'; +import RichTextEditorPreviewerV1 from '../../common/RichTextEditor/RichTextEditorPreviewerV1'; import TagsViewer from '../../Tag/TagsViewer/TagsViewer'; type Props = { @@ -38,7 +38,7 @@ const TableDataCardBody: FunctionComponent = ({
{description.trim() ? ( - ; + return ; } else { return ( diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Database/TableDescription/TableDescription.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Database/TableDescription/TableDescription.test.tsx index 0da7d0c4cab3..2c3d679509bc 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Database/TableDescription/TableDescription.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Database/TableDescription/TableDescription.test.tsx @@ -20,7 +20,7 @@ jest.mock('../../../pages/TasksPage/EntityTasks/EntityTasks.component', () => { return jest.fn().mockReturnValue(

EntityTasks

); }); -jest.mock('../../common/RichTextEditor/RichTextEditorPreviewer', () => { +jest.mock('../../common/RichTextEditor/RichTextEditorPreviewerV1', () => { return jest.fn().mockReturnValue(

RichTextEditorPreviewer

); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Database/TableQueries/QueryCard.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Database/TableQueries/QueryCard.tsx index 843954e93d49..f247f98e7442 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Database/TableQueries/QueryCard.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Database/TableQueries/QueryCard.tsx @@ -130,7 +130,7 @@ const QueryCard: FC = ({ return ( existingTable ?? { id: (option.value as string) ?? '', - displayName: option.label as string, + displayName: option.labelName as string, type: EntityType.TABLE, } ); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Database/TableQueries/QueryUsedByOtherTable/QueryUsedByOtherTable.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Database/TableQueries/QueryUsedByOtherTable/QueryUsedByOtherTable.component.tsx index bc6ae182ca41..cab6ccf16eb0 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Database/TableQueries/QueryUsedByOtherTable/QueryUsedByOtherTable.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Database/TableQueries/QueryUsedByOtherTable/QueryUsedByOtherTable.component.tsx @@ -25,7 +25,7 @@ import { QUERY_USED_BY_TABLE_VIEW_CAP } from '../../../../constants/Query.consta import { EntityType } from '../../../../enums/entity.enum'; import { SearchIndex } from '../../../../enums/search.enum'; import { searchData } from '../../../../rest/miscAPI'; -import { getEntityName } from '../../../../utils/EntityUtils'; +import { getEntityLabel, getEntityName } from '../../../../utils/EntityUtils'; import { AsyncSelect } from '../../../common/AsyncSelect/AsyncSelect'; import Loader from '../../../common/Loader/Loader'; import { @@ -135,8 +135,9 @@ const QueryUsedByOtherTable = ({ ); return data.hits.hits.map((value) => ({ - label: getEntityName(value._source), + label: getEntityLabel(value._source), value: value._source.id, + labelName: getEntityName(value._source), })); } catch (error) { return []; @@ -149,11 +150,12 @@ const QueryUsedByOtherTable = ({ const options = await fetchTableEntity(); const { queryUsedIn = [] } = query; const selectedValue = queryUsedIn.map((table) => ({ - label: getEntityName(table), + label: getEntityLabel(table), value: table.id, + labelName: getEntityName(table), })); - setInitialOptions(uniqBy([...selectedValue, ...options], 'value')); + setInitialOptions(uniqBy([...selectedValue, ...options], 'labelName')); } catch (error) { setInitialOptions([]); } finally { @@ -171,10 +173,11 @@ const QueryUsedByOtherTable = ({ ) : ( { const { t } = useTranslation(); const [form] = useForm(); - const { getEntityPermission } = usePermissionProvider(); + const { getEntityPermission, permissions } = usePermissionProvider(); const history = useHistory(); const { tab: activeTab, version } = useParams<{ tab: string; version: string }>(); @@ -201,21 +203,29 @@ const DomainDetailsPage = ({ }, [domainPermission]); const addButtonContent = [ - { - label: t('label.asset-plural'), - key: '1', - onClick: () => setAssetModalVisible(true), - }, - { - label: t('label.sub-domain-plural'), - key: '2', - onClick: () => setShowAddSubDomainModal(true), - }, - { - label: t('label.data-product-plural'), - key: '3', - onClick: () => setShowAddDataProductModal(true), - }, + ...(domainPermission.Create + ? [ + { + label: t('label.asset-plural'), + key: '1', + onClick: () => setAssetModalVisible(true), + }, + { + label: t('label.sub-domain-plural'), + key: '2', + onClick: () => setShowAddSubDomainModal(true), + }, + ] + : []), + ...(permissions.dataProduct.Create + ? [ + { + label: t('label.data-product-plural'), + key: '3', + onClick: () => setShowAddDataProductModal(true), + }, + ] + : []), ]; const fetchSubDomains = useCallback(async () => { @@ -338,17 +348,17 @@ const DomainDetailsPage = ({ const fetchDomainAssets = async () => { if (domainFqn && !isVersionsView) { try { - const res = await searchData( - '', - 1, - 0, - `(domain.fullyQualifiedName:"${encodedFqn}") AND !(entityType:"dataProduct")`, - '', - '', - SearchIndex.ALL - ); - - setAssetCount(res.data.hits.total.value ?? 0); + const res = await searchQuery({ + query: '', + pageNumber: 0, + pageSize: 0, + queryFilter, + searchIndex: SearchIndex.ALL, + filters: '', + }); + + const totalCount = res?.hits?.total.value ?? 0; + setAssetCount(totalCount); } catch (error) { setAssetCount(0); } @@ -381,7 +391,7 @@ const DomainDetailsPage = ({ setShowAddDataProductModal(true); }, []); - const onNameSave = (obj: { name: string; displayName?: string }) => { + const onNameSave = (obj: EntityName) => { const { displayName } = obj; let updatedDetails = cloneDeep(domain); @@ -496,6 +506,10 @@ const DomainDetailsPage = ({ : []), ]; + const queryFilter = useMemo(() => { + return getQueryFilterForDomain(domainFqn); + }, [domainFqn]); + const tabs = useMemo(() => { return [ { @@ -510,6 +524,7 @@ const DomainDetailsPage = ({ onUpdate(data as Domain)} /> ), @@ -575,6 +590,7 @@ const DomainDetailsPage = ({ entityFqn={domainFqn} isSummaryPanelOpen={false} permissions={domainPermission} + queryFilter={queryFilter} ref={assetTabRef} type={AssetsOfEntity.DOMAIN} onAddAsset={() => setAssetModalVisible(true)} @@ -617,6 +633,7 @@ const DomainDetailsPage = ({ activeTab, subDomains, isSubDomainsLoading, + queryFilter, ]); useEffect(() => { @@ -629,6 +646,41 @@ const DomainDetailsPage = ({ fetchSubDomains(); }, [domainFqn]); + const iconData = useMemo(() => { + if (domain.style?.iconURL) { + return ( + domain-icon + ); + } else if (isSubDomain) { + return ( + + ); + } + + return ( + + ); + }, [domain, isSubDomain]); + return ( <> - ) : ( - - ) - } + icon={iconData} serviceName="" titleColor={domain.style?.color} />
- {!isVersionsView && domainPermission.Create && ( + {!isVersionsView && addButtonContent.length > 0 && ( setAssetModalVisible(false)} onSave={handleAssetSave} @@ -824,7 +850,7 @@ const DomainDetailsPage = ({ }} /> )} - + { const { t } = useTranslation(); - const { permissions } = usePermissionProvider(); const [isDescriptionEditable, setIsDescriptionEditable] = useState(false); const [editDomainType, setEditDomainType] = useState(false); @@ -72,40 +67,40 @@ const DocumentationTab = ({ ? ResourceEntity.DOMAIN : ResourceEntity.DATA_PRODUCT; - const { editDescriptionPermission, editOwnerPermission, editAllPermission } = - useMemo(() => { - if (isVersionsView) { - return { - editDescriptionPermission: false, - editOwnerPermission: false, - editAllPermission: false, - }; - } + const { + editDescriptionPermission, + editOwnerPermission, + editAllPermission, + editCustomAttributePermission, + viewAllPermission, + } = useMemo(() => { + if (isVersionsView) { + return { + editDescriptionPermission: false, + editOwnerPermission: false, + editAllPermission: false, + editCustomAttributePermission: false, + }; + } - const editDescription = checkPermission( - Operation.EditDescription, - resourceType, - permissions - ); + const editDescription = permissions?.EditDescription; - const editOwner = checkPermission( - Operation.EditOwners, - resourceType, - permissions - ); + const editOwner = permissions?.EditOwners; - const editAll = checkPermission( - Operation.EditAll, - resourceType, - permissions - ); + const editAll = permissions?.EditAll; - return { - editDescriptionPermission: editDescription || editAll, - editOwnerPermission: editOwner || editAll, - editAllPermission: editAll, - }; - }, [permissions, isVersionsView, resourceType]); + const editCustomAttribute = permissions?.EditCustomFields; + + const viewAll = permissions?.ViewAll; + + return { + editDescriptionPermission: editAll || editDescription, + editOwnerPermission: editAll || editOwner, + editAllPermission: editAll, + editCustomAttributePermission: editAll || editCustomAttribute, + viewAllPermission: viewAll, + }; + }, [permissions, isVersionsView, resourceType]); const description = useMemo( () => @@ -202,26 +197,32 @@ const DocumentationTab = ({ {t('label.owner-plural')} - {editOwnerPermission && domain.owners && ( - handleUpdatedOwner(updatedUser)}> - -
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Domain/DomainTabs/DocumentationTab/DocumentationTab.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/Domain/DomainTabs/DocumentationTab/DocumentationTab.interface.ts index e10500799b54..faf3bb363808 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Domain/DomainTabs/DocumentationTab/DocumentationTab.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/Domain/DomainTabs/DocumentationTab/DocumentationTab.interface.ts @@ -10,6 +10,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import { OperationPermission } from '../../../../context/PermissionProvider/PermissionProvider.interface'; import { DataProduct } from '../../../../generated/entity/domains/dataProduct'; import { Domain } from '../../../../generated/entity/domains/domain'; @@ -19,8 +20,7 @@ export interface DocumentationTabProps { isVersionsView?: boolean; type?: DocumentationEntity; onExtensionUpdate?: (updatedDataProduct: DataProduct) => Promise; - editCustomAttributePermission?: boolean; - viewAllPermission?: boolean; + permissions?: OperationPermission; } export enum DocumentationEntity { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Domain/DomainTabs/DocumentationTab/DocumentationTab.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Domain/DomainTabs/DocumentationTab/DocumentationTab.test.tsx index f2fe4e09640d..dbdd8f3222ba 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Domain/DomainTabs/DocumentationTab/DocumentationTab.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Domain/DomainTabs/DocumentationTab/DocumentationTab.test.tsx @@ -14,6 +14,7 @@ import { render, screen } from '@testing-library/react'; import React from 'react'; import { MemoryRouter } from 'react-router-dom'; import { MOCK_DOMAIN } from '../../../../mocks/Domains.mock'; +import { MOCK_PERMISSIONS } from '../../../../mocks/Glossary.mock'; import DocumentationTab from './DocumentationTab.component'; // Mock the onUpdate function @@ -23,6 +24,7 @@ const defaultProps = { domain: MOCK_DOMAIN, onUpdate: mockOnUpdate, isVersionsView: false, + permissions: MOCK_PERMISSIONS, }; jest.mock('../../../common/EntityDescription/DescriptionV1', () => { @@ -33,10 +35,6 @@ jest.mock('../../../common/ProfilePicture/ProfilePicture', () => jest.fn().mockReturnValue(<>ProfilePicture) ); -jest.mock('../../../../utils/PermissionsUtils', () => ({ - checkPermission: jest.fn().mockReturnValue(true), -})); - describe('DocumentationTab', () => { it('should render the initial content', () => { const { getByTestId } = render(, { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Domain/SubDomainsTable/SubDomainsTable.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Domain/SubDomainsTable/SubDomainsTable.component.tsx index 81d341d72aac..19a1958621f5 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Domain/SubDomainsTable/SubDomainsTable.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Domain/SubDomainsTable/SubDomainsTable.component.tsx @@ -26,7 +26,7 @@ import { getDomainDetailsPath } from '../../../utils/RouterUtils'; import ErrorPlaceHolder from '../../common/ErrorWithPlaceholder/ErrorPlaceHolder'; import Loader from '../../common/Loader/Loader'; import { OwnerLabel } from '../../common/OwnerLabel/OwnerLabel.component'; -import RichTextEditorPreviewer from '../../common/RichTextEditor/RichTextEditorPreviewer'; +import RichTextEditorPreviewerV1 from '../../common/RichTextEditor/RichTextEditorPreviewerV1'; import { SubDomainsTableProps } from './SubDomainsTable.interface'; const SubDomainsTable = ({ @@ -63,7 +63,7 @@ const SubDomainsTable = ({ key: 'description', render: (description: string) => description.trim() ? ( - - {icon && {icon}} + {icon && {icon}} {t('label.remove-entity', { entity: t('label.edge-lowercase'), diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomEdge.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomEdge.component.tsx index 9af2cd4936fa..b7ede63f5ff5 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomEdge.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomEdge.component.tsx @@ -15,7 +15,8 @@ import Icon from '@ant-design/icons/lib/components/Icon'; import { Button, Tag } from 'antd'; import classNames from 'classnames'; import React, { Fragment, useCallback, useMemo } from 'react'; -import { EdgeProps, getBezierPath } from 'reactflow'; +import { EdgeProps } from 'reactflow'; +import { ReactComponent as IconEditCircle } from '../../../assets/svg/ic-edit-circle.svg'; import { ReactComponent as FunctionIcon } from '../../../assets/svg/ic-function.svg'; import { ReactComponent as IconTimesCircle } from '../../../assets/svg/ic-times-circle.svg'; import { ReactComponent as PipelineIcon } from '../../../assets/svg/pipeline-grey.svg'; @@ -26,7 +27,10 @@ import { EntityType } from '../../../enums/entity.enum'; import { StatusType } from '../../../generated/entity/data/pipeline'; import { LineageLayer } from '../../../generated/settings/settings'; import { useApplicationStore } from '../../../hooks/useApplicationStore'; -import { getColumnSourceTargetHandles } from '../../../utils/EntityLineageUtils'; +import { + getColumnSourceTargetHandles, + getEdgePathData, +} from '../../../utils/EntityLineageUtils'; import { getEntityName } from '../../../utils/EntityUtils'; import EntityPopOverCard from '../../common/PopOverCard/EntityPopOverCard'; import { CustomEdgeData } from './EntityLineage.interface'; @@ -68,6 +72,8 @@ export const CustomEdge = ({ markerEnd, data, selected, + source, + target, }: EdgeProps) => { const { edge, @@ -95,6 +101,21 @@ export const CustomEdge = ({ const { theme } = useApplicationStore(); + const { + edgePath, + edgeCenterX, + edgeCenterY, + invisibleEdgePath, + invisibleEdgePath1, + } = getEdgePathData(source, target, offset, { + sourceX, + sourceY, + targetX, + targetY, + sourcePosition, + targetPosition, + }); + const showDqTracing = useMemo(() => { return ( (activeLayer.includes(LineageLayer.DataObservability) && @@ -121,31 +142,6 @@ export const CustomEdge = ({ ); }, [isColumnLineage, tracedColumns, sourceHandle, targetHandle]); - const [edgePath, edgeCenterX, edgeCenterY] = getBezierPath({ - sourceX, - sourceY, - sourcePosition, - targetX, - targetY, - targetPosition, - }); - const [invisibleEdgePath] = getBezierPath({ - sourceX: sourceX + offset, - sourceY: sourceY + offset, - sourcePosition, - targetX: targetX + offset, - targetY: targetY + offset, - targetPosition, - }); - const [invisibleEdgePath1] = getBezierPath({ - sourceX: sourceX - offset, - sourceY: sourceY - offset, - sourcePosition, - targetX: targetX - offset, - targetY: targetY - offset, - targetPosition, - }); - const updatedStyle = useMemo(() => { const isNodeTraced = tracedNodes.includes(edge.fromEntity.id) && @@ -314,13 +310,13 @@ export const CustomEdge = ({ const getEditLineageIcon = useCallback( ( dataTestId: string, - rotate: boolean, onClick: | (( event: React.MouseEvent, data: CustomEdgeData ) => void) - | undefined + | undefined, + isPipeline?: boolean ) => { return ( @@ -331,13 +327,10 @@ export const CustomEdge = ({ } - style={{ - transform: rotate ? 'rotate(45deg)' : 'none', - }} type="link" onClick={(event) => onClick?.(event, rest as CustomEdgeData)} /> @@ -377,11 +370,11 @@ export const CustomEdge = ({ )} {isColumnLineageAllowed && isSelectedEditMode && - getEditLineageIcon('add-pipeline', true, onAddPipelineClick)} + getEditLineageIcon('add-pipeline', onAddPipelineClick, true)} {!isColumnLineageAllowed && isSelectedEditMode && isSelected && - getEditLineageIcon('delete-button', false, onColumnEdgeRemove)} + getEditLineageIcon('delete-button', onColumnEdgeRemove)} {!isColumnLineageAllowed && data.columnFunctionValue && data.isExpanded && diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/Task/TaskTab/TaskTab.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/Task/TaskTab/TaskTab.component.tsx index 4e6cd1a724cb..934efefe4d92 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/Task/TaskTab/TaskTab.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/Task/TaskTab/TaskTab.component.tsx @@ -71,6 +71,10 @@ import { import { TagLabel } from '../../../../generated/type/tagLabel'; import { useAuth } from '../../../../hooks/authHooks'; import { useApplicationStore } from '../../../../hooks/useApplicationStore'; +import { + FieldProp, + FieldTypes, +} from '../../../../interface/FormUtils.interface'; import Assignees from '../../../../pages/TasksPage/shared/Assignees'; import DescriptionTask from '../../../../pages/TasksPage/shared/DescriptionTask'; import TagsTask from '../../../../pages/TasksPage/shared/TagsTask'; @@ -84,6 +88,7 @@ import { postTestCaseIncidentStatus } from '../../../../rest/incidentManagerAPI' import { getNameFromFQN } from '../../../../utils/CommonUtils'; import EntityLink from '../../../../utils/EntityLink'; import { getEntityFQN } from '../../../../utils/FeedUtils'; +import { getField } from '../../../../utils/formUtils'; import { checkPermission } from '../../../../utils/PermissionsUtils'; import { getErrorText } from '../../../../utils/StringsUtils'; import { @@ -106,8 +111,6 @@ import { useActivityFeedProvider } from '../../../ActivityFeed/ActivityFeedProvi import InlineEdit from '../../../common/InlineEdit/InlineEdit.component'; import { OwnerLabel } from '../../../common/OwnerLabel/OwnerLabel.component'; import EntityPopOverCard from '../../../common/PopOverCard/EntityPopOverCard'; -import RichTextEditor from '../../../common/RichTextEditor/RichTextEditor'; -import { EditorContentRef as MarkdownEditorContentRef } from '../../../Modals/ModalWithMarkdownEditor/ModalWithMarkdownEditor.interface'; import TaskTabIncidentManagerHeader from '../TaskTabIncidentManagerHeader/TaskTabIncidentManagerHeader.component'; import './task-tab.less'; import { TaskTabProps } from './TaskTab.interface'; @@ -123,7 +126,6 @@ export const TaskTab = ({ const history = useHistory(); const [assigneesForm] = useForm(); const { currentUser } = useApplicationStore(); - const markdownRef = useRef(); const updatedAssignees = Form.useWatch('assignees', assigneesForm); const { permissions } = usePermissionProvider(); const { task: taskDetails } = taskThread; @@ -431,8 +433,11 @@ export const TaskTab = ({ onTaskResolve(); } setTaskAction( - TASK_ACTION_LIST.find((action) => action.key === info.key) ?? - TASK_ACTION_LIST[0] + [ + ...TASK_ACTION_LIST, + ...GLOSSARY_TASK_ACTION_LIST, + ...INCIDENT_TASK_ACTION_LIST, + ].find((action) => action.key === info.key) ?? TASK_ACTION_LIST[0] ); }; @@ -896,6 +901,32 @@ export const TaskTab = ({ ); + const descriptionField: FieldProp = useMemo( + () => ({ + name: 'testCaseFailureComment', + required: true, + label: t('label.comment'), + id: 'root/description', + type: FieldTypes.DESCRIPTION, + rules: [ + { + required: true, + message: t('label.field-required', { + field: t('label.comment'), + }), + }, + ], + props: { + 'data-testid': 'description', + initialValue: '', + placeHolder: t('message.write-your-text', { + text: t('label.comment'), + }), + }, + }), + [] + ); + return ( - - - + {getField(descriptionField)} ) : ( diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/Task/TaskTabIncidentManagerHeader/TaskTabIncidentManagerHeader.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/Task/TaskTabIncidentManagerHeader/TaskTabIncidentManagerHeader.component.tsx index 06d197178461..e58c79f86b31 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/Task/TaskTabIncidentManagerHeader/TaskTabIncidentManagerHeader.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/Task/TaskTabIncidentManagerHeader/TaskTabIncidentManagerHeader.component.tsx @@ -22,7 +22,7 @@ import { formatDateTime } from '../../../../utils/date-time/DateTimeUtils'; import { getEntityName } from '../../../../utils/EntityUtils'; import { useActivityFeedProvider } from '../../../ActivityFeed/ActivityFeedProvider/ActivityFeedProvider'; import { OwnerLabel } from '../../../common/OwnerLabel/OwnerLabel.component'; -import RichTextEditorPreviewer from '../../../common/RichTextEditor/RichTextEditorPreviewer'; +import RichTextEditorPreviewerV1 from '../../../common/RichTextEditor/RichTextEditorPreviewerV1'; import Severity from '../../../DataQuality/IncidentManager/Severity/Severity.component'; import './task-tab-incident-manager-header.style.less'; @@ -172,7 +172,7 @@ const TaskTabIncidentManagerHeader = ({ thread }: { thread: Thread }) => { {`${t('label.failure-comment')}: `} -
Severity.component
); } ); -jest.mock('../../../common/RichTextEditor/RichTextEditorPreviewer', () => { +jest.mock('../../../common/RichTextEditor/RichTextEditorPreviewerV1', () => { return jest .fn() .mockImplementation(() =>
RichTextEditorPreviewer.component
); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/VersionTable/VersionTable.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/VersionTable/VersionTable.component.tsx index 668987b72345..a56a440b474f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/VersionTable/VersionTable.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/VersionTable/VersionTable.component.tsx @@ -32,7 +32,7 @@ import { prepareConstraintIcon, } from '../../../utils/TableUtils'; import FilterTablePlaceHolder from '../../common/ErrorWithPlaceholder/FilterTablePlaceHolder'; -import RichTextEditorPreviewer from '../../common/RichTextEditor/RichTextEditorPreviewer'; +import RichTextEditorPreviewerV1 from '../../common/RichTextEditor/RichTextEditorPreviewerV1'; import Searchbar from '../../common/SearchBarComponent/SearchBar.component'; import TagsViewer from '../../Tag/TagsViewer/TagsViewer'; import { VersionTableProps } from './VersionTable.interfaces'; @@ -119,10 +119,10 @@ function VersionTable({
{deletedConstraintIcon} {addedConstraintIcon} - +
{!isEmpty(record.displayName) ? ( - + ) : null} ); @@ -158,13 +158,13 @@ function VersionTable({ return dataTypeDisplay ? ( }>
-
@@ -183,7 +183,7 @@ function VersionTable({ render: (description: T['description']) => description ? ( <> - + {getFrequentlyJoinedColumns( columnName, joins ?? [], diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/VersionTable/VersionTable.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/VersionTable/VersionTable.test.tsx index e5f0bdbc15c4..9d1e41ce055e 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/VersionTable/VersionTable.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/VersionTable/VersionTable.test.tsx @@ -28,7 +28,7 @@ jest.mock('../../Tag/TagsViewer/TagsViewer', () => jest.fn().mockImplementation(() =>
TagsViewer
) ); -jest.mock('../../common/RichTextEditor/RichTextEditorPreviewer', () => +jest.mock('../../common/RichTextEditor/RichTextEditorPreviewerV1', () => jest .fn() .mockImplementation(({ markdown }) => ( diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Explore/AdvanceSearchProvider/AdvanceSearchProvider.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Explore/AdvanceSearchProvider/AdvanceSearchProvider.component.tsx index 9f612e84c287..d3960a5ea82c 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Explore/AdvanceSearchProvider/AdvanceSearchProvider.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Explore/AdvanceSearchProvider/AdvanceSearchProvider.component.tsx @@ -27,16 +27,15 @@ import { JsonTree, Utils as QbUtils, ValueField, + ValueSource, } from 'react-awesome-query-builder'; import { useHistory, useParams } from 'react-router-dom'; -import { - emptyJsonTree, - TEXT_FIELD_OPERATORS, -} from '../../../constants/AdvancedSearch.constants'; +import { emptyJsonTree } from '../../../constants/AdvancedSearch.constants'; import { SearchIndex } from '../../../enums/search.enum'; import useCustomLocation from '../../../hooks/useCustomLocation/useCustomLocation'; import { TabsInfoData } from '../../../pages/ExplorePage/ExplorePage.interface'; import { getAllCustomProperties } from '../../../rest/metadataTypeAPI'; +import advancedSearchClassBase from '../../../utils/AdvancedSearchClassBase'; import { getTierOptions, getTreeConfig, @@ -225,12 +224,13 @@ export const AdvanceSearchProvider = ({ Object.entries(res).forEach(([_, fields]) => { if (Array.isArray(fields) && fields.length > 0) { - fields.forEach((field: { name: string; type: string }) => { + fields.forEach((field) => { if (field.name && field.type) { - subfields[field.name] = { - type: 'text', - valueSources: ['value'], - operators: TEXT_FIELD_OPERATORS, + const { subfieldsKey, dataObject } = + advancedSearchClassBase.getCustomPropertiesSubFields(field); + subfields[subfieldsKey] = { + ...dataObject, + valueSources: dataObject.valueSources as ValueSource[], }; } }); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Explore/AdvanceSearchProvider/AdvanceSearchProvider.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/Explore/AdvanceSearchProvider/AdvanceSearchProvider.interface.ts index 65506e5ab97d..9f7bbe6b11e6 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Explore/AdvanceSearchProvider/AdvanceSearchProvider.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/Explore/AdvanceSearchProvider/AdvanceSearchProvider.interface.ts @@ -51,3 +51,7 @@ export interface AdvanceSearchContext { } export type FilterObject = Record; +export interface CustomPropertyEnumConfig { + multiSelect: boolean; + values: string[]; +} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Explore/EntitySummaryPanel/DataProductSummary/DataProductSummary.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Explore/EntitySummaryPanel/DataProductSummary/DataProductSummary.component.tsx index 5a19a085866d..ca7c0b0e6ba0 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Explore/EntitySummaryPanel/DataProductSummary/DataProductSummary.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Explore/EntitySummaryPanel/DataProductSummary/DataProductSummary.component.tsx @@ -16,7 +16,7 @@ import { useTranslation } from 'react-i18next'; import { DataProduct } from '../../../../generated/entity/domains/dataProduct'; import { getEntityName } from '../../../../utils/EntityUtils'; import { OwnerLabel } from '../../../common/OwnerLabel/OwnerLabel.component'; -import RichTextEditorPreviewer from '../../../common/RichTextEditor/RichTextEditorPreviewer'; +import RichTextEditorPreviewerV1 from '../../../common/RichTextEditor/RichTextEditorPreviewerV1'; import SummaryPanelSkeleton from '../../../common/Skeleton/SummaryPanelSkeleton/SummaryPanelSkeleton.component'; interface DataProductSummaryProps { @@ -62,7 +62,7 @@ const DataProductSummary = ({
{entityDetails.description?.trim() ? ( - diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Explore/EntitySummaryPanel/SummaryList/SummaryListItems/SummaryListItems.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Explore/EntitySummaryPanel/SummaryList/SummaryListItems/SummaryListItems.component.tsx index 722a94324d08..88fb1b75e24a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Explore/EntitySummaryPanel/SummaryList/SummaryListItems/SummaryListItems.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Explore/EntitySummaryPanel/SummaryList/SummaryListItems/SummaryListItems.component.tsx @@ -17,7 +17,7 @@ import { useTranslation } from 'react-i18next'; import { MAX_CHAR_LIMIT_ENTITY_SUMMARY } from '../../../../../constants/constants'; import { getTagValue } from '../../../../../utils/CommonUtils'; import { prepareConstraintIcon } from '../../../../../utils/TableUtils'; -import RichTextEditorPreviewer from '../../../../common/RichTextEditor/RichTextEditorPreviewer'; +import RichTextEditorPreviewerV1 from '../../../../common/RichTextEditor/RichTextEditorPreviewerV1'; import TagsViewer from '../../../../Tag/TagsViewer/TagsViewer'; import { SummaryListItemProps } from './SummaryListItems.interface'; @@ -72,7 +72,7 @@ function SummaryListItem({
{entityDetails.description ? ( - diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Explore/EntitySummaryPanel/SummaryList/SummaryListItems/SummaryListItems.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Explore/EntitySummaryPanel/SummaryList/SummaryListItems/SummaryListItems.test.tsx index 6682a2d91135..42ee8ca96978 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Explore/EntitySummaryPanel/SummaryList/SummaryListItems/SummaryListItems.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Explore/EntitySummaryPanel/SummaryList/SummaryListItems/SummaryListItems.test.tsx @@ -21,7 +21,7 @@ import { } from '../../mocks/SummaryListItems.mock'; import SummaryListItem from './SummaryListItems.component'; -jest.mock('../../../../common/RichTextEditor/RichTextEditorPreviewer', () => +jest.mock('../../../../common/RichTextEditor/RichTextEditorPreviewerV1', () => jest .fn() .mockImplementation(() => ( diff --git a/openmetadata-ui/src/main/resources/ui/src/components/GenericProvider/GenericProvider.tsx b/openmetadata-ui/src/main/resources/ui/src/components/GenericProvider/GenericProvider.tsx deleted file mode 100644 index 44886698508d..000000000000 --- a/openmetadata-ui/src/main/resources/ui/src/components/GenericProvider/GenericProvider.tsx +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright 2024 Collate. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import { once } from 'lodash'; -import React, { useContext, useMemo } from 'react'; -import { OperationPermission } from '../../context/PermissionProvider/PermissionProvider.interface'; -import { EntityType } from '../../enums/entity.enum'; - -interface GenericProviderProps { - children?: React.ReactNode; - data: T; - type: EntityType; - onUpdate: (updatedData: T) => Promise; - isVersionView?: boolean; - permissions: OperationPermission; -} - -interface GenericContextType { - data: T; - type: EntityType; - onUpdate: (updatedData: T) => Promise; - isVersionView?: boolean; - permissions: OperationPermission; -} - -const createGenericContext = once(() => - React.createContext({} as GenericContextType) -); - -export const GenericProvider = ({ - children, - data, - type, - onUpdate, - isVersionView, - permissions, -}: GenericProviderProps) => { - const GenericContext = createGenericContext(); - - const values = useMemo( - () => ({ data, type, onUpdate, isVersionView, permissions }), - [data, type, onUpdate, isVersionView, permissions] - ); - - return ( - {children} - ); -}; - -export const useGenericContext = () => - useContext(createGenericContext()); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/AddGlossaryTermForm/AddGlossaryTermForm.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/AddGlossaryTermForm/AddGlossaryTermForm.component.tsx index 958dba68ee54..20950863841c 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/AddGlossaryTermForm/AddGlossaryTermForm.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/AddGlossaryTermForm/AddGlossaryTermForm.component.tsx @@ -201,7 +201,7 @@ const AddGlossaryTermForm = ({ type: FieldTypes.DESCRIPTION, props: { 'data-testid': 'description', - initialValue: '', + initialValue: glossaryTerm?.description, height: 'auto', }, rules: [ diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/CustomiseWidgets/CustomizeTabWidget/CustomizeTabWidget.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/CustomiseWidgets/CustomizeTabWidget/CustomizeTabWidget.tsx deleted file mode 100644 index 1047b18c2b5b..000000000000 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/CustomiseWidgets/CustomizeTabWidget/CustomizeTabWidget.tsx +++ /dev/null @@ -1,326 +0,0 @@ -/* - * Copyright 2024 Collate. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import Icon from '@ant-design/icons'; -import { Input, Modal, Tooltip } from 'antd'; -import { isEmpty, isNil, toString, uniqueId } from 'lodash'; -import React, { useCallback, useMemo, useState } from 'react'; -import RGL, { Layout, WidthProvider } from 'react-grid-layout'; -import { useTranslation } from 'react-i18next'; -import { ReactComponent as EditIcon } from '../../../../assets/svg/edit-new.svg'; -import { CommonWidgetType } from '../../../../constants/CustomizeWidgets.constants'; -import { EntityTabs } from '../../../../enums/entity.enum'; -import { Document } from '../../../../generated/entity/docStore/document'; -import { Page, Tab } from '../../../../generated/system/ui/page'; -import { PageType } from '../../../../generated/system/ui/uiCustomization'; -import { useGridLayoutDirection } from '../../../../hooks/useGridLayoutDirection'; -import { - WidgetCommonProps, - WidgetConfig, -} from '../../../../pages/CustomizablePage/CustomizablePage.interface'; -import { useCustomizeStore } from '../../../../pages/CustomizablePage/CustomizeStore'; -import customizeGlossaryTermPageClassBase from '../../../../utils/CustomiseGlossaryTermPage/CustomizeGlossaryTermPage'; -import { - getAddWidgetHandler, - getLayoutUpdateHandler, - getLayoutWithEmptyWidgetPlaceholder, - getRemoveWidgetHandler, - getUniqueFilteredLayout, -} from '../../../../utils/CustomizableLandingPageUtils'; -import { - getCustomizableWidgetByPage, - getDefaultTabs, - getDefaultWidgetForTab, -} from '../../../../utils/CustomizePage/CustomizePageUtils'; -import { getEntityName } from '../../../../utils/EntityUtils'; -import { getWidgetFromKey } from '../../../../utils/GlossaryTerm/GlossaryTermUtil'; -import { DraggableTabs } from '../../../common/DraggableTabs/DraggableTabs'; -import AddDetailsPageWidgetModal from '../../../MyData/CustomizableComponents/AddDetailsPageWidgetModal/AddDetailsPageWidgetModal'; - -const ReactGridLayout = WidthProvider(RGL); - -export type CustomizeTabWidgetProps = WidgetCommonProps; - -type TargetKey = React.MouseEvent | React.KeyboardEvent | string; - -export const CustomizeTabWidget = () => { - const { currentPage, currentPageType, updateCurrentPage } = - useCustomizeStore(); - const [items, setItems] = useState( - currentPage?.tabs ?? getDefaultTabs(currentPageType as PageType) - ); - const [activeKey, setActiveKey] = useState( - (items[0]?.id as EntityTabs) ?? null - ); - const { t } = useTranslation(); - - const [editableItem, setEditableItem] = useState(null); - const [tabLayouts, setTabLayouts] = useState( - getLayoutWithEmptyWidgetPlaceholder( - (items.find((item) => item.id === activeKey)?.layout as WidgetConfig[]) ?? - getDefaultWidgetForTab( - currentPageType as PageType, - (activeKey as EntityTabs) ?? EntityTabs.OVERVIEW - ), - 2, - 3 - ) - ); - const [isWidgetModalOpen, setIsWidgetModalOpen] = useState(false); - const [placeholderWidgetKey, setPlaceholderWidgetKey] = useState(''); - - const onChange = (tabKey: string) => { - const key = tabKey as EntityTabs; - setActiveKey(key); - const newTab = items.find((item) => item.id === key); - setTabLayouts( - getLayoutWithEmptyWidgetPlaceholder( - isEmpty(newTab?.layout) - ? getDefaultWidgetForTab(currentPageType as PageType, key) - : (newTab?.layout as WidgetConfig[]), - 2, - 3 - ) - ); - }; - - const add = () => { - const newActiveKey = uniqueId(`newTab`); - setItems((items) => [ - ...items, - { - name: 'New Tab', - layout: [], - id: newActiveKey, - editable: true, - } as Tab, - ]); - onChange(newActiveKey); - }; - - const remove = (targetKey: TargetKey) => { - let newActiveKey = activeKey; - let lastIndex = -1; - items.forEach((item, i) => { - if (item.id === targetKey) { - lastIndex = i - 1; - } - }); - const newPanes = items.filter((item) => item.id !== targetKey); - if (newPanes.length && newActiveKey === targetKey) { - if (lastIndex >= 0) { - newActiveKey = newPanes[lastIndex].id as EntityTabs; - } else { - newActiveKey = newPanes[0].id as EntityTabs; - } - } - setItems(newPanes); - onChange(newActiveKey ?? EntityTabs.OVERVIEW); - }; - - const onEdit = ( - targetKey: React.MouseEvent | React.KeyboardEvent | string, - action: 'add' | 'remove' - ) => { - if (action === 'add') { - add(); - } else { - remove(targetKey); - } - }; - - const handleTabEditClick = (key: string) => { - setEditableItem(items.find((item) => item.id === key) || null); - }; - - const handleRenameSave = () => { - if (editableItem) { - const newItems = items.map((item) => - item.id === editableItem.id ? editableItem : item - ); - setItems(newItems); - updateCurrentPage({ - ...currentPage, - tabs: newItems, - } as Page); - setEditableItem(null); - } - }; - - const handleChange: React.ChangeEventHandler = (event) => { - editableItem && - setEditableItem({ - ...editableItem, - displayName: event.target.value ?? '', - }); - }; - - const handleOpenAddWidgetModal = () => { - setIsWidgetModalOpen(true); - }; - - const handlePlaceholderWidgetKey = (value: string) => { - setPlaceholderWidgetKey(value); - }; - - const handleRemoveWidget = (widgetKey: string) => { - setTabLayouts(getRemoveWidgetHandler(widgetKey, 3, 3.5)); - }; - - const widgets = useMemo( - () => - tabLayouts.map((widget) => ( -
- {getWidgetFromKey({ - widgetConfig: widget, - handleOpenAddWidgetModal: handleOpenAddWidgetModal, - handlePlaceholderWidgetKey: handlePlaceholderWidgetKey, - handleRemoveWidget: handleRemoveWidget, - isEditView: true, - })} -
- )), - [tabLayouts] - ); - - const handleLayoutUpdate = useCallback( - (updatedLayout: Layout[]) => { - if (!isEmpty(tabLayouts) && !isEmpty(updatedLayout)) { - setTabLayouts(getLayoutUpdateHandler(updatedLayout)); - updateCurrentPage({ - ...currentPage, - tabs: items.map((item) => - item.id === activeKey - ? { ...item, layout: getUniqueFilteredLayout(updatedLayout) } - : item - ), - } as Page); - } - }, - [tabLayouts] - ); - - const handleMainPanelAddWidget = useCallback( - ( - newWidgetData: CommonWidgetType, - placeholderWidgetKey: string, - widgetSize: number - ) => { - setTabLayouts( - getAddWidgetHandler( - newWidgetData as unknown as Document, - placeholderWidgetKey, - widgetSize, - customizeGlossaryTermPageClassBase.detailPageMaxGridSize - ) - ); - setIsWidgetModalOpen(false); - }, - [] - ); - - const onTabPositionChange = (newOrder: React.Key[]) => { - const newItems = newOrder.map( - (key) => items.find((item) => item.id === key) as Tab - ); - setItems(newItems); - - updateCurrentPage({ - ...currentPage, - tabs: newItems, - } as Page); - }; - - // eslint-disable-next-line no-console - console.log('widgets', tabLayouts, currentPage); - - // call the hook to set the direction of the grid layout - useGridLayoutDirection(); - - return ( - <> - ({ - key: item.id, - label: ( - { - event.stopPropagation(); - item.editable && onChange(item.id); - }}> - - {getEntityName(item)} - { - event.stopPropagation(); - handleTabEditClick(item.id); - }} - /> - - - ), - closable: true, - }))} - size="small" - tabBarGutter={2} - type="editable-card" - onChange={onChange} - onEdit={onEdit} - onTabChange={onTabPositionChange} - /> - - - {widgets} - - - {currentPageType && ( - setIsWidgetModalOpen(false)} - maxGridSizeSupport={8} - open={isWidgetModalOpen} - placeholderWidgetKey={placeholderWidgetKey} - widgetsList={getCustomizableWidgetByPage(currentPageType)} - /> - )} - {editableItem && ( - setEditableItem(null)} - onOk={handleRenameSave}> - - - )} - - ); -}; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/CustomiseWidgets/SynonymsWidget/GenericWidget.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/CustomiseWidgets/SynonymsWidget/GenericWidget.tsx deleted file mode 100644 index 74b73ad9b076..000000000000 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/CustomiseWidgets/SynonymsWidget/GenericWidget.tsx +++ /dev/null @@ -1,411 +0,0 @@ -/* - * Copyright 2024 Collate. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import { CloseOutlined, DragOutlined } from '@ant-design/icons'; -import { Card, Space } from 'antd'; -import { noop, startCase } from 'lodash'; -import React, { useMemo } from 'react'; -import { - DetailPageWidgetKeys, - GlossaryTermDetailPageWidgetKeys, -} from '../../../../enums/CustomizeDetailPage.enum'; -import { EntityType } from '../../../../enums/entity.enum'; -import { DataType, Table } from '../../../../generated/entity/data/table'; -import { EntityReference } from '../../../../generated/tests/testCase'; -import { TagSource } from '../../../../generated/type/tagLabel'; -import { WidgetCommonProps } from '../../../../pages/CustomizablePage/CustomizablePage.interface'; -import { FrequentlyJoinedTables } from '../../../../pages/TableDetailsPageV1/FrequentlyJoinedTables/FrequentlyJoinedTables.component'; -import { renderReferenceElement } from '../../../../utils/GlossaryUtils'; -import tableClassBase from '../../../../utils/TableClassBase'; -import { getJoinsFromTableJoins } from '../../../../utils/TableUtils'; -import { ExtensionTable } from '../../../common/CustomPropertyTable/ExtensionTable'; -import { DomainLabel } from '../../../common/DomainLabel/DomainLabel.component'; -import { OwnerLabel } from '../../../common/OwnerLabel/OwnerLabel.component'; -import RichTextEditorPreviewer from '../../../common/RichTextEditor/RichTextEditorPreviewer'; -import TagButton from '../../../common/TagButton/TagButton.component'; -import SchemaTable from '../../../Database/SchemaTable/SchemaTable.component'; -import DataProductsContainer from '../../../DataProducts/DataProductsContainer/DataProductsContainer.component'; -import TagsViewer from '../../../Tag/TagsViewer/TagsViewer'; -import { DisplayType } from '../../../Tag/TagsViewer/TagsViewer.interface'; - -export const GenericWidget = (props: WidgetCommonProps) => { - const handleRemoveClick = () => { - if (props.handleRemoveWidget) { - props.handleRemoveWidget(props.widgetKey); - } - }; - - const widgetName = startCase(props.widgetKey.replace('KnowledgePanel.', '')); - - const cardContent = useMemo(() => { - if ( - props.widgetKey.startsWith(DetailPageWidgetKeys.GLOSSARY_TERMS) || - props.widgetKey.startsWith(GlossaryTermDetailPageWidgetKeys.RELATED_TERMS) - ) { - return ( - - ); - } else if ( - props.widgetKey.startsWith(GlossaryTermDetailPageWidgetKeys.SYNONYMS) - ) { - return ( - - ); - } else if ( - props.widgetKey.startsWith(DetailPageWidgetKeys.DOMAIN) || - props.widgetKey.startsWith(GlossaryTermDetailPageWidgetKeys.DOMAIN) - ) { - return ( - - ); - } else if ( - props.widgetKey.startsWith(GlossaryTermDetailPageWidgetKeys.REFERENCES) - ) { - return [ - { - name: 'Google', - endpoint: 'https://www.google.com', - }, - { - name: 'Collate', - endpoint: 'https://www.getcollate.io', - }, - ].map((term) => renderReferenceElement(term)); - } else if ( - props.widgetKey.startsWith(DetailPageWidgetKeys.TAGS) || - props.widgetKey.startsWith(GlossaryTermDetailPageWidgetKeys.TAGS) - ) { - return ( - - ); - } else if ( - props.widgetKey.startsWith(GlossaryTermDetailPageWidgetKeys.OWNER) - ) { - return ( - - ); - } else if ( - props.widgetKey.startsWith(DetailPageWidgetKeys.CUSTOM_PROPERTIES) || - props.widgetKey.startsWith( - GlossaryTermDetailPageWidgetKeys.CUSTOM_PROPERTIES - ) - ) { - return ( - - ); - } else if ( - props.widgetKey.startsWith(GlossaryTermDetailPageWidgetKeys.REVIEWER) - ) { - return ( - - ); - } else if ( - props.widgetKey.startsWith(DetailPageWidgetKeys.DESCRIPTION) || - props.widgetKey.startsWith(GlossaryTermDetailPageWidgetKeys.DESCRIPTION) - ) { - return ( - // eslint-disable-next-line max-len - - ); - } else if (props.widgetKey.startsWith(DetailPageWidgetKeys.TABLE_SCHEMA)) { - return ( - noop()} - /> - ); - } else if ( - props.widgetKey.startsWith(DetailPageWidgetKeys.FREQUENTLY_JOINED_TABLES) - ) { - return ( - - ); - } else if (props.widgetKey.startsWith(DetailPageWidgetKeys.DATA_PRODUCTS)) { - return ( - - ); - } - - return widgetName; - }, [props.widgetKey]); - - return ( - - {widgetName} - {props.isEditView && ( - - - - - )} - - }> - {cardContent} - - ); -}; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetails/GlossaryDetails.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetails/GlossaryDetails.component.tsx index 470709b66484..0ffba4c955a2 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetails/GlossaryDetails.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetails/GlossaryDetails.component.tsx @@ -11,58 +11,36 @@ * limitations under the License. */ -import { Col, Row, Tabs } from 'antd'; +import { Col, Row, Space, Tabs } from 'antd'; import classNames from 'classnames'; import { isEmpty, noop } from 'lodash'; import React, { useCallback, useEffect, useMemo, useState } from 'react'; -import RGL, { WidthProvider } from 'react-grid-layout'; import { useTranslation } from 'react-i18next'; import { useHistory, useParams } from 'react-router-dom'; -import { FQN_SEPARATOR_CHAR } from '../../../constants/char.constants'; import { getGlossaryTermDetailsPath } from '../../../constants/constants'; import { FEED_COUNT_INITIAL_DATA } from '../../../constants/entity.constants'; import { EntityField } from '../../../constants/Feeds.constants'; -import { GlossaryTermDetailPageWidgetKeys } from '../../../enums/CustomizeDetailPage.enum'; -import { EntityTabs, EntityType } from '../../../enums/entity.enum'; +import { COMMON_RESIZABLE_PANEL_CONFIG } from '../../../constants/ResizablePanel.constants'; +import { EntityType } from '../../../enums/entity.enum'; import { Glossary } from '../../../generated/entity/data/glossary'; import { ChangeDescription } from '../../../generated/entity/type'; -import { Page, PageType, Tab } from '../../../generated/system/ui/page'; -import { TagLabel } from '../../../generated/tests/testCase'; -import { TagSource } from '../../../generated/type/tagLabel'; -import { useApplicationStore } from '../../../hooks/useApplicationStore'; -import { useGridLayoutDirection } from '../../../hooks/useGridLayoutDirection'; import { FeedCounts } from '../../../interface/feed.interface'; -import { WidgetConfig } from '../../../pages/CustomizablePage/CustomizablePage.interface'; -import { useCustomizeStore } from '../../../pages/CustomizablePage/CustomizeStore'; -import { getDocumentByFQN } from '../../../rest/DocStoreAPI'; import { getFeedCounts } from '../../../utils/CommonUtils'; -import customizeGlossaryPageClassBase from '../../../utils/CustomizeGlossaryPage/CustomizeGlossaryPage'; import { getEntityName } from '../../../utils/EntityUtils'; -import { - getEntityVersionByField, - getEntityVersionTags, -} from '../../../utils/EntityVersionUtils'; -import { - getGlossaryTermDetailTabs, - getTabLabelMap, - getWidgetFromKey, -} from '../../../utils/GlossaryTerm/GlossaryTermUtil'; +import { getEntityVersionByField } from '../../../utils/EntityVersionUtils'; import { ActivityFeedTab } from '../../ActivityFeed/ActivityFeedTab/ActivityFeedTab.component'; import DescriptionV1 from '../../common/EntityDescription/DescriptionV1'; +import ResizablePanels from '../../common/ResizablePanels/ResizablePanels'; import TabsLabel from '../../common/TabsLabel/TabsLabel.component'; -import { DomainLabelV2 } from '../../DataAssets/DomainLabelV2/DomainLabelV2'; -import { OwnerLabelV2 } from '../../DataAssets/OwnerLabelV2/OwnerLabelV2'; -import { ReviewerLabelV2 } from '../../DataAssets/ReviewerLabelV2/ReviewerLabelV2'; -import { GenericProvider } from '../../GenericProvider/GenericProvider'; -import TagsContainerV2 from '../../Tag/TagsContainerV2/TagsContainerV2'; -import { DisplayType } from '../../Tag/TagsViewer/TagsViewer.interface'; +import GlossaryDetailsRightPanel from '../GlossaryDetailsRightPanel/GlossaryDetailsRightPanel.component'; import GlossaryHeader from '../GlossaryHeader/GlossaryHeader.component'; import GlossaryTermTab from '../GlossaryTermTab/GlossaryTermTab.component'; import { useGlossaryStore } from '../useGlossary.store'; import './glossary-details.less'; -import { GlossaryDetailsProps } from './GlossaryDetails.interface'; - -const ReactGridLayout = WidthProvider(RGL); +import { + GlossaryDetailsProps, + GlossaryTabs, +} from './GlossaryDetails.interface'; const GlossaryDetails = ({ permissions, @@ -79,35 +57,12 @@ const GlossaryDetails = ({ const { t } = useTranslation(); const history = useHistory(); const { activeGlossary: glossary } = useGlossaryStore(); + const { tab: activeTab } = useParams<{ tab: string }>(); const [feedCount, setFeedCount] = useState( FEED_COUNT_INITIAL_DATA ); - const { selectedPersona } = useApplicationStore(); const [isDescriptionEditable, setIsDescriptionEditable] = useState(false); - const { currentPersonaDocStore } = useCustomizeStore(); - // Since we are rendering this component for all customized tabs we need tab ID to get layout form store - const { tab: activeTab = EntityTabs.TERMS } = - useParams<{ tab: EntityTabs }>(); - const [customizedPage, setCustomizedPage] = useState(null); - - useGridLayoutDirection(); - - const layout = useMemo(() => { - if (!currentPersonaDocStore) { - return customizeGlossaryPageClassBase.getDefaultWidgetForTab(activeTab); - } - - const page = currentPersonaDocStore?.data?.pages.find( - (p: Page) => p.pageType === PageType.Glossary - ); - - if (page) { - return page.tabs.find((t: Tab) => t.id === activeTab)?.layout; - } else { - return customizeGlossaryPageClassBase.getDefaultWidgetForTab(activeTab); - } - }, [currentPersonaDocStore, activeTab]); const handleFeedCount = useCallback((data: FeedCounts) => { setFeedCount(data); @@ -139,37 +94,44 @@ const GlossaryDetails = ({ } }; - const updatedGlossary = useMemo(() => { - const updatedDescription = isVersionView - ? getEntityVersionByField( - glossary.changeDescription as ChangeDescription, - EntityField.DESCRIPTION, - glossary.description - ) - : glossary.description; + const description = useMemo( + () => + isVersionView + ? getEntityVersionByField( + glossary.changeDescription as ChangeDescription, + EntityField.DESCRIPTION, + glossary.description + ) + : glossary.description, + + [glossary, isVersionView] + ); + + const name = useMemo( + () => + isVersionView + ? getEntityVersionByField( + glossary.changeDescription as ChangeDescription, + EntityField.NAME, + glossary.name + ) + : glossary.name, + + [glossary, isVersionView] + ); - const updatedName = isVersionView - ? getEntityVersionByField( - glossary.changeDescription as ChangeDescription, - EntityField.NAME, - glossary.name - ) - : glossary.name; - const updatedDisplayName = isVersionView - ? getEntityVersionByField( - glossary.changeDescription as ChangeDescription, - EntityField.DISPLAYNAME, - glossary.displayName - ) - : glossary.displayName; + const displayName = useMemo( + () => + isVersionView + ? getEntityVersionByField( + glossary.changeDescription as ChangeDescription, + EntityField.DISPLAYNAME, + glossary.displayName + ) + : glossary.displayName, - return { - ...glossary, - description: updatedDescription, - name: updatedName, - displayName: updatedDisplayName, - }; - }, [glossary, isVersionView]); + [glossary, isVersionView] + ); const handleTabChange = (activeKey: string) => { if (activeKey !== activeTab) { @@ -179,129 +141,88 @@ const GlossaryDetails = ({ } }; - const tags = useMemo( - () => - isVersionView - ? getEntityVersionTags( - glossary, - glossary.changeDescription as ChangeDescription - ) - : glossary.tags, - [isVersionView, glossary] - ); - - const tagsWidget = useMemo(() => { + const detailsContent = useMemo(() => { return ( - - await handleGlossaryUpdate({ ...glossary, tags: updatedTags }) - } - onThreadLinkSelect={onThreadLinkSelect} - /> - ); - }, [tags, glossary, handleGlossaryUpdate, permissions, onThreadLinkSelect]); + + + + + setIsDescriptionEditable(false)} + onDescriptionEdit={() => setIsDescriptionEditable(true)} + onDescriptionUpdate={onDescriptionUpdate} + onThreadLinkSelect={onThreadLinkSelect} + /> - const widgets = useMemo(() => { - const getWidgetFromKeyInternal = (widget: WidgetConfig) => { - if (widget.i.startsWith(GlossaryTermDetailPageWidgetKeys.DESCRIPTION)) { - return ( - setIsDescriptionEditable(false)} - onDescriptionEdit={() => setIsDescriptionEditable(true)} - onDescriptionUpdate={onDescriptionUpdate} - onThreadLinkSelect={onThreadLinkSelect} - /> - ); - } else if ( - widget.i.startsWith(GlossaryTermDetailPageWidgetKeys.TERMS_TABLE) - ) { - return ( - + + + ), + ...COMMON_RESIZABLE_PANEL_CONFIG.LEFT_PANEL, + }} + secondPanel={{ + children: ( + + ), + ...COMMON_RESIZABLE_PANEL_CONFIG.RIGHT_PANEL, + className: + 'entity-resizable-right-panel-container glossary-resizable-panel-container', + }} /> - ); - } else if (widget.i.startsWith(GlossaryTermDetailPageWidgetKeys.OWNER)) { - return ; - } else if (widget.i.startsWith(GlossaryTermDetailPageWidgetKeys.DOMAIN)) { - return ; - } else if ( - widget.i.startsWith(GlossaryTermDetailPageWidgetKeys.REVIEWER) - ) { - return ; - } else if (widget.i.startsWith(GlossaryTermDetailPageWidgetKeys.TAGS)) { - return tagsWidget; - } - - return getWidgetFromKey({ - widgetConfig: widget, - handleOpenAddWidgetModal: noop, - handlePlaceholderWidgetKey: noop, - handleRemoveWidget: noop, - isEditView: false, - }); - }; - - return layout.map((widget: WidgetConfig) => ( -
- {getWidgetFromKeyInternal(widget)} -
- )); - }, [tagsWidget, layout, permissions, termsLoading]); - - const detailsContent = useMemo(() => { - return ( - - {widgets} - + + ); - }, [permissions, glossary, termsLoading, isDescriptionEditable, widgets]); + }, [ + isVersionView, + permissions, + glossary, + termsLoading, + description, + isDescriptionEditable, + ]); const tabs = useMemo(() => { - const tabLabelMap = getTabLabelMap(customizedPage?.tabs); - - const items = [ + return [ { label: ( ), - key: EntityTabs.TERMS, + key: GlossaryTabs.TERMS, children: detailsContent, }, ...(!isVersionView @@ -310,15 +231,12 @@ const GlossaryDetails = ({ label: ( ), - key: EntityTabs.ACTIVITY_FEED, + key: GlossaryTabs.ACTIVITY_FEED, children: ( { - const pageFQN = `${EntityType.PERSONA}${FQN_SEPARATOR_CHAR}${selectedPersona.fullyQualifiedName}`; - try { - const doc = await getDocumentByFQN(pageFQN); - setCustomizedPage( - doc.data?.pages?.find((p: Page) => p.pageType === PageType.Glossary) - ); - } catch (error) { - // fail silent - } - }, [selectedPersona.fullyQualifiedName]); - - useEffect(() => { - if (selectedPersona?.fullyQualifiedName) { - fetchDocument(); - } - }, [selectedPersona]); - return ( - - data={updatedGlossary} - isVersionView={isVersionView} - permissions={permissions} - type={EntityType.GLOSSARY} - onUpdate={handleGlossaryUpdate}> - - - - - - - - - + + + + + + + + ); }; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetails/GlossaryDetails.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetails/GlossaryDetails.interface.ts index 388f47023d2b..12847a912abc 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetails/GlossaryDetails.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetails/GlossaryDetails.interface.ts @@ -16,6 +16,11 @@ import { Glossary } from '../../../generated/entity/data/glossary'; import { GlossaryTerm } from '../../../generated/entity/data/glossaryTerm'; import { VotingDataProps } from '../../Entity/Voting/voting.interface'; +export enum GlossaryTabs { + TERMS = 'terms', + ACTIVITY_FEED = 'activity_feed', +} + export type GlossaryDetailsProps = { isVersionView?: boolean; permissions: OperationPermission; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetails/GlossaryDetails.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetails/GlossaryDetails.test.tsx index c27b6899d8f8..d8fd176edf8b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetails/GlossaryDetails.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetails/GlossaryDetails.test.tsx @@ -46,6 +46,13 @@ jest.mock( }) ); +jest.mock( + '../GlossaryDetailsRightPanel/GlossaryDetailsRightPanel.component', + () => { + return jest.fn().mockImplementation(() => <>testGlossaryRightPanel); + } +); + jest.mock('../../common/EntityDescription/DescriptionV1', () => jest.fn().mockImplementation(() =>
DescriptionV1
) ); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetailsRightPanel/GlossaryDetailsRightPanel.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetailsRightPanel/GlossaryDetailsRightPanel.component.tsx new file mode 100644 index 000000000000..25e64c16ae50 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetailsRightPanel/GlossaryDetailsRightPanel.component.tsx @@ -0,0 +1,310 @@ +/* + * Copyright 2023 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Button, Col, Row, Space, Tooltip, Typography } from 'antd'; +import { t } from 'i18next'; +import { cloneDeep, includes, isEqual } from 'lodash'; +import React, { useMemo } from 'react'; +import { ReactComponent as EditIcon } from '../../../assets/svg/edit-new.svg'; +import { ReactComponent as PlusIcon } from '../../../assets/svg/plus-primary.svg'; +import { UserTeamSelectableList } from '../../../components/common/UserTeamSelectableList/UserTeamSelectableList.component'; +import { DE_ACTIVE_COLOR } from '../../../constants/constants'; +import { OperationPermission } from '../../../context/PermissionProvider/PermissionProvider.interface'; +import { EntityType, TabSpecificField } from '../../../enums/entity.enum'; +import { Glossary, TagSource } from '../../../generated/entity/data/glossary'; +import { + GlossaryTerm, + TagLabel, +} from '../../../generated/entity/data/glossaryTerm'; +import { ChangeDescription } from '../../../generated/entity/type'; +import { EntityReference } from '../../../generated/type/entityReference'; +import { + getEntityVersionTags, + getOwnerVersionLabel, +} from '../../../utils/EntityVersionUtils'; +import { CustomPropertyTable } from '../../common/CustomPropertyTable/CustomPropertyTable'; +import { ExtentionEntitiesKeys } from '../../common/CustomPropertyTable/CustomPropertyTable.interface'; +import { DomainLabel } from '../../common/DomainLabel/DomainLabel.component'; +import TagButton from '../../common/TagButton/TagButton.component'; +import TagsContainerV2 from '../../Tag/TagsContainerV2/TagsContainerV2'; +import { DisplayType } from '../../Tag/TagsViewer/TagsViewer.interface'; + +type Props = { + isVersionView?: boolean; + permissions: OperationPermission; + selectedData: Glossary | GlossaryTerm; + isGlossary: boolean; + onUpdate: (data: GlossaryTerm | Glossary) => void | Promise; + onThreadLinkSelect: (value: string) => void; + entityType: EntityType; + refreshGlossaryTerms?: () => void; + editCustomAttributePermission?: boolean; + onExtensionUpdate?: (updatedTable: GlossaryTerm) => Promise; +}; + +const GlossaryDetailsRightPanel = ({ + permissions, + selectedData, + isGlossary, + onUpdate, + isVersionView, + onThreadLinkSelect, + refreshGlossaryTerms, + entityType, + editCustomAttributePermission, + onExtensionUpdate, +}: Props) => { + const hasEditReviewerAccess = useMemo(() => { + return permissions.EditAll || permissions.EditReviewers; + }, [permissions]); + + const hasViewAllPermission = useMemo(() => { + return permissions.ViewAll; + }, [permissions]); + + const { assignedReviewers, hasReviewers } = useMemo(() => { + const inheritedReviewers: EntityReference[] = []; + const assignedReviewers: EntityReference[] = []; + + selectedData.reviewers?.forEach((item) => { + if (item.inherited) { + inheritedReviewers.push(item); + } else { + assignedReviewers.push(item); + } + }); + + return { + inheritedReviewers, + assignedReviewers, + hasReviewers: selectedData.reviewers && selectedData.reviewers.length > 0, + }; + }, [selectedData.reviewers]); + + const handleTagsUpdate = async (updatedTags: TagLabel[]) => { + if (updatedTags) { + const updatedData = { + ...selectedData, + tags: updatedTags, + }; + + await onUpdate(updatedData); + } + }; + + const handleReviewerSave = async (data?: EntityReference[]) => { + const reviewers: EntityReference[] = data ?? []; + + if (!isEqual(reviewers, assignedReviewers)) { + let updatedGlossary = cloneDeep(selectedData); + const oldReviewer = reviewers.filter((d) => + includes(assignedReviewers, d) + ); + const newReviewer = reviewers + .filter((d) => !includes(assignedReviewers, d)) + .map((d) => ({ id: d.id, type: d.type })); + updatedGlossary = { + ...updatedGlossary, + reviewers: [...oldReviewer, ...newReviewer], + }; + await onUpdate(updatedGlossary); + } + }; + + const handleUpdatedOwner = async (newOwner?: EntityReference[]) => { + const updatedData = { + ...selectedData, + owners: newOwner, + }; + await onUpdate(updatedData); + refreshGlossaryTerms?.(); + }; + + const tags = useMemo( + () => + isVersionView + ? getEntityVersionTags( + selectedData, + selectedData.changeDescription as ChangeDescription + ) + : selectedData.tags, + [isVersionView, selectedData] + ); + + return ( + + + + + +
+ + {t('label.owner-plural')} + + {(permissions.EditOwners || permissions.EditAll) && + selectedData.owners && + selectedData.owners.length > 0 && ( + handleUpdatedOwner(updatedUser)}> + +
+ + {getOwnerVersionLabel( + selectedData, + isVersionView ?? false, + TabSpecificField.OWNERS, + permissions.EditOwners || permissions.EditAll + )} + + {selectedData.owners?.length === 0 && + (permissions.EditOwners || permissions.EditAll) && ( + handleUpdatedOwner(updatedUser)}> + } + label={t('label.add')} + tooltip="" + /> + + )} + + +
+ + {t('label.reviewer-plural')} + + {hasEditReviewerAccess && hasReviewers && ( + + +
+
+
+ {getOwnerVersionLabel( + selectedData, + isVersionView ?? false, + TabSpecificField.REVIEWERS, + hasEditReviewerAccess + )} +
+ + {hasEditReviewerAccess && !hasReviewers && ( + + } + label={t('label.add')} + tooltip="" + /> + + )} +
+ + {isGlossary && ( + +
+ +
+ + )} + + {!isGlossary && selectedData && ( + { + await onExtensionUpdate?.(updatedTable as GlossaryTerm); + }} + hasEditAccess={Boolean(editCustomAttributePermission)} + hasPermission={hasViewAllPermission} + maxDataCap={5} + /> + )} + + + ); +}; + +export default GlossaryDetailsRightPanel; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetailsRightPanel/GlossaryDetailsRightPanel.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetailsRightPanel/GlossaryDetailsRightPanel.test.tsx new file mode 100644 index 000000000000..adfca2cf6cd9 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetailsRightPanel/GlossaryDetailsRightPanel.test.tsx @@ -0,0 +1,90 @@ +/* + * Copyright 2023 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { render } from '@testing-library/react'; +import React from 'react'; +import { BrowserRouter } from 'react-router-dom'; +import { OperationPermission } from '../../../context/PermissionProvider/PermissionProvider.interface'; +import { EntityType } from '../../../enums/entity.enum'; +import { mockedGlossaries } from '../../../mocks/Glossary.mock'; +import GlossaryDetailsRightPanel from './GlossaryDetailsRightPanel.component'; + +const mockPermissions = { + Create: true, + Delete: true, + EditAll: true, + EditCustomFields: true, + EditDataProfile: true, + EditDescription: true, + EditDisplayName: true, + EditLineage: true, + EditOwners: true, + EditQueries: true, + EditSampleData: true, + EditTags: true, + EditTests: true, + EditTier: true, + ViewAll: true, + ViewDataProfile: true, + ViewQueries: true, + ViewSampleData: true, + ViewTests: true, + ViewUsage: true, +} as OperationPermission; + +jest.mock( + '../../../components/common/UserSelectableList/UserSelectableList.component', + () => ({ + UserSelectableList: jest + .fn() + .mockImplementation(() => <>testUserSelectableList), + }) +); +jest.mock( + '../../../components/common/UserTeamSelectableList/UserTeamSelectableList.component', + () => ({ + UserTeamSelectableList: jest + .fn() + .mockImplementation(() => <>testUserTeamSelectableList), + }) +); + +jest.mock('../../../components/common/ProfilePicture/ProfilePicture', () => { + return jest.fn().mockImplementation(() => <>testProfilePicture); +}); + +describe('GlossaryDetailsRightPanel', () => { + it('should render the GlossaryDetailsRightPanel component', () => { + const { getByTestId } = render( + + + + ); + + expect(getByTestId('glossary-right-panel-owner-link')).toHaveTextContent( + 'label.owner-plural' + ); + expect(getByTestId('glossary-reviewer-heading-name')).toHaveTextContent( + 'label.reviewer-plural' + ); + expect(getByTestId('glossary-tags-name')).toHaveTextContent( + 'label.tag-plural' + ); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryHeader/GlossaryHeader.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryHeader/GlossaryHeader.component.tsx index 872cb57f05a1..a04e6102cca6 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryHeader/GlossaryHeader.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryHeader/GlossaryHeader.component.tsx @@ -43,6 +43,7 @@ import { ResourceEntity } from '../../../context/PermissionProvider/PermissionPr import { EntityAction, EntityType } from '../../../enums/entity.enum'; import { Glossary } from '../../../generated/entity/data/glossary'; import { + EntityReference, GlossaryTerm, Status, } from '../../../generated/entity/data/glossaryTerm'; @@ -69,33 +70,31 @@ import { import { showErrorToast } from '../../../utils/ToastUtils'; import { TitleBreadcrumbProps } from '../../common/TitleBreadcrumb/TitleBreadcrumb.interface'; import Voting from '../../Entity/Voting/Voting.component'; -import { useGenericContext } from '../../GenericProvider/GenericProvider'; import ChangeParentHierarchy from '../../Modals/ChangeParentHierarchy/ChangeParentHierarchy.component'; +import { EntityName } from '../../Modals/EntityNameModal/EntityNameModal.interface'; import StyleModal from '../../Modals/StyleModal/StyleModal.component'; import { GlossaryStatusBadge } from '../GlossaryStatusBadge/GlossaryStatusBadge.component'; import { GlossaryHeaderProps } from './GlossaryHeader.interface'; const GlossaryHeader = ({ + selectedData, + permissions, + onUpdate, onDelete, + isGlossary, onAssetAdd, onAddGlossaryTerm, updateVote, + isVersionView, }: GlossaryHeaderProps) => { const { t } = useTranslation(); const history = useHistory(); - const { fqn } = useFqn(); const { currentUser } = useApplicationStore(); - const { - onUpdate, - data: selectedData, - isVersionView, - permissions, - type: entityType, - } = useGenericContext(); const { version } = useParams<{ version: string; }>(); + const { fqn } = useFqn(); const { id } = useParams<{ id: string }>(); const { showModal } = useEntityExportModalProvider(); const [breadcrumb, setBreadcrumb] = useState< @@ -110,7 +109,6 @@ const GlossaryHeader = ({ const [isStyleEditing, setIsStyleEditing] = useState(false); const [openChangeParentHierarchyModal, setOpenChangeParentHierarchyModal] = useState(false); - const isGlossary = entityType === EntityType.GLOSSARY; const { permissions: globalPermissions } = usePermissionProvider(); const createGlossaryTermPermission = useMemo( @@ -154,7 +152,7 @@ const GlossaryHeader = ({ const glossaryTermStatus: Status | null = useMemo(() => { if (!isGlossary) { - return selectedData.status ?? Status.Approved; + return (selectedData as GlossaryTerm).status ?? Status.Approved; } return null; @@ -182,13 +180,13 @@ const GlossaryHeader = ({ ); } - if (selectedData.style?.iconURL) { + if ((selectedData as GlossaryTerm).style?.iconURL) { return ( ); @@ -206,7 +204,7 @@ const GlossaryHeader = ({ }, [selectedData, isGlossary]); const handleAddGlossaryTermClick = useCallback(() => { - onAddGlossaryTerm(!isGlossary ? selectedData : undefined); + onAddGlossaryTerm(!isGlossary ? (selectedData as GlossaryTerm) : undefined); }, [fqn]); const handleGlossaryImport = () => @@ -242,7 +240,7 @@ const GlossaryHeader = ({ setIsDelete(false); }; - const onNameSave = async (obj: { name: string; displayName?: string }) => { + const onNameSave = async (obj: EntityName) => { const { name, displayName } = obj; let updatedDetails = cloneDeep(selectedData); @@ -459,7 +457,8 @@ const GlossaryHeader = ({ const statusBadge = useMemo(() => { if (!isGlossary) { - const entityStatus = selectedData.status ?? Status.Approved; + const entityStatus = + (selectedData as GlossaryTerm).status ?? Status.Approved; return ; } @@ -564,7 +563,11 @@ const GlossaryHeader = ({ entityType={EntityType.GLOSSARY_TERM} icon={icon} serviceName="" - titleColor={isGlossary ? undefined : selectedData.style?.color} + titleColor={ + isGlossary + ? undefined + : (selectedData as GlossaryTerm).style?.color + } /> @@ -652,9 +655,9 @@ const GlossaryHeader = ({ /> )} - + setIsStyleEditing(false)} onSubmit={onStyleSave} /> {openChangeParentHierarchyModal && ( setOpenChangeParentHierarchyModal(false)} onSubmit={onChangeParentSave} /> diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryHeader/GlossaryHeader.interface.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryHeader/GlossaryHeader.interface.tsx index 6be1ace7e743..5681879294b9 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryHeader/GlossaryHeader.interface.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryHeader/GlossaryHeader.interface.tsx @@ -10,11 +10,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import { OperationPermission } from '../../../context/PermissionProvider/PermissionProvider.interface'; +import { Glossary } from '../../../generated/entity/data/glossary'; import { GlossaryTerm } from '../../../generated/entity/data/glossaryTerm'; import { VotingDataProps } from '../../Entity/Voting/voting.interface'; export interface GlossaryHeaderProps { + isVersionView?: boolean; supportAddOwner?: boolean; + permissions: OperationPermission; + selectedData: Glossary | GlossaryTerm; + isGlossary: boolean; + onUpdate: (data: GlossaryTerm | Glossary) => Promise; onDelete: (id: string) => Promise; onAssetAdd?: () => void; updateVote?: (data: VotingDataProps) => Promise; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryHeader/GlossaryHeader.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryHeader/GlossaryHeader.test.tsx index f32c50462e0b..f3b7a3a683f7 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryHeader/GlossaryHeader.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryHeader/GlossaryHeader.test.tsx @@ -12,7 +12,6 @@ */ import { act, fireEvent, render, screen } from '@testing-library/react'; import React from 'react'; -import { EntityType } from '../../../enums/entity.enum'; import { Glossary } from '../../../generated/entity/data/glossary'; import { mockedGlossaryTerms, @@ -24,7 +23,7 @@ import { DEFAULT_ENTITY_PERMISSION } from '../../../utils/PermissionsUtils'; import { QueryVoteType } from '../../Database/TableQueries/TableQueries.interface'; import GlossaryHeader from './GlossaryHeader.component'; -const mockGlossaryTermPermission = { +const glossaryTermPermission = { All: true, Create: true, Delete: true, @@ -38,7 +37,7 @@ const mockGlossaryTermPermission = { jest.mock('../../../context/PermissionProvider/PermissionProvider', () => ({ usePermissionProvider: jest.fn().mockImplementation(() => ({ permissions: { - glossaryTerm: mockGlossaryTermPermission, + glossaryTerm: glossaryTermPermission, }, })), })); @@ -168,28 +167,21 @@ jest.mock('../../../rest/glossaryAPI', () => ({ patchGlossaryTerm: jest.fn().mockImplementation(() => Promise.resolve()), })); +const mockOnUpdate = jest.fn(); const mockOnDelete = jest.fn(); const mockOnUpdateVote = jest.fn(); -const mockContext = { - data: { displayName: 'glossaryTest' } as Glossary, - onUpdate: jest.fn(), - isVersionView: false, - type: EntityType.GLOSSARY, - permissions: DEFAULT_ENTITY_PERMISSION, -}; - -jest.mock('../../GenericProvider/GenericProvider', () => ({ - useGenericContext: jest.fn().mockImplementation(() => mockContext), -})); - describe('GlossaryHeader component', () => { it('should render name of Glossary', () => { render( ); @@ -199,9 +191,13 @@ describe('GlossaryHeader component', () => { it('should render import and export dropdown menu items only for glossary', async () => { render( ); @@ -219,13 +215,17 @@ describe('GlossaryHeader component', () => { }); it('should not render import and export dropdown menu items if no permission', async () => { - mockGlossaryTermPermission.All = false; - mockGlossaryTermPermission.EditAll = false; + glossaryTermPermission.All = false; + glossaryTermPermission.EditAll = false; render( ); @@ -233,15 +233,15 @@ describe('GlossaryHeader component', () => { }); it('should render changeParentHierarchy and style dropdown menu items only for glossaryTerm', async () => { - mockContext.type = EntityType.GLOSSARY_TERM; - mockContext.permissions = { ...DEFAULT_ENTITY_PERMISSION, EditAll: true }; - mockGlossaryTermPermission.All = true; - mockGlossaryTermPermission.EditAll = true; render( ); @@ -259,9 +259,13 @@ describe('GlossaryHeader component', () => { it('should not render ChangeParentHierarchy component when it is close', () => { render( ); @@ -273,9 +277,13 @@ describe('GlossaryHeader component', () => { it('should render ChangeParentHierarchy component after clicking dropdown menu item', async () => { render( ); @@ -299,9 +307,13 @@ describe('GlossaryHeader component', () => { it('should not render ChangeParentHierarchy component after onCancel call', async () => { render( ); @@ -329,9 +341,13 @@ describe('GlossaryHeader component', () => { it('should call onSubmit of ChangeParentHierarchy Component along with the patch API', async () => { render( ); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryHeader/GlossaryHeaderWidget.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryHeader/GlossaryHeaderWidget.tsx deleted file mode 100644 index aad1cdb8ed67..000000000000 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryHeader/GlossaryHeaderWidget.tsx +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright 2024 Collate. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import React, { useMemo } from 'react'; -import { ReactComponent as IconTerm } from '../../../assets/svg/book.svg'; -import { ReactComponent as GlossaryIcon } from '../../../assets/svg/glossary.svg'; -import { DE_ACTIVE_COLOR } from '../../../constants/constants'; -import { EntityType } from '../../../enums/entity.enum'; -import { EntityHeader } from '../../Entity/EntityHeader/EntityHeader.component'; - -export const GlossaryHeaderWidget = ({ - isGlossary = true, -}: { - isGlossary?: boolean; -}) => { - const icon = useMemo(() => { - if (isGlossary) { - return ( - - ); - } - - return ( - - ); - }, [isGlossary]); - - return ( -
- -
- ); -}; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTermTab/GlossaryTermTab.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTermTab/GlossaryTermTab.component.tsx index 1e69fa0736fb..2db0161d7f2e 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTermTab/GlossaryTermTab.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTermTab/GlossaryTermTab.component.tsx @@ -33,7 +33,13 @@ import { AxiosError } from 'axios'; import classNames from 'classnames'; import { compare } from 'fast-json-patch'; import { cloneDeep, isEmpty, isUndefined } from 'lodash'; -import React, { useCallback, useMemo, useState } from 'react'; +import React, { + useCallback, + useEffect, + useMemo, + useRef, + useState, +} from 'react'; import { DndProvider } from 'react-dnd'; import { HTML5Backend } from 'react-dnd-html5-backend'; import { useTranslation } from 'react-i18next'; @@ -47,7 +53,7 @@ import { ReactComponent as UpDownArrowIcon } from '../../../assets/svg/ic-up-dow import { ReactComponent as PlusOutlinedIcon } from '../../../assets/svg/plus-outlined.svg'; import ErrorPlaceHolder from '../../../components/common/ErrorWithPlaceholder/ErrorPlaceHolder'; import { OwnerLabel } from '../../../components/common/OwnerLabel/OwnerLabel.component'; -import RichTextEditorPreviewer from '../../../components/common/RichTextEditor/RichTextEditorPreviewer'; +import RichTextEditorPreviewerV1 from '../../../components/common/RichTextEditor/RichTextEditorPreviewerV1'; import StatusBadge from '../../../components/common/StatusBadge/StatusBadge.component'; import { API_RES_MAX_SIZE, @@ -76,7 +82,8 @@ import Fqn from '../../../utils/Fqn'; import { buildTree, findExpandableKeysForArray, - findGlossaryTermByFqn, + findItemByFqn, + glossaryTermTableColumnsWidth, StatusClass, } from '../../../utils/GlossaryUtils'; import { getGlossaryPath } from '../../../utils/RouterUtils'; @@ -102,11 +109,20 @@ const GlossaryTermTab = ({ onEditGlossaryTerm, className, }: GlossaryTermTabProps) => { + const tableRef = useRef(null); + const [tableWidth, setTableWidth] = useState(0); const { activeGlossary, glossaryChildTerms, setGlossaryChildTerms } = useGlossaryStore(); const { t } = useTranslation(); - const glossaryTerms = (glossaryChildTerms as ModifiedGlossaryTerm[]) ?? []; + const { glossaryTerms, expandableKeys } = useMemo(() => { + const terms = (glossaryChildTerms as ModifiedGlossaryTerm[]) ?? []; + + return { + expandableKeys: findExpandableKeysForArray(terms), + glossaryTerms: terms, + }; + }, [glossaryChildTerms]); const [movedGlossaryTerm, setMovedGlossaryTerm] = useState(); @@ -140,9 +156,10 @@ const GlossaryTermTab = ({ return null; }, [isGlossary, activeGlossary]); - const expandableKeys = useMemo(() => { - return findExpandableKeysForArray(glossaryTerms); - }, [glossaryTerms]); + const tableColumnsWidth = useMemo( + () => glossaryTermTableColumnsWidth(tableWidth, permissions.Create), + [permissions.Create, tableWidth] + ); const columns = useMemo(() => { const data: ColumnsType = [ @@ -152,7 +169,7 @@ const GlossaryTermTab = ({ key: 'name', className: 'glossary-name-column', ellipsis: true, - width: '40%', + width: tableColumnsWidth.name, render: (_, record) => { const name = getEntityName(record); @@ -181,10 +198,10 @@ const GlossaryTermTab = ({ title: t('label.description'), dataIndex: 'description', key: 'description', - width: permissions.Create ? '21%' : '33%', + width: tableColumnsWidth.description, render: (description: string) => description.trim() ? ( - ( { return isEmpty(synonyms) ? (
{NO_DATA_PLACEHOLDER}
@@ -232,14 +249,18 @@ const GlossaryTermTab = ({ title: t('label.owner-plural'), dataIndex: 'owners', key: 'owners', - width: '17%', + width: tableColumnsWidth.owners, render: (owners: EntityReference[]) => , }, { title: t('label.status'), dataIndex: 'status', key: 'status', - width: '12%', + // this check is added to the width, since the last column is optional and to maintain + // the re-sizing of the column should not be affected the others columns width sizes. + ...(permissions.Create && { + width: tableColumnsWidth.status, + }), render: (_, record) => { const status = record.status ?? Status.Approved; @@ -258,7 +279,6 @@ const GlossaryTermTab = ({ data.push({ title: t('label.action-plural'), key: 'new-term', - width: '10%', render: (_, record) => { const status = record.status ?? Status.Approved; const allowAddTerm = status === Status.Approved; @@ -305,7 +325,7 @@ const GlossaryTermTab = ({ } return data; - }, [permissions]); + }, [permissions, tableColumnsWidth]); const defaultCheckedList = useMemo( () => @@ -652,10 +672,7 @@ const GlossaryTermTab = ({ ); const terms = cloneDeep(glossaryTerms) ?? []; - const item = findGlossaryTermByFqn( - terms, - record.fullyQualifiedName ?? '' - ); + const item = findItemByFqn(terms, record.fullyQualifiedName ?? ''); (item as ModifiedGlossary).children = data; @@ -790,6 +807,12 @@ const GlossaryTermTab = ({ return expandedRowKeys.length === expandableKeys.length; }, [expandedRowKeys, expandableKeys]); + useEffect(() => { + if (tableRef.current) { + setTableWidth(tableRef.current.offsetWidth); + } + }, []); + if (termsLoading) { return ; } @@ -812,6 +835,10 @@ const GlossaryTermTab = ({ ); } + const filteredGlossaryTerms = glossaryTerms.filter((term) => + selectedStatus.includes(term.status as string) + ); + return ( @@ -884,21 +911,20 @@ const GlossaryTermTab = ({
!col.hidden)} components={TABLE_CONSTANTS} data-testid="glossary-terms-table" - dataSource={glossaryTerms.filter((term) => - selectedStatus.includes(term.status as string) - )} + dataSource={filteredGlossaryTerms} expandable={expandableConfig} loading={isTableLoading} pagination={false} + ref={tableRef} rowKey="fullyQualifiedName" size="small" - tableLayout="fixed" onHeaderRow={onTableHeader} onRow={onTableRow} /> diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTermTab/GlossaryTermTab.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTermTab/GlossaryTermTab.test.tsx index 3fdc791128f0..4a483fd1d8c3 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTermTab/GlossaryTermTab.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTermTab/GlossaryTermTab.test.tsx @@ -49,7 +49,7 @@ jest.mock('../../../rest/glossaryAPI', () => ({ .mockImplementation(() => Promise.resolve({ data: mockedGlossaryTerms })), patchGlossaryTerm: jest.fn().mockImplementation(() => Promise.resolve()), })); -jest.mock('../../common/RichTextEditor/RichTextEditorPreviewer', () => +jest.mock('../../common/RichTextEditor/RichTextEditorPreviewerV1', () => jest .fn() .mockImplementation(({ markdown }) => ( diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/GlossaryTermsV1.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/GlossaryTermsV1.component.tsx index 5d31387fcb2f..fddce22633e6 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/GlossaryTermsV1.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/GlossaryTermsV1.component.tsx @@ -22,33 +22,23 @@ import React, { useState, } from 'react'; import { useHistory, useParams } from 'react-router-dom'; -import { FQN_SEPARATOR_CHAR } from '../../../constants/char.constants'; import { getGlossaryTermDetailsPath } from '../../../constants/constants'; import { FEED_COUNT_INITIAL_DATA } from '../../../constants/entity.constants'; import { EntityField } from '../../../constants/Feeds.constants'; import { EntityTabs, EntityType } from '../../../enums/entity.enum'; import { SearchIndex } from '../../../enums/search.enum'; -import { - ChangeDescription, - Glossary, -} from '../../../generated/entity/data/glossary'; +import { Glossary } from '../../../generated/entity/data/glossary'; import { GlossaryTerm, Status, } from '../../../generated/entity/data/glossaryTerm'; -import { Page, PageType } from '../../../generated/system/ui/page'; -import { useApplicationStore } from '../../../hooks/useApplicationStore'; +import { ChangeDescription } from '../../../generated/entity/type'; import { useFqn } from '../../../hooks/useFqn'; import { FeedCounts } from '../../../interface/feed.interface'; import { MOCK_GLOSSARY_NO_PERMISSIONS } from '../../../mocks/Glossary.mock'; -import { getDocumentByFQN } from '../../../rest/DocStoreAPI'; import { searchData } from '../../../rest/miscAPI'; import { getCountBadge, getFeedCounts } from '../../../utils/CommonUtils'; import { getEntityVersionByField } from '../../../utils/EntityVersionUtils'; -import { - getGlossaryTermDetailTabs, - getTabLabelMap, -} from '../../../utils/GlossaryTerm/GlossaryTermUtil'; import { getQueryFilterToExcludeTerm } from '../../../utils/GlossaryUtils'; import { getGlossaryTermsVersionsPath } from '../../../utils/RouterUtils'; import { @@ -59,7 +49,7 @@ import { ActivityFeedTab } from '../../ActivityFeed/ActivityFeedTab/ActivityFeed import { CustomPropertyTable } from '../../common/CustomPropertyTable/CustomPropertyTable'; import TabsLabel from '../../common/TabsLabel/TabsLabel.component'; import { AssetSelectionModal } from '../../DataAssets/AssetsSelectionModal/AssetSelectionModal'; -import { GenericProvider } from '../../GenericProvider/GenericProvider'; +import { GlossaryTabs } from '../GlossaryDetails/GlossaryDetails.interface'; import GlossaryHeader from '../GlossaryHeader/GlossaryHeader.component'; import GlossaryTermTab from '../GlossaryTermTab/GlossaryTermTab.component'; import { useGlossaryStore } from '../useGlossary.store'; @@ -92,11 +82,9 @@ const GlossaryTermsV1 = ({ const [feedCount, setFeedCount] = useState( FEED_COUNT_INITIAL_DATA ); - const { selectedPersona } = useApplicationStore(); const [assetCount, setAssetCount] = useState(0); const { glossaryChildTerms } = useGlossaryStore(); const childGlossaryTerms = glossaryChildTerms ?? []; - const [customizedPage, setCustomizedPage] = useState(null); const assetPermissions = useMemo(() => { const glossaryTermStatus = glossaryTerm.status ?? Status.Approved; @@ -107,7 +95,7 @@ const GlossaryTermsV1 = ({ }, [glossaryTerm, permissions]); const activeTab = useMemo(() => { - return tab ?? EntityTabs.OVERVIEW; + return tab ?? 'overview'; }, [tab]); const activeTabHandler = (tab: string) => { @@ -179,24 +167,23 @@ const GlossaryTermsV1 = ({ }; const tabItems = useMemo(() => { - const tabLabelMap = getTabLabelMap(customizedPage?.tabs); - const items = [ { - label: ( -
- {tabLabelMap[EntityTabs.OVERVIEW] ?? t('label.overview')} -
- ), - key: EntityTabs.OVERVIEW, + label:
{t('label.overview')}
, + key: 'overview', children: ( ), }, @@ -205,8 +192,7 @@ const GlossaryTermsV1 = ({ { label: (
- {tabLabelMap[EntityTabs.GLOSSARY_TERMS] ?? - t('label.glossary-term-plural')} + {t('label.glossary-term-plural')} {getCountBadge( childGlossaryTerms.length, @@ -216,7 +202,7 @@ const GlossaryTermsV1 = ({
), - key: EntityTabs.GLOSSARY_TERMS, + key: 'terms', children: ( - {tabLabelMap[EntityTabs.ASSETS] ?? t('label.asset-plural')} + {t('label.asset-plural')} {getCountBadge(assetCount ?? 0, '', activeTab === 'assets')} ), - key: EntityTabs.ASSETS, + key: 'assets', children: ( ), - key: EntityTabs.ACTIVITY_FEED, + key: GlossaryTabs.ACTIVITY_FEED, children: ( ), key: EntityTabs.CUSTOM_PROPERTIES, @@ -307,9 +287,8 @@ const GlossaryTermsV1 = ({ : []), ]; - return getGlossaryTermDetailTabs(items, customizedPage?.tabs); + return items; }, [ - customizedPage?.tabs, glossaryTerm, permissions, termsLoading, @@ -332,62 +311,46 @@ const GlossaryTermsV1 = ({ getEntityFeedCount(); }, [glossaryFqn]); - const fetchDocument = useCallback(async () => { - const pageFQN = `${EntityType.PERSONA}${FQN_SEPARATOR_CHAR}${selectedPersona.fullyQualifiedName}`; - try { - const doc = await getDocumentByFQN(pageFQN); - setCustomizedPage( - doc.data?.pages?.find((p: Page) => p.pageType === PageType.GlossaryTerm) - ); - } catch (error) { - // fail silent - } - }, [selectedPersona.fullyQualifiedName]); - - useEffect(() => { - if (selectedPersona?.fullyQualifiedName) { - fetchDocument(); - } - }, [selectedPersona]); + const name = useMemo( + () => + isVersionView + ? getEntityVersionByField( + glossaryTerm.changeDescription as ChangeDescription, + EntityField.NAME, + glossaryTerm.name + ) + : glossaryTerm.name, - const updatedGlossaryTerm = useMemo(() => { - const name = isVersionView - ? getEntityVersionByField( - glossaryTerm.changeDescription as ChangeDescription, - EntityField.NAME, - glossaryTerm.name - ) - : glossaryTerm.name; + [glossaryTerm, isVersionView] + ); - const displayName = isVersionView - ? getEntityVersionByField( - glossaryTerm.changeDescription as ChangeDescription, - EntityField.DISPLAYNAME, - glossaryTerm.displayName - ) - : glossaryTerm.displayName; + const displayName = useMemo( + () => + isVersionView + ? getEntityVersionByField( + glossaryTerm.changeDescription as ChangeDescription, + EntityField.DISPLAYNAME, + glossaryTerm.displayName + ) + : glossaryTerm.displayName, - return { - ...glossaryTerm, - name, - displayName, - }; - }, [glossaryTerm, isVersionView]); + [glossaryTerm, isVersionView] + ); return ( - + <> setAssetModalVisible(true)} onDelete={handleGlossaryTermDelete} + onUpdate={onTermUpdate} /> @@ -413,7 +376,7 @@ const GlossaryTermsV1 = ({ onSave={handleAssetSave} /> )} - + ); }; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/GlossaryTermsV1.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/GlossaryTermsV1.test.tsx index d4be445a1349..f75af90044ab 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/GlossaryTermsV1.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/GlossaryTermsV1.test.tsx @@ -11,7 +11,7 @@ * limitations under the License. */ -import { render, screen } from '@testing-library/react'; +import { act, render, screen } from '@testing-library/react'; import React from 'react'; import { OperationPermission } from '../../../context/PermissionProvider/PermissionProvider.interface'; import { @@ -44,6 +44,15 @@ jest.mock('../../../rest/miscAPI', () => ({ .mockImplementation(() => Promise.resolve(MOCK_ASSETS_DATA)), })); +jest.mock('./tabs/RelatedTerms', () => + jest.fn().mockReturnValue(
RelatedTermsComponent
) +); +jest.mock('./tabs/GlossaryTermSynonyms', () => + jest.fn().mockReturnValue(
GlossaryTermSynonymsComponent
) +); +jest.mock('./tabs/GlossaryTermReferences', () => + jest.fn().mockReturnValue(
GlossaryTermReferencesComponent
) +); jest.mock('./tabs/AssetsTabs.component', () => jest.fn().mockReturnValue(
AssetsTabs
) ); @@ -53,9 +62,6 @@ jest.mock('../GlossaryTermTab/GlossaryTermTab.component', () => jest.mock('../GlossaryHeader/GlossaryHeader.component', () => jest.fn().mockReturnValue(
GlossaryHeader.component
) ); -jest.mock('./tabs/GlossaryOverviewTab.component', () => - jest.fn().mockReturnValue(
GlossaryOverviewTab.component
) -); const mockProps = { isSummaryPanelOpen: false, @@ -80,15 +86,13 @@ const mockProps = { onThreadLinkSelect: jest.fn(), }; -jest.mock('../../../utils/GlossaryTerm/GlossaryTermUtil', () => ({ - getGlossaryTermDetailTabs: jest.fn().mockImplementation((itemes) => itemes), - getTabLabelMap: jest.fn().mockReturnValue({}), -})); - -describe.skip('Test Glossary-term component', () => { +describe('Test Glossary-term component', () => { it('Should render Glossary-term component', async () => { render(); + act(() => { + jest.runAllTimers(); + }); const glossaryTerm = screen.getByTestId('glossary-term'); const tabs = await screen.findAllByRole('tab'); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/AssetsTabs.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/AssetsTabs.component.tsx index 579be93c4fd4..9ea136af49ed 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/AssetsTabs.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/AssetsTabs.component.tsx @@ -43,10 +43,7 @@ import { ReactComponent as AddPlaceHolderIcon } from '../../../../assets/svg/add import { ReactComponent as DeleteIcon } from '../../../../assets/svg/ic-delete.svg'; import { ReactComponent as FilterIcon } from '../../../../assets/svg/ic-feeds-filter.svg'; import { ReactComponent as IconDropdown } from '../../../../assets/svg/menu.svg'; -import { - AssetsFilterOptions, - ASSET_MENU_KEYS, -} from '../../../../constants/Assets.constants'; +import { ASSET_MENU_KEYS } from '../../../../constants/Assets.constants'; import { ES_UPDATE_DELAY } from '../../../../constants/constants'; import { GLOSSARIES_DOCS } from '../../../../constants/docs.constants'; import { ERROR_PLACEHOLDER_TYPE } from '../../../../enums/common.enum'; @@ -60,6 +57,7 @@ import { usePaging } from '../../../../hooks/paging/usePaging'; import { useApplicationStore } from '../../../../hooks/useApplicationStore'; import { useFqn } from '../../../../hooks/useFqn'; import { Aggregations } from '../../../../interface/search.interface'; +import { QueryFilterInterface } from '../../../../pages/ExplorePage/ExplorePage.interface'; import { getDataProductByName, removeAssetsFromDataProduct, @@ -81,6 +79,7 @@ import { getEntityName, getEntityReferenceFromEntity, } from '../../../../utils/EntityUtils'; +import { getCombinedQueryFilterObject } from '../../../../utils/ExplorePage/ExplorePageUtils'; import { getAggregations, getQuickFilterQuery, @@ -130,17 +129,14 @@ const AssetsTabs = forwardRef( ref ) => { const { theme } = useApplicationStore(); - const [itemCount, setItemCount] = useState>( - {} as Record - ); const [assetRemoving, setAssetRemoving] = useState(false); - const [activeFilter, _] = useState([]); const { fqn } = useFqn(); const [isLoading, setIsLoading] = useState(true); const [data, setData] = useState([]); const [quickFilterQuery, setQuickFilterQuery] = - useState>(); + useState(); + const { currentPage, pageSize, @@ -165,7 +161,6 @@ const AssetsTabs = forwardRef( const [selectedCard, setSelectedCard] = useState(); const [visible, setVisible] = useState(false); const [openKeys, setOpenKeys] = useState([]); - const [isCountLoading, setIsCountLoading] = useState(true); const [showDeleteModal, setShowDeleteModal] = useState(false); const [assetToDelete, setAssetToDelete] = useState(); const [activeEntity, setActiveEntity] = useState< @@ -201,8 +196,7 @@ const AssetsTabs = forwardRef( const encodedFqn = getEncodedFqn(escapeESReservedCharacters(entityFqn)); switch (type) { case AssetsOfEntity.DOMAIN: - return `(domain.fullyQualifiedName:"${encodedFqn}") AND !(entityType:"dataProduct")`; - + return ''; case AssetsOfEntity.DATA_PRODUCT: return `(dataProducts.fullyQualifiedName:"${encodedFqn}")`; @@ -224,9 +218,11 @@ const AssetsTabs = forwardRef( async ({ index = activeFilter, page = currentPage, + queryFilter, }: { index?: SearchIndex[]; page?: number; + queryFilter?: Record; }) => { try { setIsLoading(true); @@ -235,23 +231,10 @@ const AssetsTabs = forwardRef( pageSize: pageSize, searchIndex: index, query: `*${searchValue}*`, - filters: queryParam, - queryFilter: quickFilterQuery, + filters: queryParam as string, + queryFilter: queryFilter, }); const hits = res.hits.hits as SearchedDataProps['data']; - const totalCount = res?.hits?.total.value ?? 0; - - // Find EntityType for selected searchIndex - const entityType = AssetsFilterOptions.find((f) => - activeFilter.includes(f.value) - )?.label; - - entityType && - setItemCount((prevCount) => ({ - ...prevCount, - [entityType]: totalCount, - })); - handlePagingChange({ total: res.hits.total.value ?? 0 }); setData(hits); setAggregations(getAggregations(res?.aggregations)); @@ -262,14 +245,7 @@ const AssetsTabs = forwardRef( setIsLoading(false); } }, - [ - activeFilter, - currentPage, - pageSize, - searchValue, - queryParam, - quickFilterQuery, - ] + [activeFilter, currentPage, pageSize, searchValue, queryParam] ); const hideNotification = () => { @@ -362,34 +338,6 @@ const AssetsTabs = forwardRef( }); }; - const fetchCountsByEntity = async () => { - try { - setIsCountLoading(true); - - const res = await searchQuery({ - query: `*${searchValue}*`, - pageNumber: 0, - pageSize: 0, - queryFilter: quickFilterQuery, - searchIndex: SearchIndex.ALL, - filters: queryParam, - }); - - const buckets = res.aggregations[`index_count`].buckets; - const counts: Record = {}; - buckets.forEach((item) => { - if (item) { - counts[item.key ?? ''] = item.doc_count; - } - }); - setItemCount(counts as Record); - } catch (err) { - showErrorToast(err as AxiosError); - } finally { - setIsCountLoading(false); - } - }; - const onAssetRemove = useCallback( async (assetsData: SourceType[]) => { if (!activeEntity) { @@ -465,8 +413,6 @@ const AssetsTabs = forwardRef( }, [selectedItems]); useEffect(() => { - fetchCountsByEntity(); - return () => { onAssetClick?.(undefined); hideNotification(); @@ -715,7 +661,6 @@ const AssetsTabs = forwardRef( openKeys, visible, currentPage, - itemCount, onOpenChange, handleAssetButtonVisibleChange, onSelectAll, @@ -748,11 +693,24 @@ const AssetsTabs = forwardRef( ]); useEffect(() => { + const newFilter = getCombinedQueryFilterObject( + queryFilter as unknown as QueryFilterInterface, + quickFilterQuery as QueryFilterInterface + ); + fetchAssets({ index: isEmpty(activeFilter) ? [SearchIndex.ALL] : activeFilter, page: currentPage, + queryFilter: newFilter, }); - }, [activeFilter, currentPage, pageSize, searchValue, quickFilterQuery]); + }, [ + activeFilter, + currentPage, + pageSize, + searchValue, + queryFilter, + quickFilterQuery, + ]); useEffect(() => { const dropdownItems = getAssetsPageQuickFilters(type); @@ -794,14 +752,20 @@ const AssetsTabs = forwardRef( refreshAssets() { // Reset page to one and trigger fetchAssets handlePageChange(1); + + const newFilter = getCombinedQueryFilterObject( + queryFilter as unknown as QueryFilterInterface, + quickFilterQuery as QueryFilterInterface + ); + // If current page is already 1 it won't trigger fetchAset from useEffect // Hence need to manually trigger it for this case currentPage === 1 && fetchAssets({ index: isEmpty(activeFilter) ? [SearchIndex.ALL] : activeFilter, page: 1, + queryFilter: newFilter, }); - fetchCountsByEntity(); }, closeSummaryPanel() { setSelectedCard(undefined); @@ -877,7 +841,7 @@ const AssetsTabs = forwardRef( )} - {isLoading || isCountLoading ? ( + {isLoading ? ( @@ -908,7 +872,7 @@ const AssetsTabs = forwardRef( } /> - {!(isLoading || isCountLoading) && ( + {!isLoading && (
0, diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/AssetsTabs.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/AssetsTabs.interface.ts index 41c39355603a..9f458a943a2f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/AssetsTabs.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/AssetsTabs.interface.ts @@ -36,7 +36,7 @@ export interface AssetsTabsProps { isSummaryPanelOpen: boolean; isEntityDeleted?: boolean; type?: AssetsOfEntity; - queryFilter?: string; + queryFilter?: string | Record; noDataPlaceholder?: string | AssetNoDataPlaceholderProps; } diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/GlossaryOverviewTab.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/GlossaryOverviewTab.component.tsx index e9fb9bc11c8d..4a85cb9f81e2 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/GlossaryOverviewTab.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/GlossaryOverviewTab.component.tsx @@ -10,85 +10,55 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { noop } from 'lodash'; +import { Col, Row, Space } from 'antd'; import React, { useMemo, useState } from 'react'; -import RGL, { WidthProvider } from 'react-grid-layout'; -import { useParams } from 'react-router-dom'; import { EntityField } from '../../../../constants/Feeds.constants'; -import { GlossaryTermDetailPageWidgetKeys } from '../../../../enums/CustomizeDetailPage.enum'; -import { EntityTabs, EntityType } from '../../../../enums/entity.enum'; +import { COMMON_RESIZABLE_PANEL_CONFIG } from '../../../../constants/ResizablePanel.constants'; +import { OperationPermission } from '../../../../context/PermissionProvider/PermissionProvider.interface'; +import { EntityType } from '../../../../enums/entity.enum'; import { Glossary } from '../../../../generated/entity/data/glossary'; import { GlossaryTerm } from '../../../../generated/entity/data/glossaryTerm'; import { ChangeDescription } from '../../../../generated/entity/type'; -import { Page, PageType, Tab } from '../../../../generated/system/ui/page'; import { TagLabel, TagSource } from '../../../../generated/type/tagLabel'; -import { useGridLayoutDirection } from '../../../../hooks/useGridLayoutDirection'; -import { WidgetConfig } from '../../../../pages/CustomizablePage/CustomizablePage.interface'; -import { useCustomizeStore } from '../../../../pages/CustomizablePage/CustomizeStore'; -import customizeGlossaryTermPageClassBase from '../../../../utils/CustomiseGlossaryTermPage/CustomizeGlossaryTermPage'; import { getEntityName } from '../../../../utils/EntityUtils'; import { getEntityVersionByField, getEntityVersionTags, } from '../../../../utils/EntityVersionUtils'; -import { getWidgetFromKey } from '../../../../utils/GlossaryTerm/GlossaryTermUtil'; -import { CustomPropertyTable } from '../../../common/CustomPropertyTable/CustomPropertyTable'; -import { DomainLabel } from '../../../common/DomainLabel/DomainLabel.component'; import DescriptionV1 from '../../../common/EntityDescription/DescriptionV1'; -import { OwnerLabelV2 } from '../../../DataAssets/OwnerLabelV2/OwnerLabelV2'; -import { ReviewerLabelV2 } from '../../../DataAssets/ReviewerLabelV2/ReviewerLabelV2'; -import { useGenericContext } from '../../../GenericProvider/GenericProvider'; +import ResizablePanels from '../../../common/ResizablePanels/ResizablePanels'; import TagsContainerV2 from '../../../Tag/TagsContainerV2/TagsContainerV2'; import { DisplayType } from '../../../Tag/TagsViewer/TagsViewer.interface'; +import GlossaryDetailsRightPanel from '../../GlossaryDetailsRightPanel/GlossaryDetailsRightPanel.component'; import { GlossaryUpdateConfirmationModal } from '../../GlossaryUpdateConfirmationModal/GlossaryUpdateConfirmationModal'; import GlossaryTermReferences from './GlossaryTermReferences'; import GlossaryTermSynonyms from './GlossaryTermSynonyms'; import RelatedTerms from './RelatedTerms'; -const ReactGridLayout = WidthProvider(RGL); - type Props = { + selectedData: Glossary | GlossaryTerm; + permissions: OperationPermission; + onUpdate: (data: GlossaryTerm | Glossary) => Promise; + isGlossary: boolean; + isVersionView?: boolean; onThreadLinkSelect: (value: string) => void; editCustomAttributePermission: boolean; onExtensionUpdate: (updatedTable: GlossaryTerm) => Promise; }; const GlossaryOverviewTab = ({ + selectedData, + permissions, + onUpdate, + isGlossary, + isVersionView, onThreadLinkSelect, editCustomAttributePermission, onExtensionUpdate, }: Props) => { const [isDescriptionEditable, setIsDescriptionEditable] = useState(false); - const [tagsUpdating, setTagsUpdating] = useState(); - const { currentPersonaDocStore } = useCustomizeStore(); - // Since we are rendering this component for all customized tabs we need tab ID to get layout form store - const { tab = EntityTabs.OVERVIEW } = useParams<{ tab: EntityTabs }>(); - const { - data: selectedData, - permissions, - onUpdate, - isVersionView, - type: entityType, - } = useGenericContext(); - - const isGlossary = entityType === EntityType.GLOSSARY; - - const layout = useMemo(() => { - if (!currentPersonaDocStore) { - return customizeGlossaryTermPageClassBase.getDefaultWidgetForTab(tab); - } - const pageType = isGlossary ? PageType.Glossary : PageType.GlossaryTerm; - const page = currentPersonaDocStore?.data?.pages.find( - (p: Page) => p.pageType === pageType - ); - - if (page) { - return page.tabs.find((t: Tab) => t.id === tab)?.layout; - } else { - return customizeGlossaryTermPageClassBase.getDefaultWidgetForTab(tab); - } - }, [currentPersonaDocStore, isGlossary, tab]); + const [tagsUpdatating, setTagsUpdating] = useState(); const onDescriptionUpdate = async (updatedHTML: string) => { if (selectedData.description !== updatedHTML) { @@ -107,10 +77,6 @@ const GlossaryOverviewTab = ({ return permissions.EditAll || permissions.EditTags; }, [permissions]); - const hasViewAllPermission = useMemo(() => { - return permissions.ViewAll; - }, [permissions]); - const glossaryDescription = useMemo(() => { if (isVersionView) { return getEntityVersionByField( @@ -142,191 +108,118 @@ const GlossaryOverviewTab = ({ if (selectedData) { await onUpdate({ ...selectedData, - tags: tagsUpdating, + tags: tagsUpdatating, }); } }; - const descriptionWidget = useMemo(() => { - return ( - setIsDescriptionEditable(false)} - onDescriptionEdit={() => setIsDescriptionEditable(true)} - onDescriptionUpdate={onDescriptionUpdate} - onThreadLinkSelect={onThreadLinkSelect} - /> - ); - }, [ - glossaryDescription, - isDescriptionEditable, - selectedData, - onDescriptionUpdate, - onThreadLinkSelect, - permissions, - ]); - - const tagsWidget = useMemo(() => { - return ( - - ); - }, [ - tags, - selectedData.fullyQualifiedName, - hasEditTagsPermissions, - onThreadLinkSelect, - ]); - - const domainWidget = useMemo(() => { - return ( - - ); - }, [ - selectedData.domain, - selectedData.fullyQualifiedName, - selectedData.id, - permissions.EditAll, - isGlossary, - ]); - - const customPropertyWidget = useMemo(() => { - return ( - { - await onExtensionUpdate?.(updatedTable); - }} - hasEditAccess={Boolean(editCustomAttributePermission)} - hasPermission={hasViewAllPermission} - maxDataCap={5} - /> - ); - }, [selectedData, editCustomAttributePermission, hasViewAllPermission]); - - const widgets = useMemo(() => { - const getWidgetFromKeyInternal = (widgetConfig: WidgetConfig) => { - if ( - widgetConfig.i.startsWith( - GlossaryTermDetailPageWidgetKeys.RELATED_TERMS - ) - ) { - return ; - } else if ( - widgetConfig.i.startsWith(GlossaryTermDetailPageWidgetKeys.SYNONYMS) - ) { - return ; - } else if ( - widgetConfig.i.startsWith(GlossaryTermDetailPageWidgetKeys.TAGS) - ) { - return tagsWidget; - } else if ( - widgetConfig.i.startsWith(GlossaryTermDetailPageWidgetKeys.REFERENCES) - ) { - return ; - } else if ( - widgetConfig.i.startsWith(GlossaryTermDetailPageWidgetKeys.DESCRIPTION) - ) { - return descriptionWidget; - } else if ( - widgetConfig.i.startsWith(GlossaryTermDetailPageWidgetKeys.OWNER) - ) { - return ; - } else if ( - widgetConfig.i.startsWith(GlossaryTermDetailPageWidgetKeys.DOMAIN) - ) { - return domainWidget; - } else if ( - widgetConfig.i.startsWith(GlossaryTermDetailPageWidgetKeys.REVIEWER) - ) { - return ; - } else if ( - widgetConfig.i.startsWith( - GlossaryTermDetailPageWidgetKeys.CUSTOM_PROPERTIES - ) && - !isGlossary - ) { - return customPropertyWidget; - } - - return getWidgetFromKey({ - widgetConfig: widgetConfig, - handleOpenAddWidgetModal: noop, - handlePlaceholderWidgetKey: noop, - handleRemoveWidget: noop, - isEditView: false, - }); - }; - - return layout.map((widget: WidgetConfig) => ( -
- {getWidgetFromKeyInternal(widget)} -
- )); - }, [ - layout, - descriptionWidget, - tagsWidget, - domainWidget, - customPropertyWidget, - isGlossary, - ]); - - // call the hook to set the direction of the grid layout - useGridLayoutDirection(); - return ( - <> - - {widgets} - - {tagsUpdating && ( + +
+ + + + setIsDescriptionEditable(false)} + onDescriptionEdit={() => setIsDescriptionEditable(true)} + onDescriptionUpdate={onDescriptionUpdate} + onThreadLinkSelect={onThreadLinkSelect} + /> + + + + {!isGlossary && ( + <> + + + + + + + + + + + )} + + + + + + + + + + + ), + ...COMMON_RESIZABLE_PANEL_CONFIG.LEFT_PANEL, + }} + secondPanel={{ + children: ( + + ), + ...COMMON_RESIZABLE_PANEL_CONFIG.RIGHT_PANEL, + className: 'entity-resizable-right-panel-container', + }} + /> + + + {tagsUpdatating && ( setTagsUpdating(undefined)} onValidationSuccess={handleGlossaryTagUpdateValidationConfirm} /> )} - + ); }; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/GlossaryOverviewTab.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/GlossaryOverviewTab.test.tsx index b408ae8fafcd..67b12b7682ab 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/GlossaryOverviewTab.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/GlossaryOverviewTab.test.tsx @@ -13,6 +13,10 @@ import { act, render } from '@testing-library/react'; import React from 'react'; import { MemoryRouter } from 'react-router-dom'; +import { + MOCKED_GLOSSARY_TERMS, + MOCK_PERMISSIONS, +} from '../../../../mocks/Glossary.mock'; import GlossaryOverviewTab from './GlossaryOverviewTab.component'; jest.mock('./GlossaryTermSynonyms', () => { @@ -29,6 +33,13 @@ jest.mock('../../../common/EntityDescription/DescriptionV1', () => { return jest.fn().mockReturnValue(

Description

); }); +jest.mock( + '../../GlossaryDetailsRightPanel/GlossaryDetailsRightPanel.component', + () => { + return jest.fn().mockImplementation(() => <>testGlossaryRightPanel); + } +); + jest.mock('../../../common/ResizablePanels/ResizablePanels', () => { return jest.fn().mockImplementation(({ firstPanel, secondPanel }) => (
@@ -37,13 +48,22 @@ jest.mock('../../../common/ResizablePanels/ResizablePanels', () => { )); }); -describe.skip('GlossaryOverviewTab', () => { +const onUpdate = jest.fn(); + +describe('GlossaryOverviewTab', () => { + const selectedData = MOCKED_GLOSSARY_TERMS[0]; + const permissions = MOCK_PERMISSIONS; + it('renders the component', async () => { const { findByText } = render( , { wrapper: MemoryRouter } ); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/GlossaryTermReferences.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/GlossaryTermReferences.test.tsx index 96517ebdfbd1..a38bbdb3af6d 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/GlossaryTermReferences.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/GlossaryTermReferences.test.tsx @@ -18,23 +18,19 @@ import { } from '../../../../mocks/Glossary.mock'; import GlossaryTermReferences from './GlossaryTermReferences'; -const [mockGlossaryTerm1, mockGlossaryTerm2] = MOCKED_GLOSSARY_TERMS; - -const mockContext = { - data: mockGlossaryTerm1, - onUpdate: jest.fn(), - isVersionView: false, - permissions: MOCK_PERMISSIONS, -}; - -jest.mock('../../../GenericProvider/GenericProvider', () => ({ - useGenericContext: jest.fn().mockImplementation(() => mockContext), -})); +const mockOnGlossaryTermUpdate = jest.fn(); describe('GlossaryTermReferences', () => { it('renders glossary term references', async () => { - mockContext.data = mockGlossaryTerm2; - const { getByText, getByTestId } = render(); + const mockGlossaryTerm = MOCKED_GLOSSARY_TERMS[1]; + const mockPermissions = MOCK_PERMISSIONS; + const { getByText, getByTestId } = render( + + ); const sectionTitle = getByTestId('section-label.reference-plural'); const editBtn = getByTestId('edit-button'); @@ -57,18 +53,29 @@ describe('GlossaryTermReferences', () => { }); it('renders add button', async () => { - mockContext.data = mockGlossaryTerm1; - - const { getByTestId } = render(); + const mockGlossaryTerm = MOCKED_GLOSSARY_TERMS[0]; + const mockPermissions = MOCK_PERMISSIONS; + const { getByTestId } = render( + + ); expect(getByTestId('term-references-add-button')).toBeInTheDocument(); }); it('should not render add button if no permission', async () => { - mockContext.data = mockGlossaryTerm1; - mockContext.permissions = { ...MOCK_PERMISSIONS, EditAll: false }; - - const { queryByTestId, findByText } = render(); + const mockGlossaryTerm = MOCKED_GLOSSARY_TERMS[0]; + const mockPermissions = { ...MOCK_PERMISSIONS, EditAll: false }; + const { queryByTestId, findByText } = render( + + ); expect(queryByTestId('term-references-add-button')).toBeNull(); @@ -78,7 +85,15 @@ describe('GlossaryTermReferences', () => { }); it('should not render edit button if no permission', async () => { - const { queryByTestId } = render(); + const mockGlossaryTerm = MOCKED_GLOSSARY_TERMS[1]; + const mockPermissions = { ...MOCK_PERMISSIONS, EditAll: false }; + const { queryByTestId } = render( + + ); expect(queryByTestId('edit-button')).toBeNull(); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/GlossaryTermReferences.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/GlossaryTermReferences.tsx index a72a5a1a5a92..8f0dff3c8b71 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/GlossaryTermReferences.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/GlossaryTermReferences.tsx @@ -11,18 +11,26 @@ * limitations under the License. */ -import { Button, Space, Tooltip, Typography } from 'antd'; +import Icon from '@ant-design/icons/lib/components/Icon'; +import { Button, Space, Tag, Tooltip, Typography } from 'antd'; +import classNames from 'classnames'; import { t } from 'i18next'; import { cloneDeep, isEmpty, isEqual } from 'lodash'; import React, { useCallback, useEffect, useState } from 'react'; import { ReactComponent as EditIcon } from '../../../../assets/svg/edit-new.svg'; +import { ReactComponent as ExternalLinkIcon } from '../../../../assets/svg/external-links.svg'; import { ReactComponent as PlusIcon } from '../../../../assets/svg/plus-primary.svg'; import { DE_ACTIVE_COLOR, + ICON_DIMENSION, NO_DATA_PLACEHOLDER, + SUCCESS_COLOR, + TEXT_BODY_COLOR, + TEXT_GREY_MUTED, } from '../../../../constants/constants'; import { EntityField } from '../../../../constants/Feeds.constants'; import { NO_PERMISSION_FOR_ACTION } from '../../../../constants/HelperTextUtil'; +import { OperationPermission } from '../../../../context/PermissionProvider/PermissionProvider.interface'; import { GlossaryTerm, TermReference, @@ -33,20 +41,25 @@ import { getChangedEntityOldValue, getDiffByFieldName, } from '../../../../utils/EntityVersionUtils'; -import { renderReferenceElement } from '../../../../utils/GlossaryUtils'; +import { VersionStatus } from '../../../../utils/EntityVersionUtils.interface'; import TagButton from '../../../common/TagButton/TagButton.component'; -import { useGenericContext } from '../../../GenericProvider/GenericProvider'; import GlossaryTermReferencesModal from '../GlossaryTermReferencesModal.component'; -const GlossaryTermReferences = () => { +interface GlossaryTermReferencesProps { + isVersionView?: boolean; + glossaryTerm: GlossaryTerm; + permissions: OperationPermission; + onGlossaryTermUpdate: (glossaryTerm: GlossaryTerm) => Promise; +} + +const GlossaryTermReferences = ({ + glossaryTerm, + permissions, + onGlossaryTermUpdate, + isVersionView, +}: GlossaryTermReferencesProps) => { const [references, setReferences] = useState([]); const [isViewMode, setIsViewMode] = useState(true); - const { - data: glossaryTerm, - onUpdate: onGlossaryTermUpdate, - isVersionView, - permissions, - } = useGenericContext(); const handleReferencesSave = async ( newReferences: TermReference[], @@ -78,6 +91,52 @@ const GlossaryTermReferences = () => { setReferences(glossaryTerm.references ? glossaryTerm.references : []); }, [glossaryTerm.references]); + const getReferenceElement = useCallback( + (ref: TermReference, versionStatus?: VersionStatus) => { + let iconColor: string; + let textClassName: string; + if (versionStatus?.added) { + iconColor = SUCCESS_COLOR; + textClassName = 'text-success'; + } else if (versionStatus?.removed) { + iconColor = TEXT_GREY_MUTED; + textClassName = 'text-grey-muted'; + } else { + iconColor = TEXT_BODY_COLOR; + textClassName = 'text-body'; + } + + return ( + + + +
+ + {ref.name} +
+ + + + ); + }, + [] + ); + const getVersionReferenceElements = useCallback(() => { const changeDescription = glossaryTerm.changeDescription; const referencesDiff = getDiffByFieldName( @@ -113,14 +172,12 @@ const GlossaryTermReferences = () => { return (
- {unchangedReferences.map((reference) => - renderReferenceElement(reference) - )} + {unchangedReferences.map((reference) => getReferenceElement(reference))} {addedReferences.map((reference) => - renderReferenceElement(reference, { added: true }) + getReferenceElement(reference, { added: true }) )} {deletedReferences.map((reference) => - renderReferenceElement(reference, { removed: true }) + getReferenceElement(reference, { removed: true }) )}
); @@ -163,7 +220,7 @@ const GlossaryTermReferences = () => { getVersionReferenceElements() ) : (
- {references.map((ref) => renderReferenceElement(ref))} + {references.map((ref) => getReferenceElement(ref))} {permissions.EditAll && references.length === 0 && ( ({ - useGenericContext: jest.fn().mockImplementation(() => mockContext), -})); +const onGlossaryTermUpdate = jest.fn(); describe('GlossaryTermSynonyms', () => { it('renders synonyms and edit button', () => { - mockContext.data = mockGlossaryTerm2; - const { getByTestId, getByText } = render(); + const glossaryTerm = MOCKED_GLOSSARY_TERMS[1]; + const permissions = MOCK_PERMISSIONS; + const { getByTestId, getByText } = render( + + ); const synonymsContainer = getByTestId('synonyms-container'); const synonymItem = getByText('accessory'); const editBtn = getByTestId('edit-button'); @@ -45,8 +41,15 @@ describe('GlossaryTermSynonyms', () => { }); it('renders add button', () => { - mockContext.data = mockGlossaryTerm1; - const { getByTestId } = render(); + const glossaryTerm = MOCKED_GLOSSARY_TERMS[0]; + const permissions = MOCK_PERMISSIONS; + const { getByTestId } = render( + + ); const synonymsContainer = getByTestId('synonyms-container'); const synonymAddBtn = getByTestId('synonym-add-button'); @@ -55,10 +58,14 @@ describe('GlossaryTermSynonyms', () => { }); it('should not render add button if no permission', async () => { - mockContext.data = mockGlossaryTerm1; - mockContext.permissions = { ...MOCK_PERMISSIONS, EditAll: false }; + const glossaryTerm = MOCKED_GLOSSARY_TERMS[0]; + const permissions = { ...MOCK_PERMISSIONS, EditAll: false }; const { getByTestId, queryByTestId, findByText } = render( - + ); const synonymsContainer = getByTestId('synonyms-container'); const synonymAddBtn = queryByTestId('synonym-add-button'); @@ -72,9 +79,15 @@ describe('GlossaryTermSynonyms', () => { }); it('should not render edit button if no permission', () => { - mockContext.data = mockGlossaryTerm2; - mockContext.permissions = { ...MOCK_PERMISSIONS, EditAll: false }; - const { getByTestId, queryByTestId } = render(); + const glossaryTerm = MOCKED_GLOSSARY_TERMS[1]; + const permissions = { ...MOCK_PERMISSIONS, EditAll: false }; + const { getByTestId, queryByTestId } = render( + + ); const synonymsContainer = getByTestId('synonyms-container'); const editBtn = queryByTestId('edit-button'); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/GlossaryTermSynonyms.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/GlossaryTermSynonyms.tsx index a1fb34d9a824..e66fdc58c635 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/GlossaryTermSynonyms.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/GlossaryTermSynonyms.tsx @@ -24,6 +24,7 @@ import { } from '../../../../constants/constants'; import { EntityField } from '../../../../constants/Feeds.constants'; import { NO_PERMISSION_FOR_ACTION } from '../../../../constants/HelperTextUtil'; +import { OperationPermission } from '../../../../context/PermissionProvider/PermissionProvider.interface'; import { GlossaryTerm } from '../../../../generated/entity/data/glossaryTerm'; import { ChangeDescription } from '../../../../generated/entity/type'; import { @@ -32,18 +33,23 @@ import { getDiffByFieldName, } from '../../../../utils/EntityVersionUtils'; import TagButton from '../../../common/TagButton/TagButton.component'; -import { useGenericContext } from '../../../GenericProvider/GenericProvider'; -const GlossaryTermSynonyms = () => { +interface GlossaryTermSynonymsProps { + isVersionView?: boolean; + permissions: OperationPermission; + glossaryTerm: GlossaryTerm; + onGlossaryTermUpdate: (glossaryTerm: GlossaryTerm) => Promise; +} + +const GlossaryTermSynonyms = ({ + permissions, + glossaryTerm, + onGlossaryTermUpdate, + isVersionView, +}: GlossaryTermSynonymsProps) => { const [isViewMode, setIsViewMode] = useState(true); const [synonyms, setSynonyms] = useState([]); const [saving, setSaving] = useState(false); - const { - data: glossaryTerm, - onUpdate: onGlossaryTermUpdate, - isVersionView, - permissions, - } = useGenericContext(); const getSynonyms = () => (
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/RelatedTerms.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/RelatedTerms.test.tsx index 02efd7c68c5e..23a3481a6a89 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/RelatedTerms.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/RelatedTerms.test.tsx @@ -18,40 +18,57 @@ import { } from '../../../../mocks/Glossary.mock'; import RelatedTerms from './RelatedTerms'; -const mockContext = { - data: MOCKED_GLOSSARY_TERMS[2], - onUpdate: jest.fn(), - isVersionView: false, - permissions: MOCK_PERMISSIONS, -}; +const glossaryTerm = MOCKED_GLOSSARY_TERMS[2]; -jest.mock('../../../GenericProvider/GenericProvider', () => ({ - useGenericContext: jest.fn().mockImplementation(() => mockContext), -})); +const permissions = MOCK_PERMISSIONS; + +const onGlossaryTermUpdate = jest.fn(); describe('RelatedTerms', () => { it('should render the component', () => { - const { container } = render(); + const { container } = render( + + ); expect(container).toBeInTheDocument(); }); it('should show the related terms', () => { - const { getByText } = render(); + const { getByText } = render( + + ); expect(getByText('Business Customer')).toBeInTheDocument(); }); it('should show the add button if there are no related terms and the user has edit permissions', () => { - mockContext.data = { ...mockContext.data, relatedTerms: [] }; - const { getByTestId } = render(); + const { getByTestId } = render( + + ); expect(getByTestId('related-term-add-button')).toBeInTheDocument(); }); it('should not show the add button if there are no related terms and the user does not have edit permissions', async () => { - mockContext.permissions = { ...mockContext.permissions, EditAll: false }; - const { queryByTestId, findByText } = render(); + const { queryByTestId, findByText } = render( + + ); expect(queryByTestId('related-term-add-button')).toBeNull(); @@ -61,16 +78,25 @@ describe('RelatedTerms', () => { }); it('should show the edit button if there are related terms and the user has edit permissions', () => { - mockContext.permissions = MOCK_PERMISSIONS; - mockContext.data = { ...MOCKED_GLOSSARY_TERMS[2] }; - const { getByTestId } = render(); + const { getByTestId } = render( + + ); expect(getByTestId('edit-button')).toBeInTheDocument(); }); it('should not show the edit button if there are no related terms and the user has edit permissions', () => { - mockContext.data = { ...MOCKED_GLOSSARY_TERMS[2], relatedTerms: [] }; - const { queryByTestId } = render(); + const { queryByTestId } = render( + + ); expect(queryByTestId('edit-button')).toBeNull(); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/RelatedTerms.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/RelatedTerms.tsx index d9a528bb19e7..d0221548db31 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/RelatedTerms.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/RelatedTerms.tsx @@ -27,6 +27,7 @@ import { } from '../../../../constants/constants'; import { EntityField } from '../../../../constants/Feeds.constants'; import { NO_PERMISSION_FOR_ACTION } from '../../../../constants/HelperTextUtil'; +import { OperationPermission } from '../../../../context/PermissionProvider/PermissionProvider.interface'; import { EntityType } from '../../../../enums/entity.enum'; import { GlossaryTerm } from '../../../../generated/entity/data/glossaryTerm'; import { @@ -46,17 +47,21 @@ import { VersionStatus } from '../../../../utils/EntityVersionUtils.interface'; import { getGlossaryPath } from '../../../../utils/RouterUtils'; import { SelectOption } from '../../../common/AsyncSelectList/AsyncSelectList.interface'; import TagButton from '../../../common/TagButton/TagButton.component'; -import { useGenericContext } from '../../../GenericProvider/GenericProvider'; -const RelatedTerms = () => { - const history = useHistory(); - const { - data: glossaryTerm, - onUpdate, - isVersionView, - permissions, - } = useGenericContext(); +interface RelatedTermsProps { + isVersionView?: boolean; + permissions: OperationPermission; + glossaryTerm: GlossaryTerm; + onGlossaryTermUpdate: (data: GlossaryTerm) => Promise; +} +const RelatedTerms = ({ + isVersionView, + glossaryTerm, + permissions, + onGlossaryTermUpdate, +}: RelatedTermsProps) => { + const history = useHistory(); const [isIconVisible, setIsIconVisible] = useState(true); const [selectedOption, setSelectedOption] = useState([]); @@ -99,7 +104,7 @@ const RelatedTerms = () => { relatedTerms: newOptions, }; - await onUpdate(updatedGlossaryTerm); + await onGlossaryTermUpdate(updatedGlossaryTerm); setIsIconVisible(true); }; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryV1.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryV1.component.tsx index 87ac05380160..a3c165428aa1 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryV1.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryV1.component.tsx @@ -25,12 +25,11 @@ import { OperationPermission, ResourceEntity, } from '../../context/PermissionProvider/PermissionProvider.interface'; -import { EntityAction, EntityTabs } from '../../enums/entity.enum'; +import { EntityAction } from '../../enums/entity.enum'; import { CreateThread, ThreadType, } from '../../generated/api/feed/createThread'; -import { Glossary } from '../../generated/entity/data/glossary'; import { GlossaryTerm } from '../../generated/entity/data/glossaryTerm'; import { VERSION_VIEW_GLOSSARY_PERMISSION } from '../../mocks/Glossary.mock'; import { postThread } from '../../rest/feedsAPI'; @@ -99,8 +98,12 @@ const GlossaryV1 = ({ const [editMode, setEditMode] = useState(false); - const { activeGlossary, glossaryChildTerms, setGlossaryChildTerms } = - useGlossaryStore(); + const { + activeGlossary, + glossaryChildTerms, + setGlossaryChildTerms, + insertNewGlossaryTermToChildTerms, + } = useGlossaryStore(); const { id, fullyQualifiedName } = activeGlossary ?? {}; @@ -129,11 +132,9 @@ const GlossaryV1 = ({ const { data } = await getFirstLevelGlossaryTerms( params?.glossary ?? params?.parent ?? '' ); - const children = data.map((data) => - data.childrenCount ?? 0 > 0 ? { ...data, children: [] } : data - ); - - setGlossaryChildTerms(children as ModifiedGlossary[]); + // We are considering childrenCount fot expand collapse state + // Hence don't need any intervention to list response here + setGlossaryChildTerms(data as ModifiedGlossary[]); } catch (error) { showErrorToast(error as AxiosError); } finally { @@ -232,7 +233,11 @@ const GlossaryV1 = ({ entity: t('label.glossary-term'), }); } else { - updateGlossaryTermInStore(response); + updateGlossaryTermInStore({ + ...response, + // Since patch didn't respond with childrenCount preserve it from currentData + childrenCount: currentData.childrenCount, + }); setIsEditModalOpen(false); } } catch (error) { @@ -257,29 +262,38 @@ const GlossaryV1 = ({ } }; - const onTermModalSuccess = useCallback(() => { - loadGlossaryTerms(true); - if (!isGlossaryActive && tab !== 'terms') { - history.push( - getGlossaryTermDetailsPath( - selectedData.fullyQualifiedName || '', - EntityTabs.TERMS - ) - ); - } - setIsEditModalOpen(false); - }, [isGlossaryActive, tab, selectedData]); + const onTermModalSuccess = useCallback( + (term: GlossaryTerm) => { + // Setting loading so that nested terms are rendered again on table with change + setIsTermsLoading(true); + // Update store with newly created term + insertNewGlossaryTermToChildTerms(term); + if (!isGlossaryActive && tab !== 'terms') { + history.push( + getGlossaryTermDetailsPath( + selectedData.fullyQualifiedName || '', + 'terms' + ) + ); + } + // Close modal and set loading to false + setIsEditModalOpen(false); + setIsTermsLoading(false); + }, + [isGlossaryActive, tab, selectedData] + ); const handleGlossaryTermAdd = async (formData: GlossaryTermForm) => { try { - await addGlossaryTerm({ + const term = await addGlossaryTerm({ ...formData, glossary: activeGlossaryTerm?.glossary?.name || (selectedData.fullyQualifiedName ?? ''), parent: activeGlossaryTerm?.fullyQualifiedName, }); - onTermModalSuccess(); + + onTermModalSuccess(term); } catch (error) { if ( (error as AxiosError).response?.status === HTTP_STATUS_CODE.CONFLICT @@ -341,17 +355,6 @@ const GlossaryV1 = ({ } }; - const handleGlossaryUpdate = async (newGlossary: Glossary) => { - const jsonPatch = compare(selectedData, newGlossary); - - const shouldRefreshTerms = jsonPatch.some((patch) => - patch.path.startsWith('/owners') - ); - - await updateGlossary(newGlossary); - shouldRefreshTerms && loadGlossaryTerms(true); - }; - const initPermissions = async () => { setIsPermissionLoading(true); const permissionFetch = isGlossaryActive @@ -393,7 +396,7 @@ const GlossaryV1 = ({ permissions={glossaryPermission} refreshGlossaryTerms={() => loadGlossaryTerms(true)} termsLoading={isTermsLoading} - updateGlossary={handleGlossaryUpdate} + updateGlossary={updateGlossary} updateVote={updateVote} onAddGlossaryTerm={(term) => handleGlossaryTermModalAction(false, term ?? null) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/useGlossary.store.ts b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/useGlossary.store.ts index 81ec1d9bef89..90bcd4dc45c0 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/useGlossary.store.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/useGlossary.store.ts @@ -12,10 +12,13 @@ */ import { create } from 'zustand'; import { Glossary } from '../../generated/entity/data/glossary'; +import { GlossaryTerm } from '../../generated/entity/data/glossaryTerm'; import { GlossaryTermWithChildren } from '../../rest/glossaryAPI'; +import { findAndUpdateNested } from '../../utils/GlossaryUtils'; export type ModifiedGlossary = Glossary & { children?: GlossaryTermWithChildren[]; + childrenCount?: number; }; export const useGlossaryStore = create<{ @@ -27,6 +30,7 @@ export const useGlossaryStore = create<{ updateGlossary: (glossary: Glossary) => void; updateActiveGlossary: (glossary: Partial) => void; setGlossaryChildTerms: (glossaryChildTerms: ModifiedGlossary[]) => void; + insertNewGlossaryTermToChildTerms: (glossary: GlossaryTerm) => void; }>()((set, get) => ({ glossaries: [], activeGlossary: {} as ModifiedGlossary, @@ -67,6 +71,30 @@ export const useGlossaryStore = create<{ glossaries[index] = updatedGlossary; } }, + insertNewGlossaryTermToChildTerms: (glossary: GlossaryTerm) => { + const { glossaryChildTerms, activeGlossary } = get(); + + const glossaryTerm = 'glossary' in activeGlossary; + + // If activeGlossary is Glossary term & User is adding term to the activeGlossary term + // we don't need to find in hierarchy + if ( + glossaryTerm && + activeGlossary.fullyQualifiedName === glossary.parent?.fullyQualifiedName + ) { + set({ + glossaryChildTerms: [ + ...glossaryChildTerms, + glossary, + ] as ModifiedGlossary[], + }); + } else { + // Typically used to updated the glossary term list in the glossary page + set({ + glossaryChildTerms: findAndUpdateNested(glossaryChildTerms, glossary), + }); + } + }, setGlossaryChildTerms: (glossaryChildTerms: ModifiedGlossary[]) => { set({ glossaryChildTerms }); }, diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MlModel/MlModelDetail/MlModelFeaturesList.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/MlModel/MlModelDetail/MlModelFeaturesList.test.tsx index af3a60eb66b7..8389d5255a33 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/MlModel/MlModelDetail/MlModelFeaturesList.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/MlModel/MlModelDetail/MlModelFeaturesList.test.tsx @@ -122,7 +122,7 @@ jest.mock('../../../utils/CommonUtils', () => ({ getHtmlForNonAdminAction: jest.fn().mockReturnValue('admin action'), })); -jest.mock('../../common/RichTextEditor/RichTextEditorPreviewer', () => { +jest.mock('../../common/RichTextEditor/RichTextEditorPreviewerV1', () => { return jest.fn().mockReturnValue(

RichTextEditorPreviewer

); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MlModel/MlModelVersion/MlModelVersion.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/MlModel/MlModelVersion/MlModelVersion.component.tsx index e0cfe79ad904..940a20155b89 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/MlModel/MlModelVersion/MlModelVersion.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/MlModel/MlModelVersion/MlModelVersion.component.tsx @@ -42,7 +42,7 @@ import { CustomPropertyTable } from '../../common/CustomPropertyTable/CustomProp import DescriptionV1 from '../../common/EntityDescription/DescriptionV1'; import ErrorPlaceHolder from '../../common/ErrorWithPlaceholder/ErrorPlaceHolder'; import Loader from '../../common/Loader/Loader'; -import RichTextEditorPreviewer from '../../common/RichTextEditor/RichTextEditorPreviewer'; +import RichTextEditorPreviewerV1 from '../../common/RichTextEditor/RichTextEditorPreviewerV1'; import TabsLabel from '../../common/TabsLabel/TabsLabel.component'; import DataAssetsVersionHeader from '../../DataAssets/DataAssetsVersionHeader/DataAssetsVersionHeader'; import DataProductsContainer from '../../DataProducts/DataProductsContainer/DataProductsContainer.component'; @@ -251,7 +251,7 @@ const MlModelVersion: FC = ({
{feature.description ? ( - diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Modals/AddTableModal/AddTableModal.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Modals/AddTableModal/AddTableModal.component.tsx new file mode 100644 index 000000000000..042fa66b944a --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/Modals/AddTableModal/AddTableModal.component.tsx @@ -0,0 +1,97 @@ +/* + * Copyright 2023 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Button, Form, FormProps, Modal, Typography } from 'antd'; +import React, { useEffect, useState } from 'react'; +import { useTranslation } from 'react-i18next'; +import SanitizedInput from '../../common/SanitizedInput/SanitizedInput'; +import { AddTableModalProps } from './AddTableModal.interface'; + +const AddTableModal: React.FC = ({ + visible, + entity, + onCancel, + onSave, + title, + // re-name will update actual name of the entity, it will impact across application + // By default its disabled, send allowRename true to get the functionality + allowRename = false, + additionalFields, +}) => { + const { t } = useTranslation(); + const [form] = Form.useForm(); + const [isLoading, setIsLoading] = useState(false); + let entityData = entity; + + function buildName(e: string) { + entityData.name = e.trim().replace(' ', '_').replace(/[-:,\(\)]/g, '').normalize("NFD").replace(/[\u0300-\u036f]/g, ""); + form.setFieldsValue(entityData); + } + + const handleSave: FormProps['onFinish'] = async (obj) => { + setIsLoading(true); + await form.validateFields(); + // Error must be handled by the parent component + await onSave(obj); + setIsLoading(false); + }; + + useEffect(() => { + form.setFieldsValue(entityData); + }, [visible]); + + return ( + + {t('label.cancel')} + , + , + ]} + maskClosable={false} + okText={t('label.save')} + open={visible} + title={ + + {title} + + } + onCancel={onCancel}> +
+ + + + + buildName(e.target.value)} placeholder={t('message.enter-display-name')} /> + + + {additionalFields} + + + ); +}; + +export default AddTableModal; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Modals/AddTableModal/AddTableModal.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/Modals/AddTableModal/AddTableModal.interface.ts new file mode 100644 index 000000000000..6c0bbe5b20f1 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/Modals/AddTableModal/AddTableModal.interface.ts @@ -0,0 +1,31 @@ +/* + * Copyright 2023 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Rule } from 'antd/lib/form'; +import { Constraint } from '../../../generated/entity/data/table'; + +export type EntityName = { name: string; displayName?: string; id?: string }; + +export type EntityNameWithAdditionFields = EntityName & { + constraint: Constraint; +}; + +export interface AddTableModalProps { + visible: boolean; + allowRename?: boolean; + onCancel: () => void; + onSave: (obj: EntityName) => void | Promise; + entity: Partial; + title: string; + nameValidationRules?: Rule[]; + additionalFields?: React.ReactNode; +} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Modals/AnnouncementModal/AddAnnouncementModal.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Modals/AnnouncementModal/AddAnnouncementModal.tsx index a98a30577611..f2ac332d6900 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Modals/AnnouncementModal/AddAnnouncementModal.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Modals/AnnouncementModal/AddAnnouncementModal.tsx @@ -14,7 +14,7 @@ import { DatePicker, Form, Input, Modal, Space } from 'antd'; import { AxiosError } from 'axios'; import { Moment } from 'moment'; -import React, { FC, useState } from 'react'; +import React, { FC, useMemo, useState } from 'react'; import { useTranslation } from 'react-i18next'; import { VALIDATION_MESSAGES } from '../../../constants/constants'; import { @@ -27,7 +27,8 @@ import { getEntityFeedLink } from '../../../utils/EntityUtils'; import { showErrorToast, showSuccessToast } from '../../../utils/ToastUtils'; import { useApplicationStore } from '../../../hooks/useApplicationStore'; -import RichTextEditor from '../../common/RichTextEditor/RichTextEditor'; +import { FieldProp, FieldTypes } from '../../../interface/FormUtils.interface'; +import { getField } from '../../../utils/formUtils'; import './announcement-modal.less'; interface Props { @@ -96,6 +97,22 @@ const AddAnnouncementModal: FC = ({ } }; + const descriptionField: FieldProp = useMemo( + () => ({ + name: 'description', + required: false, + label: `${t('label.description')}:`, + id: 'root/description', + type: FieldTypes.DESCRIPTION, + props: { + 'data-testid': 'description', + initialValue: '', + placeHolder: t('message.write-your-announcement-lowercase'), + }, + }), + [] + ); + return ( = ({ - - - + {getField(descriptionField)} ); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Modals/AnnouncementModal/EditAnnouncementModal.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Modals/AnnouncementModal/EditAnnouncementModal.tsx index 12bb09076356..b47a818c5a14 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Modals/AnnouncementModal/EditAnnouncementModal.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Modals/AnnouncementModal/EditAnnouncementModal.tsx @@ -13,13 +13,14 @@ import { DatePicker, Form, Input, Modal, Space } from 'antd'; import moment from 'moment'; -import React, { FC } from 'react'; +import React, { FC, useMemo } from 'react'; import { useTranslation } from 'react-i18next'; import { VALIDATION_MESSAGES } from '../../../constants/constants'; import { AnnouncementDetails } from '../../../generated/entity/feed/thread'; +import { FieldProp, FieldTypes } from '../../../interface/FormUtils.interface'; import { getTimeZone } from '../../../utils/date-time/DateTimeUtils'; +import { getField } from '../../../utils/formUtils'; import { showErrorToast } from '../../../utils/ToastUtils'; -import RichTextEditor from '../../common/RichTextEditor/RichTextEditor'; import { CreateAnnouncement } from './AddAnnouncementModal'; import './announcement-modal.less'; @@ -63,6 +64,22 @@ const EditAnnouncementModal: FC = ({ } }; + const descriptionField: FieldProp = useMemo( + () => ({ + name: 'description', + required: false, + label: `${t('label.description')}:`, + id: 'root/description', + type: FieldTypes.DESCRIPTION, + props: { + 'data-testid': 'description', + initialValue: announcement.description, + placeHolder: t('message.write-your-announcement-lowercase'), + }, + }), + [announcement.description] + ); + return ( = ({ - - - + {getField(descriptionField)} ); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Modals/EntityNameModal/EntityNameModal.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Modals/EntityNameModal/EntityNameModal.component.tsx index ec9fd1069182..c983a2a6abdc 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Modals/EntityNameModal/EntityNameModal.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Modals/EntityNameModal/EntityNameModal.component.tsx @@ -15,9 +15,9 @@ import React, { useEffect, useState } from 'react'; import { useTranslation } from 'react-i18next'; import { ENTITY_NAME_REGEX } from '../../../constants/regex.constants'; import SanitizedInput from '../../common/SanitizedInput/SanitizedInput'; -import { EntityName, EntityNameModalProps } from './EntityNameModal.interface'; +import { EntityNameModalProps } from './EntityNameModal.interface'; -const EntityNameModal = ({ +const EntityNameModal: React.FC = ({ visible, entity, onCancel, @@ -28,7 +28,7 @@ const EntityNameModal = ({ allowRename = false, nameValidationRules = [], additionalFields, -}: EntityNameModalProps) => { +}) => { const { t } = useTranslation(); const [form] = Form.useForm(); const [isLoading, setIsLoading] = useState(false); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Modals/EntityNameModal/EntityNameModal.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/Modals/EntityNameModal/EntityNameModal.interface.ts index 2106081055e3..9d217278538e 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Modals/EntityNameModal/EntityNameModal.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/Modals/EntityNameModal/EntityNameModal.interface.ts @@ -19,14 +19,12 @@ export type EntityNameWithAdditionFields = EntityName & { constraint: Constraint; }; -export interface EntityNameModalProps< - T extends { name: string; displayName?: string } -> { +export interface EntityNameModalProps { visible: boolean; allowRename?: boolean; onCancel: () => void; - onSave: (obj: T) => void | Promise; - entity: T; + onSave: (obj: EntityName) => void | Promise; + entity: Partial; title: string; nameValidationRules?: Rule[]; additionalFields?: React.ReactNode; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithMarkdownEditor/ModalWithMarkdownEditor.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithMarkdownEditor/ModalWithMarkdownEditor.tsx index 10683e3ebf5a..d9638f795d1b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithMarkdownEditor/ModalWithMarkdownEditor.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithMarkdownEditor/ModalWithMarkdownEditor.tsx @@ -35,13 +35,15 @@ export const ModalWithMarkdownEditor: FunctionComponent(false); - const markdownRef = useRef(); + const markdownRef = useRef({} as EditorContentRef); const handleSaveData = async () => { if (markdownRef.current) { setIsLoading(true); try { - await onSave?.(markdownRef.current?.getEditorContent().trim() ?? ''); + const content = + markdownRef.current?.getEditorContent?.()?.trim() ?? ''; + await onSave?.(content); } catch (error) { showErrorToast(error as AxiosError); } finally { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Modals/WhatsNewModal/ChangeLogs.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Modals/WhatsNewModal/ChangeLogs.tsx index e32b1b6cb5cd..5f3566a0041f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Modals/WhatsNewModal/ChangeLogs.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Modals/WhatsNewModal/ChangeLogs.tsx @@ -14,7 +14,7 @@ /* eslint-disable max-len */ import React from 'react'; -import RichTextEditorPreviewer from '../../common/RichTextEditor/RichTextEditorPreviewer'; +import RichTextEditorPreviewerV1 from '../../common/RichTextEditor/RichTextEditorPreviewerV1'; type Props = { data: { [name: string]: string }; @@ -29,13 +29,13 @@ const ChangeLogs = ({ data }: Props) => {

-

- diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Modals/WhatsNewModal/FeaturesCarousel.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Modals/WhatsNewModal/FeaturesCarousel.tsx index e342d8d11965..4c9875370cfe 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Modals/WhatsNewModal/FeaturesCarousel.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Modals/WhatsNewModal/FeaturesCarousel.tsx @@ -14,7 +14,7 @@ import { Carousel } from 'antd'; import { uniqueId } from 'lodash'; import React from 'react'; -import RichTextEditorPreviewer from '../../common/RichTextEditor/RichTextEditorPreviewer'; +import RichTextEditorPreviewerV1 from '../../common/RichTextEditor/RichTextEditorPreviewerV1'; import { FeaturesCarouselProps } from './FeaturesCarousel.interface'; const FeaturesCarousel = ({ data }: FeaturesCarouselProps) => { @@ -25,7 +25,7 @@ const FeaturesCarousel = ({ data }: FeaturesCarouselProps) => {

{d.title}

- diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Modals/WhatsNewModal/WhatsNewModal.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Modals/WhatsNewModal/WhatsNewModal.tsx index 21c5efcef282..0f7af90ec41a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Modals/WhatsNewModal/WhatsNewModal.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Modals/WhatsNewModal/WhatsNewModal.tsx @@ -139,9 +139,7 @@ const WhatsNewModal: FunctionComponent = ({ onClick={() => { handleToggleChange(ToggleType.CHANGE_LOG); }}> - {t('label.change-entity', { - entity: t('label.log-plural'), - })} + {t('label.change-log-plural')}
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Modals/WhatsNewModal/whats-new-modal.less b/openmetadata-ui/src/main/resources/ui/src/components/Modals/WhatsNewModal/whats-new-modal.less index 67f4a9800018..c83c2d6e73d6 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Modals/WhatsNewModal/whats-new-modal.less +++ b/openmetadata-ui/src/main/resources/ui/src/components/Modals/WhatsNewModal/whats-new-modal.less @@ -26,6 +26,10 @@ margin: 0 4px !important; vertical-align: middle; } + + .node-image { + display: inline-block; + } } .whats-new-modal-button-container { display: inline-flex; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Modals/WhatsNewModal/whatsNewData.ts b/openmetadata-ui/src/main/resources/ui/src/components/Modals/WhatsNewModal/whatsNewData.ts index 18562023fcbf..7cee6d909e7d 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Modals/WhatsNewModal/whatsNewData.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/Modals/WhatsNewModal/whatsNewData.ts @@ -17,7 +17,7 @@ import incidentManagerSampleData from '../../../assets/img/incidentManagerSample import profilerConfigPage from '../../../assets/img/profilerConfigPage.png'; import collateIcon from '../../../assets/svg/ic-collate.svg'; -export const COOKIE_VERSION = 'VERSION_1_5_12'; // To be changed with each release. +export const COOKIE_VERSION = 'VERSION_1_6_4'; // To be changed with each release. // for youtube video make isImage = false and path = {video embed id} // embed:- youtube video => share => click on embed and take {url with id} from it @@ -1170,7 +1170,7 @@ To continue pursuing this objective, the application was completely refactored t { id: 55, version: 'v1.6.0', - description: 'Released on 5th December 2024.', + description: 'Released on 10th December 2024.', features: [ { title: `Visualizing Your Data Landscape with Entity Relationship (ER) Diagrams (Collate)`, @@ -1310,4 +1310,306 @@ Since we are introducing the Auto Classification workflow, we are going to remov [`Service Spec for the Ingestion Framework`]: `This impacts users who maintain their own connectors for the ingestion framework that are NOT part of the OpenMetadata python library (openmetadata-ingestion). Introducing the "connector specifcication class (ServiceSpec)". The ServiceSpec class serves as the entrypoint for the connector and holds the references for the classes that will be used to ingest and process the metadata from the source. You can see postgres for an implementation example.`, }, }, + { + id: 56, + version: 'v1.6.1', + description: 'Released on 10th December 2024.', + note: "In 1.6.1, Fixes tags listing for explore page on top of 1.6.0 release. Don't miss out the release highlights!", + features: [ + { + title: `Visualizing Your Data Landscape with Entity Relationship (ER) Diagrams (Collate)`, + description: `Understanding complex database schemas can be challenging without clear visualization. While OpenMetadata's best-in-class Lineage UI helps track data flow, there are better options for viewing structural relationships between tables. Collate 1.6 introduces ER diagrams as a new feature to let you: + +1. Visualize table connections through primary and foreign key constraints + +2. Navigate between data assets to discover relationships + +3. Modify connections using the built-in UI editor + +ER diagrams help you better understand and manage your data architecture by showing how your database tables relate to each other.`, + isImage: false, + path: 'https://www.youtube.com/embed/3m2xHpIsYuM', + }, + { + title: + 'Establishing Smooth Data Governance with Automated Glossary Approval Workflows (Collate)', + description: `Organizations often struggle with data governance due to rigid, pre-defined manual workflows. OpenMetadata 1.6 introduces a new, automated data governance framework designed to be customized to each organization's needs. + +In Collate 1.6, the Glossary Approval Workflow has been migrated to this new framework. Now, you can create custom approval processes with specific conditions and rules and easily visualize them through intuitive workflow diagrams. You can also create smart approval processes for glossary terms with real-time state changes and task creation to save time and streamline work. `, + isImage: false, + path: 'https://www.youtube.com/embed/yKJNWUb_ucA', + }, + { + title: + 'Data Certification Workflows for Automated Bronze, Silver, & Gold Data Standardization (Collate)', + description: `Collate 1.6 also leverages the new data governance framework for a new Data Certification Workflow, allowing you to define your organization's rules to certify your data as Bronze, Silver, or Gold. Certified assets are a great way to help users discover the right data and inform them which data has been properly curated. + +Our vision is to expand our governance framework to allow our users to create their own Custom Governance workflows. We want to enable data teams to implement and automate data governance processes that perfectly fit your organization, promoting data quality and compliance.`, + isImage: false, + path: 'https://www.youtube.com/embed/hqxtn6uAvt4', + }, + { + title: + 'Maintaining a Healthy Data Platform with Observability Dashboards (Collate)', + description: `Monitoring data quality and incident management across platforms can be challenging. OpenMetadata has been a pillar for data quality implementations, with its ability to create tests from the UI, native observability alerts, and Incident Manager. It offers data quality insights on a per-table level. . + +In Collate 1.6, we’re introducing platform-wide observability dashboards that allow you to track overall data quality coverage trends and analyze incident response performance across your entire data estate. Quickly identify root causes through enhanced asset and lineage views and enable proactive data quality management across your entire data ecosystem.`, + isImage: false, + path: 'https://www.youtube.com/embed/DQ-abGXOsHE', + }, + { + title: 'Elevating Metric Management with Dedicated Metric Entities', + description: `Metrics are essential for data-driven organizations, but OpenMetadata previously lacked dedicated metric management, forcing users to use glossary terms as a workaround. The new "Metric" entity in OpenMetadata 1.6 provides a purpose-built solution to:. + +1. Document detailed metric calculations and descriptions + +2. Record calculation formulas and implementation code (Python, Java, SQL, LaTeX) + +3. Visualize metric lineage from source data to insights + +This new addition helps teams better manage, understand, and calculate their business KPIs, for improved data literacy and consistency across data teams. `, + isImage: false, + path: 'https://www.youtube.com/embed/Nf97_oWNAmM', + }, + { + title: 'Reinforcing Data Security with Search RBAC', + description: `OpenMetadata's Roles and Policies enable granular permission control, ensuring appropriate access to metadata across different domains and teams. Some data teams may wish to enable data discovery to search for other tables while still enforcing controls with access requests. Other data teams in more restrictive environments may also wish to control the search experience. + +OpenMetadata 1.6 extends Role-Based Access Control (RBAC) to search functionality, allowing administrators to tailor user search experience. This provides personalized search results, with users only seeing assets they have permission to access, as well as stronger data governance by ensuring users only interact with data within their defined roles and responsibilities.`, + isImage: false, + path: 'https://www.youtube.com/embed/03ke9uv0PG0', + }, + { + title: 'Streamlining Data Management with Additional Enhancements', + description: `Release 1.6 comes with several other notable improvements: + +- **Asynchronous Export APIs** : Enjoy increased efficiency when exporting and importing large datasets with new asynchronous APIs. + +- **Faster Search Re-indexing**: Experience significantly improved performance in search re-indexing, making data discovery even smoother. + +- **Improved Data Insights Custom Dashboards UI (Collate)**: To make it even easier to write your own insights dashboards in Collate. + +- **Slack Integration (Collate)**: Collate is releasing a new Application that lets your users find and share assets directly within your Slack workspace! + +- **Alert Debuggability**: Allowing users to test the destinations and see whenever the alert was triggered.`, + isImage: false, + path: 'https://www.youtube.com/embed/7pUF9ZK2iK4', + }, + { + title: 'Expanded Connector Ecosystem and Diversity', + description: `OpenMetadata’s ingestion framework contains 80+ native connectors. These connectors are the foundation of the platform and bring in all the metadata your team needs: technical metadata, lineage, usage, profiling, etc. + +We bring new connectors in each release, continuously expanding our coverage. This time, release 1.6 comes with seven new connectors: + +1. **OpenAPI**: Extract rich metadata from OpenAPI specifications, including endpoints and schemas. + +2. **Sigma**: Bringing in your BI dashboard information. + +3. **Exasol**: Gain insights into your Exasol database, now supported thanks to Nicola Coretti’s OSS contribution! + +And in Collate, we are bringing four ETL, dashboarding and ML tools: **Matillion, Azure Data Factory, Stitch, PowerBI Server** and **Vertex AI!**`, + isImage: false, + path: '', + }, + ], + changeLogs: { + ['Backward Incompatible Changes']: ` + +**Ingestion Workflow Status:** + +We are updating how we compute the success percentage. Previously, we took into account for partial success the results of the Source (e.g., the tables we were able to properly retrieve from Snowflake, Redshift, etc.). This means that we had an error threshold in there were if up to 90% of the tables were successfully ingested, we would still consider the workflow as successful. However, any errors when sending the information to OpenMetadata would be considered as a failure. +Now, we're changing this behavior to consider the success rate of all the steps involved in the workflow. The UI will then show more Partial Success statuses rather than Failed, properly reflecting the real state of the workflow. + +**Profiler & Auto Classification Workflow:** + +We are creating a new Auto Classification workflow that will take care of managing the sample data and PII classification, which was previously done by the Profiler workflow. This change will allow us to have a more modular and scalable system. +The Profiler workflow will now only focus on the profiling part of the data, while the Auto Classification will take care of the rest. + +This means that we are removing these properties from the DatabaseServiceProfilerPipeline schema: + +- generateSampleData +- processPiiSensitive +- confidence which will be moved to the new +- Adding Glossary Term view is improved. Now we show glossary terms hierarchically enabling a better understanding of how the terms are setup while adding it to a table or dashboard. +- DatabaseServiceAutoClassificationPipeline schema. + +What you will need to do: + +- If you are using the EXTERNAL ingestion for the profiler (YAML configuration), you will need to update your configuration, removing these properties as well. +- If you still want to use the Auto PII Classification and sampling features, you can create the new workflow from the UI. +`, + + ['RBAC Policy Updates for EditTags']: `We have given more granularity to the EditTags policy. Previously, it was a single policy that allowed the user to manage any kind of tagging to the assets, including adding tags, glossary terms, and Tiers. + +Now, we have split this policy to give further control on which kind of tagging the user can manage. The EditTags policy has been +split into: + +- **EditTags**: to add tags. +- **EditGlossaryTerms**: to add Glossary Terms. +- **EditTier**: to add Tier tags.`, + + [`Metadata Actions for ML Tagging - Deprecation Notice ${CollateIconWithLinkMD}`]: ` +Since we are introducing the Auto Classification workflow, we are going to remove in 1.7 the ML Tagging action from the Metadata Actions. That feature will be covered already by the Auto Classification workflow, which even brings more flexibility allow the on-the-fly usage of the sample data for classification purposes without having to store it in the database.`, + + [`Service Spec for the Ingestion Framework`]: `This impacts users who maintain their own connectors for the ingestion framework that are NOT part of the OpenMetadata python library (openmetadata-ingestion). Introducing the "connector specifcication class (ServiceSpec)". The ServiceSpec class serves as the entrypoint for the connector and holds the references for the classes that will be used to ingest and process the metadata from the source. You can see postgres for an implementation example.`, + }, + }, + { + id: 58, + version: 'v1.6.2', + description: 'Released on 10th January 2025.', + features: [], + changeLogs: { + Improvements: `- **Fix**: Test case getting removed from logical test suite after editing the test case. +- **Fix**: Edit Lineage Operation not working with isOwner() condition +- **Fix**: EditLineage permission not allowing users to edit the lineage. +- **Fix**: ViewAll permission not working with matchAnyTag() and isOwner() conditions +- **Fix**: Vulnerability security on 1.5.6 version package com.google.protobuf_protobuf-java. +- **Fix**: DBT Data ingestion not working. +- **Fix**: Table owners not shown properly after a dbt ingestion and re-indexing. +- **Fix**: Glossary Listing Limits to 50 without scrolling to next page. +- **Fix**: Mask encrypted password for email. +- **Fix**: Profiler failing on ingesting data type for postgres. +- **Fix**: Column lineage ingestion failed to parse column due to subquery raw_name AttributeError. +- **Fix**: Data Insight Tier Filter does not work. +- **Fix**: Add depth support for storage connector. +- **Fix**: Replace the description editor with a new block editor. +- **Fix**: Redshift Metadata ingestion failing for Stored Procedure. +- **Fix**: Lineage view not showing all the nodes in case of circular lineage. +- **Fix**: Deleting Data Product should delete the data asset relationships. +- **Fix**: styling (color, icon) is lost if a glossaryTerm is updated via the bulk upload. +- **Fix**: Unable to see complete column type info for long column type. +- **Fix**: ApiEndpoint reindexing failure. +- **Fix**: Auto Classification Ingestion - AttributeError: 'DataType' object has no attribute 'dialect_impl'. +- **Fix**: Adding the profiler for doris failing to execute. +- **Fix**: Unable to remove existing values from custom property (enum data type). +- **Improvement**: Ability to sort the DI charts based on date or term. +- **Improvement**: Support test connection api cancellation on click of cancel. +- **Improvement**: Highlight the search term for schema table on table details page. +- **Improvement**: Add Algorithm option for authentication token validation in yaml. +- **Improvement**: Make all Test Suites executable. +- **Improvement**: Activity feed pagination. +- **Fix**: Custom DI description getting added with HTML p tag. ${CollateIconWithLinkMD} +- **Fix**: Knowledge Page hierarchy state doesn't persist on refresh. ${CollateIconWithLinkMD} +- **Fix**: Reindex Page Entitiy is Missing on Collate. ${CollateIconWithLinkMD} +- **Fix**: Avoid pluralizing for custom charts. ${CollateIconWithLinkMD} +- **Improvement**: Add the missing filters for different assets in the Automator(Ex. Database filter for Database Schema asset). ${CollateIconWithLinkMD} +- **Improvement**: Add Glossary Term and Metric as assets for Automation. ${CollateIconWithLinkMD} +`, + }, + }, + { + id: 59, + version: 'v1.6.3', + description: 'Released on 29th January 2025.', + features: [], + changeLogs: { + Improvements: `- **Fix**: Adds percona server for postgresql support. +- **Fix**: Inherited Ownership for Data Products. +- **Fix**: Favicon not being updated in the browser tab. +- **Fix**: Fix Search Index for ER Model. +- **Fix**: dbt ingestion picks up wrong service to patch metadata. +- **Fix**: Wrong team count displayed on team tab. +- **Fix**: Tracing highlighter in lineage after edge clicked. +- **Fix**: Api should not called after time out in Test connection. +- **Fix**: Get only non-deleted entities in export. +- **Fix**: The permissions call made for search service. +- **Fix**: Kafkaconnect validation errors. +- **Fix**: DI Filter not getting applied. +- **Fix**: Redash Get Dashboards flow. +- **Fix**: Description not rendered in Glossary Modal while edit. +- **Fix**: The persona JSON schema is named Team. +- **Fix**: Redirection issue on IDP initiated calls. +- **Fix**: Async export csv not happening in lineage. +- **Fix**: Description renderer having tags in glossary,team and user import. +- **Fix**: RichTextEditor output in case on no data save. +- **Fix**: s3 storage parquet structureFormat ingestion. +- **Fix**: Data Insights index mapping. +- **Fix**: Edit description permission for domain owner. +- **Fix**: Model dump dict key names. +- **Fix**: Broken looker lineage. +- **Fix**: Refresh call concurrency for multiple browser tabs. +- **Fix**: Infinite loading for refresh attempted on app visit. +- **Fix**: Duplicate table constraints. +- **Fix**: Updated MSSQL queries causing arithmetic overflow error. +- **Fix**: PowerBI tables, datamodel metadata missing. +- **Fix**: Wrong dataset and project id in filter of system metric query. +- **Fix**: Data Insight fix custom property filter. +- **Fix**: Entity Hierarchy Schema. +- **Fix**: Salesforce column description with toggle api. +- **Fix**: Update glossary term table upon new term added. +- **Fix**: Remove unwanted spacing around the list in block editor. +- **Fix**: Postgres parse json schema. +- **Fix**: Optimize multithreading for lineage. +- **Fix**: Fetch Stored Procedures from account usage . +- **Fix**: Add MaterializedView & DynamicTable for lineage computation. +- **Fix**: MariaDB Lineage Dialect Issue. +- **Fix**: DQ Dashboard: update order of the pie chart. ${CollateIconWithLinkMD} +- **Fix**: Lineage Propagation when Entity doesn't have a given field. ${CollateIconWithLinkMD} +- **Minor**: Optimize Snowflake SP Query. +- **Minor**: Hide description tooltip for tag edit mode. +- **Minor**: BigQuery Improvement, Hive Partitioned Tables, Nonetype issue resolved +- **Minor**: Typo for datetime attribute. +- **Minor**: Get missing dataProducts and pipeline properties in /customProperties api. +- **Minor**: Improve cron expression validations. +- **Minor**: Change log localization improvement. +- **Minor**: Async test case result deletion. +- **Minor**: Retention period 'Cancel' international display issue. +- **Minor**: Added limits configuration in telemetry payload. ${CollateIconWithLinkMD} +- **Improvement**: Logout user on unsuccessful refresh attempt. +- **Improvement**: Support for Domain hierarchy listing. +- **Improvement**: Avoid usage of CONCAT in WHERE clause. +- **Improvement**: Glossary column width sizes for the resizable columns. +- **Improvement**: Move Recreate Out of executors. +- **Improvement**: Supported the task filter on landing page feed widget. +- **Improvement**: Implement Data Quality Dashboards (Incident Manager + Data Quality). +- **Improvement**: Added loading state, and manage error notification in TestSuite. +- **Improvement**: Enhance Kafka SSL configuration support with consumerConfigSSL. +- **Improvement**: Add prometheus counter for search and database. +- **Improvement**: Retention Application : Delete change_events, activity threads, versions based on admin retention policies. +- **Improvement**: Show displayName for custom dashboards. ${CollateIconWithLinkMD} +- **Improvement**: Support rename for custom dashboard and charts. ${CollateIconWithLinkMD} +- **Improvement**: Improve Onboarding Application. ${CollateIconWithLinkMD} +`, + }, + }, + { + id: 60, + version: 'v1.6.4', + description: 'Released on 19th February 2025.', + features: [], + changeLogs: { + Improvements: `- **Improvement**: Trino Add missing import. +- **Improvement**: Optimise Pipeline Lineage Extraction. +- **Improvement**: Powerbi fetch workspaces failure handle. +- **Fix**: Powerbi test connection sucess with bad credentials. +- **Fix**: Remove description check for columnDescriptionStatus. +- **Fix**: Markdown editor fix. +- **Fix**: Postgres usage not terminating with bad connection. +- **Fix**: Fix followers for Data Insights index. +- **Fix**: Add support for temp table lineage. +- **Fix**: Exclude deleted Stored Procedure Snowflake. +- **Fix**: Fix databricks schema not found. +- **Fix**: API service schema fields of object type not listed. +- **Fix**: Multiple Tier selection not resulting correct DQ dashboard view. +- **Fix**: Not able to edit sql query from test case details page. +- **Fix**: Implement the right SQA Sampler for UnityCatalog. +- **Fix**: Fix dbt Test case Timestamp issue. +- **Fix**: Delete pipelines from logical suites at deletion. +- **Fix**: Table Update Sys Metric shows wrong value. +- **Fix**: Fix unity catalog lineage - handle errors. +- **Improvement**: Validate basic suites do have basicEntityRef. +- **Improvement**: Add support for cluster key information - bigquery. +- **Improvement**: Automator - Remove tags by label type. +- **Improvement**: Show sub domain assets to top level. +- **Improvement**: Sort Enum type Custom Property Values. +- **Improvement**: Modify the appeariance of self connecting edge lineage. +- **Improvement**: Global search should persist quick filter in explore. +- **Improvement**: Show sourceUrl if present. +- **Improvement**: Modify the lineage alignment algorithm to tree view. + + +`, + }, + }, ]; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MyData/CustomizableComponents/AddDetailsPageWidgetModal/AddDetailsPageWidgetModal.tsx b/openmetadata-ui/src/main/resources/ui/src/components/MyData/CustomizableComponents/AddDetailsPageWidgetModal/AddDetailsPageWidgetModal.tsx deleted file mode 100644 index 0b19ee104540..000000000000 --- a/openmetadata-ui/src/main/resources/ui/src/components/MyData/CustomizableComponents/AddDetailsPageWidgetModal/AddDetailsPageWidgetModal.tsx +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Copyright 2023 Collate. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { CheckOutlined } from '@ant-design/icons'; -import { Modal, Space, Tabs, TabsProps } from 'antd'; -import { isEmpty, toString } from 'lodash'; -import { default as React, useCallback, useMemo } from 'react'; -import { useTranslation } from 'react-i18next'; -import { LIGHT_GREEN_COLOR } from '../../../../constants/constants'; -import { - CommonWidgetType, - GridSizes, -} from '../../../../constants/CustomizeWidgets.constants'; -import { ERROR_PLACEHOLDER_TYPE } from '../../../../enums/common.enum'; -import { WidgetWidths } from '../../../../enums/CustomizablePage.enum'; -import { Document } from '../../../../generated/entity/docStore/document'; -import { getWidgetWidthLabelFromKey } from '../../../../utils/CustomizableLandingPageUtils'; -import ErrorPlaceHolder from '../../../common/ErrorWithPlaceholder/ErrorPlaceHolder'; -import { WidgetSizeInfo } from '../AddWidgetModal/AddWidgetModal.interface'; -import AddWidgetTabContent from '../AddWidgetModal/AddWidgetTabContent'; - -interface Props { - open: boolean; - maxGridSizeSupport: number; - placeholderWidgetKey: string; - addedWidgetsList: Array; - handleCloseAddWidgetModal: () => void; - handleAddWidget: ( - widget: CommonWidgetType, - widgetKey: string, - widgetSize: number - ) => void; - widgetsList: Array; -} - -function AddDetailsPageWidgetModal({ - open, - widgetsList, - addedWidgetsList, - handleCloseAddWidgetModal, - handleAddWidget, - maxGridSizeSupport, - placeholderWidgetKey, -}: Readonly) { - const { t } = useTranslation(); - - const getAddWidgetHandler = useCallback( - (widget: Document, widgetSize: number) => () => - handleAddWidget( - widget as unknown as CommonWidgetType, - placeholderWidgetKey, - widgetSize - ), - [handleAddWidget, placeholderWidgetKey] - ); - - const tabItems: TabsProps['items'] = useMemo( - () => - widgetsList?.map((widget) => { - const widgetSizeOptions: Array = - widget.data.gridSizes.map((size: GridSizes) => ({ - label: ( - - {getWidgetWidthLabelFromKey(toString(size))} - - ), - value: WidgetWidths[size], - })); - - return { - label: ( - - {widget.name} - {addedWidgetsList.some( - (w) => - w.startsWith(widget.fullyQualifiedName) && - !w.includes('EmptyWidgetPlaceholder') - ) && ( - - )} - - ), - key: widget.fullyQualifiedName, - children: ( - - ), - }; - }), - [widgetsList, addedWidgetsList, getAddWidgetHandler, maxGridSizeSupport] - ); - - const widgetsInfo = useMemo(() => { - if (isEmpty(widgetsList)) { - return ( - - {t('message.no-widgets-to-add')} - - ); - } - - return ( - - ); - }, [widgetsList, tabItems]); - - return ( - - {widgetsInfo} - - ); -} - -export default AddDetailsPageWidgetModal; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MyData/CustomizableComponents/AddWidgetModal/AddWidgetTabContent.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/MyData/CustomizableComponents/AddWidgetModal/AddWidgetTabContent.test.tsx index 276df5dba826..22fe5a7a5aa2 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/MyData/CustomizableComponents/AddWidgetModal/AddWidgetTabContent.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/MyData/CustomizableComponents/AddWidgetModal/AddWidgetTabContent.test.tsx @@ -28,7 +28,7 @@ const mockProps: AddWidgetTabContentProps = { widgetSizeOptions: mockWidgetSizes, }; -jest.mock('../../../../utils/CustomizeMyDataPageClassBase', () => ({ +jest.mock('../../../../utils/CustomizePageClassBase', () => ({ getWidgetImageFromKey: jest.fn().mockImplementation(() => ''), })); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MyData/CustomizableComponents/AddWidgetModal/AddWidgetTabContent.tsx b/openmetadata-ui/src/main/resources/ui/src/components/MyData/CustomizableComponents/AddWidgetModal/AddWidgetTabContent.tsx index 50d44a2f95bb..2d02ad294156 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/MyData/CustomizableComponents/AddWidgetModal/AddWidgetTabContent.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/MyData/CustomizableComponents/AddWidgetModal/AddWidgetTabContent.tsx @@ -25,7 +25,7 @@ import { } from 'antd'; import React, { useCallback, useMemo, useState } from 'react'; import { useTranslation } from 'react-i18next'; -import customizePageClassBase from '../../../../utils/CustomizeMyDataPageClassBase'; +import customizePageClassBase from '../../../../utils/CustomizePageClassBase'; import { AddWidgetTabContentProps } from './AddWidgetModal.interface'; function AddWidgetTabContent({ diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MyData/CustomizableComponents/CustomiseGlossaryTermDetailPage/CustomiseGlossaryTermDetailPage.tsx b/openmetadata-ui/src/main/resources/ui/src/components/MyData/CustomizableComponents/CustomiseGlossaryTermDetailPage/CustomiseGlossaryTermDetailPage.tsx deleted file mode 100644 index 772fcfc3174b..000000000000 --- a/openmetadata-ui/src/main/resources/ui/src/components/MyData/CustomizableComponents/CustomiseGlossaryTermDetailPage/CustomiseGlossaryTermDetailPage.tsx +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright 2023 Collate. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import React, { useCallback, useState } from 'react'; -import { useTranslation } from 'react-i18next'; -import gridBgImg from '../../../../assets/img/grid-bg-img.png'; -import { Page } from '../../../../generated/system/ui/page'; -import { useGridLayoutDirection } from '../../../../hooks/useGridLayoutDirection'; -import { WidgetConfig } from '../../../../pages/CustomizablePage/CustomizablePage.interface'; -import { useCustomizeStore } from '../../../../pages/CustomizablePage/CustomizeStore'; -import '../../../../pages/MyDataPage/my-data.less'; -import customizeGlossaryTermPageClassBase from '../../../../utils/CustomiseGlossaryTermPage/CustomizeGlossaryTermPage'; -import { - getLayoutWithEmptyWidgetPlaceholder, - getUniqueFilteredLayout, -} from '../../../../utils/CustomizableLandingPageUtils'; -import { getEntityName } from '../../../../utils/EntityUtils'; -import { CustomizeTabWidget } from '../../../Glossary/CustomiseWidgets/CustomizeTabWidget/CustomizeTabWidget'; -import { GlossaryHeaderWidget } from '../../../Glossary/GlossaryHeader/GlossaryHeaderWidget'; -import PageLayoutV1 from '../../../PageLayoutV1/PageLayoutV1'; -import { CustomizablePageHeader } from '../CustomizablePageHeader/CustomizablePageHeader'; -import { CustomizeMyDataProps } from '../CustomizeMyData/CustomizeMyData.interface'; - -function CustomizeGlossaryTermDetailPage({ - personaDetails, - onSaveLayout, - isGlossary, -}: Readonly) { - const { t } = useTranslation(); - const { currentPage, currentPageType } = useCustomizeStore(); - - const [layout, setLayout] = useState>( - (currentPage?.layout as WidgetConfig[]) ?? - customizeGlossaryTermPageClassBase.defaultLayout - ); - - const handleReset = useCallback(async () => { - // Get default layout with the empty widget added at the end - const newMainPanelLayout = getLayoutWithEmptyWidgetPlaceholder( - customizeGlossaryTermPageClassBase.defaultLayout, - 2, - 4 - ); - setLayout(newMainPanelLayout); - await onSaveLayout(); - }, []); - - const handleSave = async () => { - await onSaveLayout({ - ...(currentPage ?? ({ pageType: currentPageType } as Page)), - layout: getUniqueFilteredLayout(layout), - }); - }; - - // call the hook to set the direction of the grid layout - useGridLayoutDirection(); - - return ( - - - - - - ); -} - -export default CustomizeGlossaryTermDetailPage; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MyData/CustomizableComponents/CustomizablePageHeader/CustomizablePageHeader.tsx b/openmetadata-ui/src/main/resources/ui/src/components/MyData/CustomizableComponents/CustomizablePageHeader/CustomizablePageHeader.tsx deleted file mode 100644 index 2257b40ed277..000000000000 --- a/openmetadata-ui/src/main/resources/ui/src/components/MyData/CustomizableComponents/CustomizablePageHeader/CustomizablePageHeader.tsx +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Copyright 2024 Collate. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import { Button, Col, Modal, Space, Typography } from 'antd'; -import { startCase } from 'lodash'; -import React, { useCallback, useMemo } from 'react'; -import { useTranslation } from 'react-i18next'; -import { Link, useHistory } from 'react-router-dom'; -import { useApplicationStore } from '../../../../hooks/useApplicationStore'; -import { useFqn } from '../../../../hooks/useFqn'; -import { useCustomizeStore } from '../../../../pages/CustomizablePage/CustomizeStore'; -import { Transi18next } from '../../../../utils/CommonUtils'; -import { getPersonaDetailsPath } from '../../../../utils/RouterUtils'; - -export const CustomizablePageHeader = ({ - onReset, - onSave, - personaName, -}: { - onSave: () => Promise; - onReset: () => void; - personaName: string; -}) => { - const { t } = useTranslation(); - const { fqn: personaFqn } = useFqn(); - const { currentPageType } = useCustomizeStore(); - const history = useHistory(); - const [isResetModalOpen, setIsResetModalOpen] = React.useState(false); - const [saving, setSaving] = React.useState(false); - const { theme } = useApplicationStore(); - - const handleCancel = () => { - history.push(getPersonaDetailsPath(personaFqn)); - }; - - const handleOpenResetModal = useCallback(() => { - setIsResetModalOpen(true); - }, []); - - const handleCloseResetModal = useCallback(() => { - setIsResetModalOpen(false); - }, []); - - const handleSave = useCallback(async () => { - setSaving(true); - await onSave(); - setSaving(false); - }, [onSave]); - - const handleReset = useCallback(async () => { - onReset(); - setIsResetModalOpen(false); - }, [onReset]); - const i18Values = useMemo( - () => ({ - persona: personaName, - pageName: startCase(currentPageType as string) ?? t('label.landing-page'), - }), - [personaName] - ); - - return ( -
-
- - - } - values={i18Values} - /> - -
- - - - - - {isResetModalOpen && ( - - {t('message.reset-layout-confirmation')} - - )} - - ); -}; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MyData/CustomizableComponents/CustomizeMyData/CustomizeMyData.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/MyData/CustomizableComponents/CustomizeMyData/CustomizeMyData.interface.ts index b867f91d0da1..18d402af23a0 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/MyData/CustomizableComponents/CustomizeMyData/CustomizeMyData.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/MyData/CustomizableComponents/CustomizeMyData/CustomizeMyData.interface.ts @@ -11,12 +11,13 @@ * limitations under the License. */ +import { Document } from '../../../../generated/entity/docStore/document'; import { Persona } from '../../../../generated/entity/teams/persona'; -import { Page } from '../../../../generated/system/ui/page'; export interface CustomizeMyDataProps { personaDetails?: Persona; - isGlossary?: boolean; - initialPageData: Page | null; - onSaveLayout: (page?: Page) => Promise; + initialPageData: Document; + onSaveLayout: () => Promise; + handlePageDataChange: (newPageData: Document) => void; + handleSaveCurrentPageLayout: (value: boolean) => void; } diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MyData/CustomizableComponents/CustomizeMyData/CustomizeMyData.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/MyData/CustomizableComponents/CustomizeMyData/CustomizeMyData.test.tsx index 5a42979f77a9..1791575e2ebc 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/MyData/CustomizableComponents/CustomizeMyData/CustomizeMyData.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/MyData/CustomizableComponents/CustomizeMyData/CustomizeMyData.test.tsx @@ -18,18 +18,22 @@ import { PageType } from '../../../../generated/system/ui/page'; import { mockActiveAnnouncementData, mockCustomizePageClassBase, + mockDefaultLayout, mockDocumentData, mockPersonaName, mockUserData, } from '../../../../mocks/MyDataPage.mock'; +import { WidgetConfig } from '../../../../pages/CustomizablePage/CustomizablePage.interface'; import CustomizeMyData from './CustomizeMyData'; import { CustomizeMyDataProps } from './CustomizeMyData.interface'; const mockPush = jest.fn(); const mockProps: CustomizeMyDataProps = { - initialPageData: mockDocumentData.data.pages[0], + initialPageData: mockDocumentData, onSaveLayout: jest.fn(), + handlePageDataChange: jest.fn(), + handleSaveCurrentPageLayout: jest.fn(), }; jest.mock( @@ -64,7 +68,7 @@ jest.mock( } ); -jest.mock('../../../../utils/CustomizeMyDataPageClassBase', () => { +jest.mock('../../../../utils/CustomizePageClassBase', () => { return mockCustomizePageClassBase; }); @@ -161,7 +165,9 @@ describe('CustomizeMyData component', () => { await act(async () => userEvent.click(cancelButton)); - expect(mockPush).toHaveBeenCalledWith('/settings/persona/testPersona'); + expect(mockPush).toHaveBeenCalledWith( + '/settings/preferences/customizeLandingPage' + ); }); it('CustomizeMyData should display reset layout confirmation modal on click of reset button', async () => { @@ -181,6 +187,9 @@ describe('CustomizeMyData component', () => { render(); }); + // handlePageDataChange is called 1 time on mount + expect(mockProps.handlePageDataChange).toHaveBeenCalledTimes(1); + const resetButton = screen.getByTestId('reset-button'); await act(async () => userEvent.click(resetButton)); @@ -191,6 +200,19 @@ describe('CustomizeMyData component', () => { await act(async () => userEvent.click(yesButton)); + expect(mockProps.handlePageDataChange).toHaveBeenCalledTimes(3); + // Check if the handlePageDataChange is passed an object with the default layout + expect(mockProps.handlePageDataChange).toHaveBeenCalledWith( + expect.objectContaining({ + ...mockDocumentData, + data: { + page: { + layout: expect.arrayContaining(mockDefaultLayout), + }, + }, + }) + ); + expect(screen.queryByTestId('reset-layout-modal')).toBeNull(); }); @@ -199,6 +221,9 @@ describe('CustomizeMyData component', () => { render(); }); + // handlePageDataChange is called 1 time on mount + expect(mockProps.handlePageDataChange).toHaveBeenCalledTimes(1); + const resetButton = screen.getByTestId('reset-button'); await act(async () => userEvent.click(resetButton)); @@ -209,6 +234,9 @@ describe('CustomizeMyData component', () => { await act(async () => userEvent.click(noButton)); + // handlePageDataChange is not called again + expect(mockProps.handlePageDataChange).toHaveBeenCalledTimes(1); + expect(screen.queryByTestId('reset-layout-modal')).toBeNull(); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MyData/CustomizableComponents/CustomizeMyData/CustomizeMyData.tsx b/openmetadata-ui/src/main/resources/ui/src/components/MyData/CustomizableComponents/CustomizeMyData/CustomizeMyData.tsx index b875cfa3aded..5d2fb94458e2 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/MyData/CustomizableComponents/CustomizeMyData/CustomizeMyData.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/MyData/CustomizableComponents/CustomizeMyData/CustomizeMyData.tsx @@ -11,24 +11,30 @@ * limitations under the License. */ +import { Button, Col, Modal, Space, Typography } from 'antd'; import { AxiosError } from 'axios'; -import { isEmpty } from 'lodash'; +import { isEmpty, isNil } from 'lodash'; import React, { useCallback, useEffect, useMemo, useState } from 'react'; import RGL, { Layout, WidthProvider } from 'react-grid-layout'; import { useTranslation } from 'react-i18next'; +import { Link, useHistory } from 'react-router-dom'; import gridBgImg from '../../../../assets/img/grid-bg-img.png'; import { KNOWLEDGE_LIST_LENGTH } from '../../../../constants/constants'; +import { + GlobalSettingOptions, + GlobalSettingsMenuCategory, +} from '../../../../constants/GlobalSettings.constants'; import { LandingPageWidgetKeys } from '../../../../enums/CustomizablePage.enum'; import { SearchIndex } from '../../../../enums/search.enum'; import { Document } from '../../../../generated/entity/docStore/document'; import { EntityReference } from '../../../../generated/entity/type'; -import { Page } from '../../../../generated/system/ui/page'; -import { PageType } from '../../../../generated/system/ui/uiCustomization'; import { useApplicationStore } from '../../../../hooks/useApplicationStore'; +import { useFqn } from '../../../../hooks/useFqn'; import { useGridLayoutDirection } from '../../../../hooks/useGridLayoutDirection'; import { WidgetConfig } from '../../../../pages/CustomizablePage/CustomizablePage.interface'; import '../../../../pages/MyDataPage/my-data.less'; import { searchQuery } from '../../../../rest/searchAPI'; +import { Transi18next } from '../../../../utils/CommonUtils'; import { getAddWidgetHandler, getLayoutUpdateHandler, @@ -37,13 +43,16 @@ import { getUniqueFilteredLayout, getWidgetFromKey, } from '../../../../utils/CustomizableLandingPageUtils'; -import customizeMyDataPageClassBase from '../../../../utils/CustomizeMyDataPageClassBase'; +import customizePageClassBase from '../../../../utils/CustomizePageClassBase'; import { getEntityName } from '../../../../utils/EntityUtils'; +import { + getPersonaDetailsPath, + getSettingPath, +} from '../../../../utils/RouterUtils'; import { showErrorToast } from '../../../../utils/ToastUtils'; import ActivityFeedProvider from '../../../ActivityFeed/ActivityFeedProvider/ActivityFeedProvider'; import PageLayoutV1 from '../../../PageLayoutV1/PageLayoutV1'; import AddWidgetModal from '../AddWidgetModal/AddWidgetModal'; -import { CustomizablePageHeader } from '../CustomizablePageHeader/CustomizablePageHeader'; import './customize-my-data.less'; import { CustomizeMyDataProps } from './CustomizeMyData.interface'; @@ -53,14 +62,17 @@ function CustomizeMyData({ personaDetails, initialPageData, onSaveLayout, + handlePageDataChange, + handleSaveCurrentPageLayout, }: Readonly) { const { t } = useTranslation(); - const { currentUser } = useApplicationStore(); - + const { currentUser, theme } = useApplicationStore(); + const history = useHistory(); + const { fqn: decodedPersonaFQN } = useFqn(); const [layout, setLayout] = useState>( getLayoutWithEmptyWidgetPlaceholder( - (initialPageData?.layout as WidgetConfig[]) ?? - customizeMyDataPageClassBase.defaultLayout, + initialPageData.data?.page?.layout ?? + customizePageClassBase.defaultLayout, 2, 4 ) @@ -70,10 +82,11 @@ function CustomizeMyData({ LandingPageWidgetKeys.EMPTY_WIDGET_PLACEHOLDER ); const [isWidgetModalOpen, setIsWidgetModalOpen] = useState(false); - + const [isResetModalOpen, setIsResetModalOpen] = useState(false); const [followedData, setFollowedData] = useState>([]); const [followedDataCount, setFollowedDataCount] = useState(0); const [isLoadingOwnedData, setIsLoadingOwnedData] = useState(false); + const [saving, setSaving] = useState(false); const handlePlaceholderWidgetKey = useCallback((value: string) => { setPlaceholderWidgetKey(value); @@ -94,7 +107,7 @@ function CustomizeMyData({ newWidgetData, placeholderWidgetKey, widgetSize, - customizeMyDataPageClassBase.landingPageMaxGridSize + customizePageClassBase.landingPageMaxGridSize ) ); setIsWidgetModalOpen(false); @@ -111,6 +124,14 @@ function CustomizeMyData({ [layout] ); + const handleOpenResetModal = useCallback(() => { + setIsResetModalOpen(true); + }, []); + + const handleCloseResetModal = useCallback(() => { + setIsResetModalOpen(false); + }, []); + const handleOpenAddWidgetModal = useCallback(() => { setIsWidgetModalOpen(true); }, []); @@ -176,15 +197,44 @@ function CustomizeMyData({ ] ); + useEffect(() => { + handlePageDataChange({ + ...initialPageData, + data: { + page: { + layout: getUniqueFilteredLayout(layout), + }, + }, + }); + }, [layout]); + + const handleCancel = useCallback(() => { + history.push( + getSettingPath( + GlobalSettingsMenuCategory.PREFERENCES, + GlobalSettingOptions.CUSTOMIZE_LANDING_PAGE + ) + ); + }, []); + const handleReset = useCallback(() => { // Get default layout with the empty widget added at the end const newMainPanelLayout = getLayoutWithEmptyWidgetPlaceholder( - customizeMyDataPageClassBase.defaultLayout, + customizePageClassBase.defaultLayout, 2, 4 ); setLayout(newMainPanelLayout); - onSaveLayout(); + handlePageDataChange({ + ...initialPageData, + data: { + page: { + layout: getUniqueFilteredLayout(newMainPanelLayout), + }, + }, + }); + handleSaveCurrentPageLayout(true); + setIsResetModalOpen(false); }, []); useEffect(() => { @@ -192,13 +242,10 @@ function CustomizeMyData({ }, []); const handleSave = async () => { - await onSaveLayout({ - ...(initialPageData ?? - ({ - pageType: PageType.LandingPage, - } as Page)), - layout: getUniqueFilteredLayout(layout), - }); + setSaving(true); + await onSaveLayout(); + + setSaving(false); }; // call the hook to set the direction of the grid layout @@ -207,6 +254,59 @@ function CustomizeMyData({ return ( +
+ + + } + values={{ + persona: isNil(personaDetails) + ? decodedPersonaFQN + : getEntityName(personaDetails), + }} + /> + +
+ + + + + + + } + headerClassName="m-0 p-0" mainContainerClassName="p-t-0" pageContainerStyle={{ backgroundImage: `url(${gridBgImg})`, @@ -214,21 +314,16 @@ function CustomizeMyData({ pageTitle={t('label.customize-entity', { entity: t('label.landing-page'), })}> - {widgets} @@ -239,13 +334,24 @@ function CustomizeMyData({ addedWidgetsList={addedWidgetsList} handleAddWidget={handleMainPanelAddWidget} handleCloseAddWidgetModal={handleCloseAddWidgetModal} - maxGridSizeSupport={ - customizeMyDataPageClassBase.landingPageMaxGridSize - } + maxGridSizeSupport={customizePageClassBase.landingPageMaxGridSize} open={isWidgetModalOpen} placeholderWidgetKey={placeholderWidgetKey} /> )} + {isResetModalOpen && ( + + {t('message.reset-layout-confirmation')} + + )} ); } diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MyData/LeftSidebar/LeftSidebar.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/MyData/LeftSidebar/LeftSidebar.component.tsx index 78fc05cd0ec1..f08046d34e6c 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/MyData/LeftSidebar/LeftSidebar.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/MyData/LeftSidebar/LeftSidebar.component.tsx @@ -13,8 +13,8 @@ import { Button, Col, Menu, MenuProps, Row, Typography } from 'antd'; import Modal from 'antd/lib/modal/Modal'; import classNames from 'classnames'; -import { isEmpty, noop } from 'lodash'; -import React, { useCallback, useEffect, useMemo, useState } from 'react'; +import { noop } from 'lodash'; +import React, { useCallback, useMemo, useState } from 'react'; import { useTranslation } from 'react-i18next'; import { Link } from 'react-router-dom'; import { @@ -25,15 +25,8 @@ import { import { SidebarItem } from '../../../enums/sidebar.enum'; import leftSidebarClassBase from '../../../utils/LeftSidebarClassBase'; -import { EntityType } from '../../../enums/entity.enum'; import { useApplicationStore } from '../../../hooks/useApplicationStore'; import useCustomLocation from '../../../hooks/useCustomLocation/useCustomLocation'; -import { useCustomizeStore } from '../../../pages/CustomizablePage/CustomizeStore'; -import { getDocumentByFQN } from '../../../rest/DocStoreAPI'; -import { - filterAndArrangeTreeByKeys, - getNestedKeysFromNavigationItems, -} from '../../../utils/CustomizaNavigation/CustomizeNavigation'; import BrandImage from '../../common/BrandImage/BrandImage'; import './left-sidebar.less'; import { LeftSidebarItem as LeftSidebarItemType } from './LeftSidebar.interface'; @@ -45,21 +38,8 @@ const LeftSidebar = () => { const { onLogoutHandler } = useApplicationStore(); const [showConfirmLogoutModal, setShowConfirmLogoutModal] = useState(false); const [isSidebarCollapsed, setIsSidebarCollapsed] = useState(true); - const { selectedPersona } = useApplicationStore(); - - const { currentPersonaDocStore, setCurrentPersonaDocStore } = - useCustomizeStore(); - - const navigationItems = useMemo(() => { - return currentPersonaDocStore?.data?.navigation; - }, [currentPersonaDocStore]); - const sideBarItems = isEmpty(navigationItems) - ? leftSidebarClassBase.getSidebarItems() - : filterAndArrangeTreeByKeys( - leftSidebarClassBase.getSidebarItems(), - getNestedKeysFromNavigationItems(navigationItems) - ); + const sideBarItems = leftSidebarClassBase.getSidebarItems(); const selectedKeys = useMemo(() => { const pathArray = location.pathname.split('/'); @@ -78,6 +58,23 @@ const LeftSidebar = () => { setShowConfirmLogoutModal(false); }; + const TOP_SIDEBAR_MENU_ITEMS: MenuProps['items'] = useMemo(() => { + return [ + ...sideBarItems.map((item) => { + return { + key: item.key, + label: , + children: item.children?.map((item: LeftSidebarItemType) => { + return { + key: item.key, + label: , + }; + }), + }; + }), + ]; + }, []); + const LOWER_SIDEBAR_TOP_SIDEBAR_MENU_ITEMS: MenuProps['items'] = useMemo( () => [SETTING_ITEM, LOGOUT_ITEM].map((item) => ({ @@ -106,23 +103,6 @@ const LeftSidebar = () => { setIsSidebarCollapsed(true); }, []); - const fetchCustomizedDocStore = useCallback(async (personaFqn: string) => { - try { - const pageLayoutFQN = `${EntityType.PERSONA}.${personaFqn}`; - - const document = await getDocumentByFQN(pageLayoutFQN); - setCurrentPersonaDocStore(document); - } catch (error) { - // silent error - } - }, []); - - useEffect(() => { - if (selectedPersona.fullyQualifiedName) { - fetchCustomizedDocStore(selectedPersona.fullyQualifiedName); - } - }, [selectedPersona]); - return (
{
{ - return { - key: item.key, - label: , - children: item.children?.map((item: LeftSidebarItemType) => { - return { - key: item.key, - label: , - }; - }), - }; - })} + items={TOP_SIDEBAR_MENU_ITEMS} mode="inline" rootClassName="left-sidebar-menu" selectedKeys={selectedKeys} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MyData/LeftSidebar/LeftSidebar.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/MyData/LeftSidebar/LeftSidebar.interface.ts index 62265f54e5e3..d39658a5589f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/MyData/LeftSidebar/LeftSidebar.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/MyData/LeftSidebar/LeftSidebar.interface.ts @@ -14,7 +14,7 @@ export interface LeftSidebarItem { key: string; isBeta?: boolean; - title: string; + label: string; redirect_url?: string; icon: SvgComponent; dataTestId: string; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MyData/LeftSidebar/LeftSidebarItem.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/MyData/LeftSidebar/LeftSidebarItem.component.tsx index 605d0b86943b..c11c9ddc14f3 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/MyData/LeftSidebar/LeftSidebarItem.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/MyData/LeftSidebar/LeftSidebarItem.component.tsx @@ -19,7 +19,7 @@ import { NavLink } from 'react-router-dom'; interface LeftSidebarItemProps { data: { key: string; - title: string; + label: string; dataTestId: string; redirect_url?: string; icon: SvgComponent; @@ -29,7 +29,7 @@ interface LeftSidebarItemProps { } const LeftSidebarItem = ({ - data: { title, redirect_url, dataTestId, icon, isBeta, onClick }, + data: { label, redirect_url, dataTestId, icon, isBeta, onClick }, }: LeftSidebarItemProps) => { const { t } = useTranslation(); @@ -42,7 +42,7 @@ const LeftSidebarItem = ({ }}>
- {title} + {label} {isBeta && ( - {title} + {label} ); }; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MyData/Persona/PersonaDetailsCard/PersonaDetailsCard.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/MyData/Persona/PersonaDetailsCard/PersonaDetailsCard.test.tsx index d298dcaca2cb..dae6a2919924 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/MyData/Persona/PersonaDetailsCard/PersonaDetailsCard.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/MyData/Persona/PersonaDetailsCard/PersonaDetailsCard.test.tsx @@ -48,9 +48,7 @@ describe('PersonaDetailsCard Component', () => { }); expect( - await screen.findByTestId( - `persona-details-card-${personaWithDescription.name}` - ) + await screen.findByTestId('persona-details-card') ).toBeInTheDocument(); }); @@ -83,7 +81,7 @@ describe('PersonaDetailsCard Component', () => { userEvent.click(personaCardTitle); }); - expect(mockPush).toHaveBeenCalledWith('/settings/persona/john-doe'); + expect(mockPush).toHaveBeenCalledWith('/settings/members/persona/john-doe'); }); it('should not navigate when persona.fullyQualifiedName is missing', async () => { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MyData/Persona/PersonaDetailsCard/PersonaDetailsCard.tsx b/openmetadata-ui/src/main/resources/ui/src/components/MyData/Persona/PersonaDetailsCard/PersonaDetailsCard.tsx index e6153d7f14d2..c74672f2852c 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/MyData/Persona/PersonaDetailsCard/PersonaDetailsCard.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/MyData/Persona/PersonaDetailsCard/PersonaDetailsCard.tsx @@ -17,7 +17,7 @@ import { useHistory } from 'react-router-dom'; import { Persona } from '../../../../generated/entity/teams/persona'; import { getEntityName } from '../../../../utils/EntityUtils'; import { getPersonaDetailsPath } from '../../../../utils/RouterUtils'; -import RichTextEditorPreviewer from '../../../common/RichTextEditor/RichTextEditorPreviewer'; +import RichTextEditorPreviewerV1 from '../../../common/RichTextEditor/RichTextEditorPreviewerV1'; interface PersonaDetailsCardProps { persona: Persona; @@ -37,13 +37,13 @@ export const PersonaDetailsCard = ({ persona }: PersonaDetailsCardProps) => { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MyData/Widgets/FeedsWidget/FeedsWidget.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/MyData/Widgets/FeedsWidget/FeedsWidget.component.tsx index b7ea93167d02..43cdd5660d7c 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/MyData/Widgets/FeedsWidget/FeedsWidget.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/MyData/Widgets/FeedsWidget/FeedsWidget.component.tsx @@ -20,6 +20,7 @@ import { Link, useHistory } from 'react-router-dom'; import { PAGE_SIZE_MEDIUM, ROUTES } from '../../../../constants/constants'; import { FEED_COUNT_INITIAL_DATA } from '../../../../constants/entity.constants'; import { mockFeedData } from '../../../../constants/mockTourData.constants'; +import { TAB_SUPPORTED_FILTER } from '../../../../constants/Widgets.constant'; import { useTourProvider } from '../../../../context/TourProvider/TourProvider'; import { EntityTabs, EntityType } from '../../../../enums/entity.enum'; import { FeedFilter } from '../../../../enums/mydata.enum'; @@ -76,7 +77,7 @@ const FeedsWidget = ({ getFeedData(FeedFilter.MENTIONS); } else if (activeTab === ActivityFeedTabs.TASKS) { getFeedData( - FeedFilter.OWNER, + defaultFilter, undefined, ThreadType.Task, undefined, @@ -106,7 +107,12 @@ const FeedsWidget = ({ [count.openTaskCount, activeTab] ); - const onTabChange = (key: string) => setActiveTab(key as ActivityFeedTabs); + const onTabChange = (key: string) => { + if (key === ActivityFeedTabs.TASKS) { + setDefaultFilter(FeedFilter.OWNER); + } + setActiveTab(key as ActivityFeedTabs); + }; const redirectToUserPage = useCallback(() => { history.push( @@ -259,13 +265,10 @@ const FeedsWidget = ({ ]} tabBarExtraContent={ - {activeTab === ActivityFeedTabs.ALL && ( + {TAB_SUPPORTED_FILTER.includes(activeTab) && ( )} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MyData/Widgets/FeedsWidget/FeedsWidget.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/MyData/Widgets/FeedsWidget/FeedsWidget.test.tsx index 229585548936..4ad32a7a273c 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/MyData/Widgets/FeedsWidget/FeedsWidget.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/MyData/Widgets/FeedsWidget/FeedsWidget.test.tsx @@ -82,7 +82,16 @@ jest.mock( jest.mock( '../../../common/FeedsFilterPopover/FeedsFilterPopover.component', - () => jest.fn().mockImplementation(({ children }) =>

{children}

) + () => + jest.fn().mockImplementation(({ onUpdate }) => ( +
+ + + +
+ )) ); jest.mock('../../../../rest/feedsAPI', () => ({ @@ -193,4 +202,56 @@ describe('FeedsWidget', () => { expect(mockHandleRemoveWidget).toHaveBeenCalledWith(widgetProps.widgetKey); }); + + it('should call api with correct parameters based on the tab selected', () => { + render(); + const tabs = screen.getAllByRole('tab'); + const conversationTab = tabs[0]; + fireEvent.click(conversationTab); + + // initial API call for the Feed + expect(conversationTab.getAttribute('aria-selected')).toBe('true'); + expect(mockUseActivityFeedProviderValue.getFeedData).toHaveBeenCalledWith( + 'OWNER_OR_FOLLOWS', + undefined, + 'Conversation', + undefined, + undefined, + undefined, + 25 + ); + + // Reset mock between checks + mockUseActivityFeedProviderValue.getFeedData.mockReset(); + + // Testing for "Task Tab", to call API with OWNER filter parameters + const taskTab = tabs[2]; + fireEvent.click(taskTab); + + expect(taskTab.getAttribute('aria-selected')).toBe('true'); + expect(mockUseActivityFeedProviderValue.getFeedData).toHaveBeenCalledWith( + 'OWNER', + undefined, + 'Task', + undefined, + undefined, + 'Open' + ); + + // Reset mock for the next check + mockUseActivityFeedProviderValue.getFeedData.mockReset(); + + // Applying the filter for the assigned button + const assignedFilterButton = screen.getByText('assigned_button'); + fireEvent.click(assignedFilterButton); + + expect(mockUseActivityFeedProviderValue.getFeedData).toHaveBeenCalledWith( + 'ASSIGNED_TO', + undefined, + 'Task', + undefined, + undefined, + 'Open' + ); + }); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/NavBar/NavBar.tsx b/openmetadata-ui/src/main/resources/ui/src/components/NavBar/NavBar.tsx index d5cfffd14b79..80bd336ee683 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/NavBar/NavBar.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/NavBar/NavBar.tsx @@ -31,7 +31,7 @@ import { AxiosError } from 'axios'; import classNames from 'classnames'; import { CookieStorage } from 'cookie-storage'; import i18next from 'i18next'; -import { debounce, upperCase } from 'lodash'; +import { debounce, startCase, upperCase } from 'lodash'; import { MenuInfo } from 'rc-menu/lib/interface'; import React, { useCallback, @@ -53,9 +53,12 @@ import { NOTIFICATION_READ_TIMER, SOCKET_EVENTS, } from '../../constants/constants'; +import { GlobalSettingsMenuCategory } from '../../constants/GlobalSettings.constants'; import { HELP_ITEMS_ENUM } from '../../constants/Navbar.constants'; import { useWebSocketConnector } from '../../context/WebSocketProvider/WebSocketProvider'; import { EntityTabs, EntityType } from '../../enums/entity.enum'; +import { EntityReference } from '../../generated/entity/type'; +import { BackgroundJob, JobType } from '../../generated/jobs/backgroundJob'; import { useApplicationStore } from '../../hooks/useApplicationStore'; import useCustomLocation from '../../hooks/useCustomLocation/useCustomLocation'; import { useDomainStore } from '../../hooks/useDomainStore'; @@ -67,6 +70,7 @@ import { shouldRequestPermission, } from '../../utils/BrowserNotificationUtils'; import { refreshPage } from '../../utils/CommonUtils'; +import { getCustomPropertyEntityPathname } from '../../utils/CustomProperty.utils'; import entityUtilClassBase from '../../utils/EntityUtilClassBase'; import { getEntityName } from '../../utils/EntityUtils'; import { @@ -81,6 +85,7 @@ import { import { isCommandKeyPress, Keys } from '../../utils/KeyboardUtil'; import { getHelpDropdownItems } from '../../utils/NavbarUtils'; import { + getSettingPath, inPageSearchOptions, isInPageSearchAllowed, } from '../../utils/RouterUtils'; @@ -90,6 +95,7 @@ import { ActivityFeedTabs } from '../ActivityFeed/ActivityFeedTab/ActivityFeedTa import SearchOptions from '../AppBar/SearchOptions'; import Suggestions from '../AppBar/Suggestions'; import CmdKIcon from '../common/CmdKIcon/CmdKIcon.component'; +import DomainSelectableList from '../common/DomainSelectableList/DomainSelectableList.component'; import { useEntityExportModalProvider } from '../Entity/EntityExportModalProvider/EntityExportModalProvider.component'; import { CSVExportWebsocketResponse } from '../Entity/EntityExportModalProvider/EntityExportModalProvider.interface'; import WhatsNewModal from '../Modals/WhatsNewModal/WhatsNewModal'; @@ -120,12 +126,8 @@ const NavBar = ({ useState(false); const location = useCustomLocation(); const history = useHistory(); - const { - domainOptions, - activeDomain, - activeDomainEntityRef, - updateActiveDomain, - } = useDomainStore(); + const { activeDomain, activeDomainEntityRef, updateActiveDomain } = + useDomainStore(); const { t } = useTranslation(); const { Option } = Select; const searchRef = useRef(null); @@ -138,6 +140,8 @@ const NavBar = ({ const [activeTab, setActiveTab] = useState('Task'); const [isFeatureModalOpen, setIsFeatureModalOpen] = useState(false); const [version, setVersion] = useState(); + const [isDomainDropdownOpen, setIsDomainDropdownOpen] = useState(false); + const domainContainerRef = useRef(null); const fetchOMVersion = async () => { try { @@ -252,15 +256,18 @@ const NavBar = ({ const showBrowserNotification = ( about: string, createdBy: string, - type: string + type: string, + backgroundJobData?: BackgroundJob ) => { if (!hasNotificationPermission()) { return; } + const entityType = getEntityType(about); const entityFQN = getEntityFQN(about) ?? ''; let body; let path: string; + switch (type) { case 'Task': body = t('message.user-assign-new-task', { @@ -280,6 +287,31 @@ const NavBar = ({ user: createdBy, }); path = prepareFeedLink(entityType as string, entityFQN as string); + + break; + + case 'BackgroundJob': { + if (!backgroundJobData) { + break; + } + + const { jobArgs, status, jobType } = backgroundJobData; + + if (jobType === JobType.CustomPropertyEnumCleanup) { + body = t('message.custom-property-update', { + propertyName: jobArgs.propertyName, + entityName: jobArgs.entityType, + status: startCase(status.toLowerCase()), + }); + + path = getSettingPath( + GlobalSettingsMenuCategory.CUSTOM_PROPERTIES, + getCustomPropertyEntityPathname(jobArgs.entityType) + ); + } + + break; + } } const notification = new Notification('Notification From OpenMetadata', { body: body, @@ -362,12 +394,25 @@ const NavBar = ({ onUpdateCSVExportJob(exportResponseData); } }); + socket.on(SOCKET_EVENTS.BACKGROUND_JOB_CHANNEL, (jobResponse) => { + if (jobResponse) { + const jobResponseData: BackgroundJob = JSON.parse(jobResponse); + showBrowserNotification( + '', + jobResponseData.createdBy, + 'BackgroundJob', + jobResponseData + ); + } + }); } return () => { socket && socket.off(SOCKET_EVENTS.TASK_CHANNEL); socket && socket.off(SOCKET_EVENTS.MENTION_CHANNEL); socket && socket.off(SOCKET_EVENTS.CSV_EXPORT_CHANNEL); + socket && socket.off(SOCKET_EVENTS.CSV_EXPORT_CHANNEL); + socket && socket.off(SOCKET_EVENTS.BACKGROUND_JOB_CHANNEL); }; }, [socket]); @@ -375,6 +420,30 @@ const NavBar = ({ fetchOMVersion(); }, []); + useEffect(() => { + const handleClickOutside = (event: MouseEvent) => { + // Check if click is inside the domain-select-popover + const isClickInPopover = (event.target as Element)?.closest( + '.domain-select-popover' + ); + + if ( + domainContainerRef.current && + !domainContainerRef.current.contains(event.target as Node) && + !isClickInPopover && + isDomainDropdownOpen + ) { + setIsDomainDropdownOpen(false); + } + }; + + document.addEventListener('mousedown', handleClickOutside); + + return () => { + document.removeEventListener('mousedown', handleClickOutside); + }; + }, [isDomainDropdownOpen]); + useEffect(() => { const targetNode = document.body; targetNode.addEventListener('keydown', handleKeyPress); @@ -382,10 +451,14 @@ const NavBar = ({ return () => targetNode.removeEventListener('keydown', handleKeyPress); }, [handleKeyPress]); - const handleDomainChange = useCallback(({ key }) => { - updateActiveDomain(key); - refreshPage(); - }, []); + const handleDomainChange = useCallback( + async (domain: EntityReference | EntityReference[]) => { + updateActiveDomain(domain as EntityReference); + setIsDomainDropdownOpen(false); + refreshPage(); + }, + [] + ); const handleLanguageChange = useCallback(({ key }) => { i18next.changeLanguage(key); @@ -507,37 +580,38 @@ const NavBar = ({
- - - - - - - - {activeDomainEntityRef - ? getEntityName(activeDomainEntityRef) - : activeDomain} - - - - - - - +
+ setIsDomainDropdownOpen(false)} + onUpdate={handleDomainChange}> + setIsDomainDropdownOpen(!isDomainDropdownOpen)}> +
+ + + + + {activeDomainEntityRef + ? getEntityName(activeDomainEntityRef) + : activeDomain} + + + + + + + + - - - diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Pipeline/PipelineVersion/PipelineVersion.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Pipeline/PipelineVersion/PipelineVersion.component.tsx index 96265ac21dcd..134f87b8a03f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Pipeline/PipelineVersion/PipelineVersion.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Pipeline/PipelineVersion/PipelineVersion.component.tsx @@ -37,7 +37,7 @@ import { getFilterTags } from '../../../utils/TableTags/TableTags.utils'; import { CustomPropertyTable } from '../../common/CustomPropertyTable/CustomPropertyTable'; import DescriptionV1 from '../../common/EntityDescription/DescriptionV1'; import Loader from '../../common/Loader/Loader'; -import RichTextEditorPreviewer from '../../common/RichTextEditor/RichTextEditorPreviewer'; +import RichTextEditorPreviewerV1 from '../../common/RichTextEditor/RichTextEditorPreviewerV1'; import TabsLabel from '../../common/TabsLabel/TabsLabel.component'; import DataAssetsVersionHeader from '../../DataAssets/DataAssetsVersionHeader/DataAssetsVersionHeader'; import DataProductsContainer from '../../DataProducts/DataProductsContainer/DataProductsContainer.component'; @@ -111,14 +111,14 @@ const PipelineVersion: FC = ({ key: 'name', width: 250, render: (_, record) => ( - + ), }, { title: t('label.task-entity', { entity: t('label.type-lowercase') }), dataIndex: 'taskType', key: 'taskType', - render: (taskType) => , + render: (taskType) => , }, { title: t('label.description'), @@ -126,7 +126,7 @@ const PipelineVersion: FC = ({ key: 'description', render: (text) => text ? ( - + ) : ( {t('label.no-description')} ), diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Pipeline/PipelineVersion/PipelineVersion.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Pipeline/PipelineVersion/PipelineVersion.test.tsx index d22292e56ce1..1266e55e1ca5 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Pipeline/PipelineVersion/PipelineVersion.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Pipeline/PipelineVersion/PipelineVersion.test.tsx @@ -36,7 +36,7 @@ jest.mock('../../Tag/TagsContainerV2/TagsContainerV2', () => jest.fn().mockImplementation(() =>
TagsContainerV2
) ); -jest.mock('../../common/RichTextEditor/RichTextEditorPreviewer', () => +jest.mock('../../common/RichTextEditor/RichTextEditorPreviewerV1', () => jest .fn() .mockImplementation(({ markdown }) => ( diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppDetails/AppDetails.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppDetails/AppDetails.component.tsx index 90d64e766489..bd8b2b2775d0 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppDetails/AppDetails.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppDetails/AppDetails.component.tsx @@ -378,6 +378,7 @@ const AppDetails = () => { {appData && ( { key: ApplicationTabs.RECENT_RUNS, children: (
- +
), }, diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppRunsHistory/AppRunsHistory.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppRunsHistory/AppRunsHistory.component.tsx index fe54986735a5..420d459ad3a7 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppRunsHistory/AppRunsHistory.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppRunsHistory/AppRunsHistory.component.tsx @@ -10,10 +10,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Button, Col, Row, Typography } from 'antd'; +import validator from '@rjsf/validator-ajv8'; +import { Button, Col, Modal, Row, Space, Typography } from 'antd'; import { ColumnsType } from 'antd/lib/table'; import { AxiosError } from 'axios'; -import { isNull } from 'lodash'; +import { isNull, noop } from 'lodash'; import React, { forwardRef, useCallback, @@ -31,8 +32,12 @@ import { } from '../../../../constants/constants'; import { GlobalSettingOptions } from '../../../../constants/GlobalSettings.constants'; import { useWebSocketConnector } from '../../../../context/WebSocketProvider/WebSocketProvider'; +import { ServiceCategory } from '../../../../enums/service.enum'; import { AppType } from '../../../../generated/entity/applications/app'; -import { Status } from '../../../../generated/entity/applications/appRunRecord'; +import { + AppRunRecord, + Status, +} from '../../../../generated/entity/applications/appRunRecord'; import { PipelineState, PipelineStatus, @@ -51,16 +56,20 @@ import { getEpochMillisForPastDays, getIntervalInMilliseconds, } from '../../../../utils/date-time/DateTimeUtils'; +import { getEntityName } from '../../../../utils/EntityUtils'; import { getLogsViewerPath } from '../../../../utils/RouterUtils'; import { showErrorToast } from '../../../../utils/ToastUtils'; import ErrorPlaceHolder from '../../../common/ErrorWithPlaceholder/ErrorPlaceHolder'; +import FormBuilder from '../../../common/FormBuilder/FormBuilder'; import NextPrevious from '../../../common/NextPrevious/NextPrevious'; import { PagingHandlerParams } from '../../../common/NextPrevious/NextPrevious.interface'; import StatusBadge from '../../../common/StatusBadge/StatusBadge.component'; import { StatusType } from '../../../common/StatusBadge/StatusBadge.interface'; import Table from '../../../common/Table/Table'; import StopScheduleModal from '../../../Modals/StopScheduleRun/StopScheduleRunModal'; +import applicationsClassBase from '../AppDetails/ApplicationsClassBase'; import AppLogsViewer from '../AppLogsViewer/AppLogsViewer.component'; +import './app-run-history.less'; import { AppRunRecordWithId, AppRunsHistoryProps, @@ -68,7 +77,12 @@ import { const AppRunsHistory = forwardRef( ( - { appData, maxRecords, showPagination = true }: AppRunsHistoryProps, + { + appData, + maxRecords, + jsonSchema, + showPagination = true, + }: AppRunsHistoryProps, ref ) => { const { socket } = useWebSocketConnector(); @@ -80,6 +94,17 @@ const AppRunsHistory = forwardRef( >([]); const [expandedRowKeys, setExpandedRowKeys] = useState([]); const [isStopModalOpen, setIsStopModalOpen] = useState(false); + const [showConfigModal, setShowConfigModal] = useState(false); + const [appRunRecordConfig, setAppRunRecordConfig] = useState< + AppRunRecord['config'] + >({}); + const UiSchema = { + ...applicationsClassBase.getJSONUISchema(), + 'ui:submitButtonProps': { + showButton: false, + buttonText: 'submit', + }, + }; const { currentPage, @@ -132,6 +157,11 @@ const AppRunsHistory = forwardRef( return false; }, []); + const showAppRunConfig = (record: AppRunRecordWithId) => { + setShowConfigModal(true); + setAppRunRecordConfig(record.config ?? {}); + }; + const getActionButton = useCallback( (record: AppRunRecordWithId, index: number) => { if ( @@ -149,6 +179,14 @@ const AppRunsHistory = forwardRef( onClick={() => handleRowExpandable(record.id)}> {t('label.log-plural')} + {/* For status running or activewitherror and supportsInterrupt is true, show stop button */} {(record.status === Status.Running || record.status === Status.ActiveError) && @@ -214,7 +252,7 @@ const AppRunsHistory = forwardRef( return record.status ? ( ) : ( @@ -408,6 +446,52 @@ const AppRunsHistory = forwardRef( }} /> )} + + + + } + maskClosable={false} + open={showConfigModal} + title={ + + {t('label.entity-configuration', { + entity: getEntityName(appData) ?? t('label.application'), + })} + + } + width={800}> + + ); } diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppRunsHistory/AppRunsHistory.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppRunsHistory/AppRunsHistory.interface.ts index 9b31944fb758..5fc0b56045f9 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppRunsHistory/AppRunsHistory.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppRunsHistory/AppRunsHistory.interface.ts @@ -11,6 +11,7 @@ * limitations under the License. */ +import { RJSFSchema } from '@rjsf/utils'; import { App } from '../../../../generated/entity/applications/app'; import { AppRunRecord } from '../../../../generated/entity/applications/appRunRecord'; @@ -26,4 +27,5 @@ export interface AppRunsHistoryProps { maxRecords?: number; appData?: App; showPagination?: boolean; + jsonSchema: RJSFSchema; } diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppRunsHistory/AppRunsHistory.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppRunsHistory/AppRunsHistory.test.tsx index b8f49f1a728e..440ed4bef698 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppRunsHistory/AppRunsHistory.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppRunsHistory/AppRunsHistory.test.tsx @@ -37,6 +37,18 @@ const mockGetApplicationRuns = jest.fn().mockReturnValue({ const mockShowErrorToast = jest.fn(); const mockPush = jest.fn(); +jest.mock('../../../../utils/EntityUtils', () => ({ + getEntityName: jest.fn().mockReturnValue('username'), +})); + +jest.mock('../../../common/FormBuilder/FormBuilder', () => + jest + .fn() + .mockImplementation(({ onSubmit }) => ( + + )) +); + jest.mock('../../../../hooks/paging/usePaging', () => ({ usePaging: jest.fn().mockReturnValue({ currentPage: 8, @@ -132,6 +144,7 @@ const mockProps1 = { appData: mockApplicationData, maxRecords: 10, showPagination: true, + jsonSchema: {}, }; const mockProps2 = { @@ -256,7 +269,7 @@ describe('AppRunsHistory component', () => { }); it('checking behaviour of component when no prop is passed', async () => { - render(); + render(); await waitForElementToBeRemoved(() => screen.getByText('TableLoader')); expect(screen.getByText('--')).toBeInTheDocument(); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppRunsHistory/app-run-history.less b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppRunsHistory/app-run-history.less new file mode 100644 index 000000000000..2bd0f014d9fc --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppRunsHistory/app-run-history.less @@ -0,0 +1,17 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +.app-config-modal { + .ant-modal-body { + padding-top: 0; + } +} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppSchedule/AppSchedule.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppSchedule/AppSchedule.component.tsx index 5ced368c8585..bc46d92eb797 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppSchedule/AppSchedule.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppSchedule/AppSchedule.component.tsx @@ -40,6 +40,7 @@ import { AppScheduleProps } from './AppScheduleProps.interface'; const AppSchedule = ({ appData, loading: { isRunLoading, isDeployLoading }, + jsonSchema, onSave, onDemandTrigger, onDeployTrigger, @@ -127,6 +128,7 @@ const AppSchedule = ({ return ( Promise; onDemandTrigger: () => Promise; onDeployTrigger: () => Promise; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/ApplicationCard/ApplicationCard.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/ApplicationCard/ApplicationCard.component.tsx index 942cc8916d61..04ed2c0f20f5 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/ApplicationCard/ApplicationCard.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/ApplicationCard/ApplicationCard.component.tsx @@ -16,7 +16,7 @@ import classNames from 'classnames'; import { kebabCase } from 'lodash'; import React from 'react'; import { useTranslation } from 'react-i18next'; -import RichTextEditorPreviewer from '../../../common/RichTextEditor/RichTextEditorPreviewer'; +import RichTextEditorPreviewerV1 from '../../../common/RichTextEditor/RichTextEditorPreviewerV1'; import AppLogo from '../AppLogo/AppLogo.component'; import { ApplicationCardProps } from './ApplicationCard.interface'; @@ -59,7 +59,7 @@ const ApplicationCard = ({ )} {showDescription && ( - diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/ApplicationCard/ApplicationCard.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/ApplicationCard/ApplicationCard.test.tsx index ad6d28fa89be..198fcc8259a6 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/ApplicationCard/ApplicationCard.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/ApplicationCard/ApplicationCard.test.tsx @@ -24,9 +24,21 @@ const props = { }; describe('ApplicationCard', () => { + beforeEach(() => { + jest.useFakeTimers(); + }); + + afterEach(() => { + jest.runOnlyPendingTimers(); + jest.useRealTimers(); + }); + it('renders the title correctly', () => { render(); + // Fast-forward until all timers have been executed + jest.runAllTimers(); + expect(screen.getByText('Search Index')).toBeInTheDocument(); expect(screen.getByText('Hello World')).toBeInTheDocument(); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/MarketPlaceAppDetails/MarketPlaceAppDetails.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/MarketPlaceAppDetails/MarketPlaceAppDetails.component.tsx index 21307cfa4ae2..c4344e8a801f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/MarketPlaceAppDetails/MarketPlaceAppDetails.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/MarketPlaceAppDetails/MarketPlaceAppDetails.component.tsx @@ -39,7 +39,7 @@ import { getEntityName } from '../../../../utils/EntityUtils'; import { getAppInstallPath } from '../../../../utils/RouterUtils'; import { showErrorToast } from '../../../../utils/ToastUtils'; import Loader from '../../../common/Loader/Loader'; -import RichTextEditorPreviewer from '../../../common/RichTextEditor/RichTextEditorPreviewer'; +import RichTextEditorPreviewerV1 from '../../../common/RichTextEditor/RichTextEditorPreviewerV1'; import PageLayoutV1 from '../../../PageLayoutV1/PageLayoutV1'; import applicationsClassBase from '../AppDetails/ApplicationsClassBase'; import AppLogo from '../AppLogo/AppLogo.component'; @@ -268,7 +268,7 @@ const MarketPlaceAppDetails = () => {
- diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/MarketPlaceAppDetails/MarketPlaceAppDetails.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/MarketPlaceAppDetails/MarketPlaceAppDetails.test.tsx index 5160c722c627..1ae2322c9ba5 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/MarketPlaceAppDetails/MarketPlaceAppDetails.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/MarketPlaceAppDetails/MarketPlaceAppDetails.test.tsx @@ -41,7 +41,7 @@ jest.mock('react-router-dom', () => ({ })), })); -jest.mock('../../../common/RichTextEditor/RichTextEditorPreviewer', () => +jest.mock('../../../common/RichTextEditor/RichTextEditorPreviewerV1', () => jest.fn().mockReturnValue(<>RichTextEditorPreviewer) ); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Bot/BotListV1/BotListV1.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Bot/BotListV1/BotListV1.component.tsx index d63e26c2a0e8..7c6b402f7cb8 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Bot/BotListV1/BotListV1.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Bot/BotListV1/BotListV1.component.tsx @@ -42,7 +42,7 @@ import ErrorPlaceHolder from '../../../common/ErrorWithPlaceholder/ErrorPlaceHol import FilterTablePlaceHolder from '../../../common/ErrorWithPlaceholder/FilterTablePlaceHolder'; import NextPrevious from '../../../common/NextPrevious/NextPrevious'; import { PagingHandlerParams } from '../../../common/NextPrevious/NextPrevious.interface'; -import RichTextEditorPreviewer from '../../../common/RichTextEditor/RichTextEditorPreviewer'; +import RichTextEditorPreviewerV1 from '../../../common/RichTextEditor/RichTextEditorPreviewerV1'; import Searchbar from '../../../common/SearchBarComponent/SearchBar.component'; import Table from '../../../common/Table/Table'; import TitleBreadcrumb from '../../../common/TitleBreadcrumb/TitleBreadcrumb.component'; @@ -134,7 +134,7 @@ const BotListV1 = ({ key: 'description', render: (_, record) => record?.description ? ( - + ) : ( {t('label.no-entity', { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/CustomProperty/CustomPropertyTable.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/CustomProperty/CustomPropertyTable.test.tsx index af99a66dbf39..4a3a20a3bd78 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/CustomProperty/CustomPropertyTable.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/CustomProperty/CustomPropertyTable.test.tsx @@ -21,7 +21,7 @@ import { import React from 'react'; import { CustomPropertyTable } from './CustomPropertyTable'; -jest.mock('../../common/RichTextEditor/RichTextEditorPreviewer', () => { +jest.mock('../../common/RichTextEditor/RichTextEditorPreviewerV1', () => { return jest.fn().mockReturnValue(

RichTextEditorPreview

); }); jest.mock('../../common/ErrorWithPlaceholder/ErrorPlaceHolder', () => { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/CustomProperty/CustomPropertyTable.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/CustomProperty/CustomPropertyTable.tsx index d0f40ce95863..b81f80aefb72 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/CustomProperty/CustomPropertyTable.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/CustomProperty/CustomPropertyTable.tsx @@ -25,7 +25,7 @@ import { ERROR_PLACEHOLDER_TYPE, OPERATION } from '../../../enums/common.enum'; import { CustomProperty } from '../../../generated/type/customProperty'; import { columnSorter, getEntityName } from '../../../utils/EntityUtils'; import ErrorPlaceHolder from '../../common/ErrorWithPlaceholder/ErrorPlaceHolder'; -import RichTextEditorPreviewer from '../../common/RichTextEditor/RichTextEditorPreviewer'; +import RichTextEditorPreviewerV1 from '../../common/RichTextEditor/RichTextEditorPreviewerV1'; import Table from '../../common/Table/Table'; import ConfirmationModal from '../../Modals/ConfirmationModal/ConfirmationModal'; import './custom-property-table.less'; @@ -200,7 +200,7 @@ export const CustomPropertyTable: FC = ({ width: 300, render: (text) => text ? ( - + ) : ( = ({ mode: 'tags', placeholder: t('label.enum-value-plural'), onChange: (value: string[]) => { - const enumConfig = customProperty.customPropertyConfig - ?.config as Config; - const updatedValues = uniq([...value, ...(enumConfig?.values ?? [])]); + const updatedValues = uniq([...value]); form.setFieldsValue({ customPropertyConfig: updatedValues }); }, open: false, @@ -164,6 +163,9 @@ const EditCustomPropertyModal: FC = ({ props: { 'data-testid': 'multiSelect', }, + formItemProps: { + style: { marginBottom: '0px' }, + }, id: 'root/multiSelect', formItemLayout: FormItemLayout.HORIZONTAL, }; @@ -233,9 +235,15 @@ const EditCustomPropertyModal: FC = ({ <> {hasEnumConfig && ( <> - {generateFormFields([enumConfigField])} - {note} - {generateFormFields([multiSelectField])} + {generateFormFields([enumConfigField, multiSelectField])} + {isSaving && ( + + )} )} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Persona/CustomizeUI/CustomizeUI.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Persona/CustomizeUI/CustomizeUI.tsx deleted file mode 100644 index 8df90f0655e8..000000000000 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Persona/CustomizeUI/CustomizeUI.tsx +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2024 Collate. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import { Col, Row } from 'antd'; -import { isEmpty } from 'lodash'; -import React from 'react'; -import { useHistory } from 'react-router-dom'; -import { useFqn } from '../../../../hooks/useFqn'; -import { getCustomizePagePath } from '../../../../utils/GlobalSettingsUtils'; -import { - getCustomizePageCategories, - getCustomizePageOptions, -} from '../../../../utils/Persona/PersonaUtils'; -import SettingItemCard from '../../SettingItemCard/SettingItemCard.component'; - -const categories = getCustomizePageCategories(); - -export const CustomizeUI = () => { - const history = useHistory(); - const { fqn: personaFQN } = useFqn(); - - const [items, setItems] = React.useState(categories); - - const handleCustomizeItemClick = (category: string) => { - const nestedItems = getCustomizePageOptions(category); - - if (isEmpty(nestedItems)) { - history.push(getCustomizePagePath(personaFQN, category)); - } else { - setItems(nestedItems); - } - }; - - return ( - - {items.map((value) => ( -
- - - ))} - - ); -}; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/AddIngestion/Steps/ScheduleInterval.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/AddIngestion/Steps/ScheduleInterval.tsx index 61c883551519..d586974b506a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/AddIngestion/Steps/ScheduleInterval.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/AddIngestion/Steps/ScheduleInterval.tsx @@ -49,6 +49,7 @@ import { import { generateFormFields } from '../../../../../utils/formUtils'; import { getCurrentLocaleForConstrue } from '../../../../../utils/i18next/i18nextUtil'; import { + cronValidator, getCron, getDefaultScheduleValue, getHourMinuteSelect, @@ -360,22 +361,7 @@ const ScheduleInterval = ({ }), }, { - validator: async (_, value) => { - // Check if cron is valid and get the description - const description = cronstrue.toString(value); - - // Check if cron has a frequency of less than an hour - const isFrequencyInMinutes = /Every \d* *minute/.test( - description - ); - if (isFrequencyInMinutes) { - return Promise.reject( - t('message.cron-less-than-hour-message') - ); - } - - return Promise.resolve(); - }, + validator: cronValidator, }, ]}> diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Ingestion/IngestionListTable/IngestionListTable.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Ingestion/IngestionListTable/IngestionListTable.test.tsx index 1bdc4beecc71..8fb97ca21f40 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Ingestion/IngestionListTable/IngestionListTable.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Ingestion/IngestionListTable/IngestionListTable.test.tsx @@ -15,11 +15,18 @@ import { act, render, screen } from '@testing-library/react'; import userEvent from '@testing-library/user-event'; import React from 'react'; import { MemoryRouter } from 'react-router-dom'; -import { mockIngestionListTableProps } from '../../../../../mocks/IngestionListTable.mock'; +import { usePermissionProvider } from '../../../../../context/PermissionProvider/PermissionProvider'; +import { mockIngestionData } from '../../../../../mocks/Ingestion.mock'; +import { + mockESIngestionData, + mockIngestionListTableProps, +} from '../../../../../mocks/IngestionListTable.mock'; import { ENTITY_PERMISSIONS } from '../../../../../mocks/Permissions.mock'; import { deleteIngestionPipelineById } from '../../../../../rest/ingestionPipelineAPI'; import IngestionListTable from './IngestionListTable'; +const mockGetEntityPermissionByFqn = jest.fn(); + jest.mock('../../../../../hooks/useApplicationStore', () => ({ useApplicationStore: jest.fn(() => ({ theme: { primaryColor: '#fff' }, @@ -82,7 +89,7 @@ jest.mock( () => jest.fn().mockImplementation(() =>
ButtonSkeleton
) ); -jest.mock('../../../../common/RichTextEditor/RichTextEditorPreviewer', () => +jest.mock('../../../../common/RichTextEditor/RichTextEditorPreviewerV1', () => jest.fn().mockImplementation(() =>
RichTextEditorPreviewer
) ); @@ -260,4 +267,33 @@ describe('Ingestion', () => { expect(deleteIngestionPipelineById).toHaveBeenCalledWith('id'); }); + + it('should fetch the permissions for all the ingestion pipelines', async () => { + (usePermissionProvider as jest.Mock).mockImplementation(() => ({ + getEntityPermissionByFqn: mockGetEntityPermissionByFqn, + })); + + await act(async () => { + render( + , + { + wrapper: MemoryRouter, + } + ); + }); + + expect(mockGetEntityPermissionByFqn).toHaveBeenNthCalledWith( + 1, + 'ingestionPipeline', + mockESIngestionData.fullyQualifiedName + ); + expect(mockGetEntityPermissionByFqn).toHaveBeenNthCalledWith( + 2, + 'ingestionPipeline', + mockIngestionData.fullyQualifiedName + ); + }); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Ingestion/IngestionListTable/IngestionListTable.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Ingestion/IngestionListTable/IngestionListTable.tsx index 86738d11dc52..c01231631036 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Ingestion/IngestionListTable/IngestionListTable.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Ingestion/IngestionListTable/IngestionListTable.tsx @@ -46,7 +46,7 @@ import { showSuccessToast, } from '../../../../../utils/ToastUtils'; import NextPrevious from '../../../../common/NextPrevious/NextPrevious'; -import RichTextEditorPreviewer from '../../../../common/RichTextEditor/RichTextEditorPreviewer'; +import RichTextEditorPreviewerV1 from '../../../../common/RichTextEditor/RichTextEditorPreviewerV1'; import ButtonSkeleton from '../../../../common/Skeleton/CommonSkeletons/ControlElements/ControlElements.component'; import Table from '../../../../common/Table/Table'; import EntityDeleteModal from '../../../../Modals/EntityDeleteModal/EntityDeleteModal'; @@ -157,7 +157,10 @@ function IngestionListTable({ const fetchIngestionPipelinesPermission = useCallback(async () => { try { const promises = ingestionData.map((item) => - getEntityPermissionByFqn(ResourceEntity.INGESTION_PIPELINE, item.name) + getEntityPermissionByFqn( + ResourceEntity.INGESTION_PIPELINE, + item.fullyQualifiedName ?? '' + ) ); const response = await Promise.allSettled(promises); @@ -257,7 +260,7 @@ function IngestionListTable({ key: 'description', render: (description: string) => !isUndefined(description) && description.trim() ? ( - diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Ingestion/PipelineActions.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Ingestion/IngestionListTable/PipelineActions/PipelineActions.interface.ts similarity index 69% rename from openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Ingestion/PipelineActions.interface.ts rename to openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Ingestion/IngestionListTable/PipelineActions/PipelineActions.interface.ts index 0ea302cd0061..75a56df992da 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Ingestion/PipelineActions.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Ingestion/IngestionListTable/PipelineActions/PipelineActions.interface.ts @@ -11,10 +11,11 @@ * limitations under the License. */ -import { IngestionServicePermission } from '../../../../context/PermissionProvider/PermissionProvider.interface'; -import { ServiceCategory } from '../../../../enums/service.enum'; -import { IngestionPipeline } from '../../../../generated/entity/services/ingestionPipelines/ingestionPipeline'; -import { SelectedRowDetails } from './ingestion.interface'; +import { ButtonProps } from 'antd'; +import { IngestionServicePermission } from '../../../../../../context/PermissionProvider/PermissionProvider.interface'; +import { ServiceCategory } from '../../../../../../enums/service.enum'; +import { IngestionPipeline } from '../../../../../../generated/entity/services/ingestionPipelines/ingestionPipeline'; +import { SelectedRowDetails } from '../../ingestion.interface'; export interface PipelineActionsProps { pipeline: IngestionPipeline; @@ -23,9 +24,10 @@ export interface PipelineActionsProps { serviceName?: string; deployIngestion?: (id: string, displayName: string) => Promise; triggerIngestion?: (id: string, displayName: string) => Promise; - handleDeleteSelection: (row: SelectedRowDetails) => void; + handleDeleteSelection?: (row: SelectedRowDetails) => void; handleEditClick?: (fqn: string) => void; handleEnableDisableIngestion?: (id: string) => Promise; handleIsConfirmationModalOpen: (value: boolean) => void; onIngestionWorkflowsUpdate?: () => void; + moreActionButtonProps?: ButtonProps; } diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Ingestion/IngestionListTable/PipelineActions/PipelineActions.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Ingestion/IngestionListTable/PipelineActions/PipelineActions.tsx index 90651e429845..2aab18773b56 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Ingestion/IngestionListTable/PipelineActions/PipelineActions.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Ingestion/IngestionListTable/PipelineActions/PipelineActions.tsx @@ -22,8 +22,8 @@ import { Operation } from '../../../../../../generated/entity/policies/accessCon import { PipelineType } from '../../../../../../generated/entity/services/ingestionPipelines/ingestionPipeline'; import { getLoadingStatus } from '../../../../../../utils/CommonUtils'; import { getLogsViewerPath } from '../../../../../../utils/RouterUtils'; -import { PipelineActionsProps } from '../../PipelineActions.interface'; import './pipeline-actions.less'; +import { PipelineActionsProps } from './PipelineActions.interface'; import PipelineActionsDropdown from './PipelineActionsDropdown'; function PipelineActions({ @@ -38,6 +38,7 @@ function PipelineActions({ handleIsConfirmationModalOpen, onIngestionWorkflowsUpdate, handleEditClick, + moreActionButtonProps, }: Readonly) { const history = useHistory(); const { t } = useTranslation(); @@ -168,6 +169,7 @@ function PipelineActions({ handleIsConfirmationModalOpen={handleIsConfirmationModalOpen} ingestion={pipeline} ingestionPipelinePermissions={ingestionPipelinePermissions} + moreActionButtonProps={moreActionButtonProps} serviceCategory={serviceCategory} serviceName={serviceName} triggerIngestion={triggerIngestion} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Ingestion/IngestionListTable/PipelineActions/PipelineActionsDropdown.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Ingestion/IngestionListTable/PipelineActions/PipelineActionsDropdown.interface.ts index a572a7b4c3c5..d860ae282979 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Ingestion/IngestionListTable/PipelineActions/PipelineActionsDropdown.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Ingestion/IngestionListTable/PipelineActions/PipelineActionsDropdown.interface.ts @@ -11,6 +11,7 @@ * limitations under the License. */ +import { ButtonProps } from 'antd'; import { IngestionServicePermission } from '../../../../../../context/PermissionProvider/PermissionProvider.interface'; import { IngestionPipeline } from '../../../../../../generated/entity/services/ingestionPipelines/ingestionPipeline'; import { SelectedRowDetails } from '../../ingestion.interface'; @@ -23,7 +24,8 @@ export interface PipelineActionsDropdownProps { deployIngestion?: (id: string, displayName: string) => Promise; handleEditClick: ((fqn: string) => void) | undefined; ingestionPipelinePermissions?: IngestionServicePermission; - handleDeleteSelection: (row: SelectedRowDetails) => void; + handleDeleteSelection?: (row: SelectedRowDetails) => void; handleIsConfirmationModalOpen: (value: boolean) => void; onIngestionWorkflowsUpdate?: () => void; + moreActionButtonProps?: ButtonProps; } diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Ingestion/IngestionListTable/PipelineActions/PipelineActionsDropdown.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Ingestion/IngestionListTable/PipelineActions/PipelineActionsDropdown.test.tsx index 708d91ae94db..93834477741a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Ingestion/IngestionListTable/PipelineActions/PipelineActionsDropdown.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Ingestion/IngestionListTable/PipelineActions/PipelineActionsDropdown.test.tsx @@ -324,4 +324,26 @@ describe('PipelineActionsDropdown', () => { expect(screen.queryByText('KillIngestionPipelineModal')).toBeNull(); }); + + it('should pass the moreActionButtonProps to the more action button', async () => { + const mockOnClick = jest.fn(); + + await act(async () => { + render( + , + { + wrapper: MemoryRouter, + } + ); + }); + + await clickOnMoreActions(); + + expect(mockOnClick).toHaveBeenCalled(); + }); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Ingestion/IngestionListTable/PipelineActions/PipelineActionsDropdown.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Ingestion/IngestionListTable/PipelineActions/PipelineActionsDropdown.tsx index 61baee49f856..105b33d11b7c 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Ingestion/IngestionListTable/PipelineActions/PipelineActionsDropdown.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Ingestion/IngestionListTable/PipelineActions/PipelineActionsDropdown.tsx @@ -50,6 +50,7 @@ function PipelineActionsDropdown({ handleIsConfirmationModalOpen, onIngestionWorkflowsUpdate, ingestionPipelinePermissions, + moreActionButtonProps, }: Readonly) { const history = useHistory(); const { t } = useTranslation(); @@ -135,7 +136,7 @@ function PipelineActionsDropdown({ const handleConfirmDelete = useCallback( (id: string, name: string, displayName?: string) => { - handleDeleteSelection({ + handleDeleteSelection?.({ id, name, displayName, @@ -270,6 +271,7 @@ function PipelineActionsDropdown({ icon={} type="link" onClick={() => setIsOpen((value) => !value)} + {...moreActionButtonProps} /> {isKillModalOpen && selectedPipeline && id === selectedPipeline?.id && ( diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Services.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Services.test.tsx index c19344bfe717..0eab37f464e2 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Services.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Services.test.tsx @@ -208,7 +208,7 @@ jest.mock('../../common/ListView/ListView.component', () => ({ )), })); -jest.mock('../../common/RichTextEditor/RichTextEditorPreviewer', () => { +jest.mock('../../common/RichTextEditor/RichTextEditorPreviewerV1', () => { return jest .fn() .mockReturnValue( diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Services.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Services.tsx index 49b4d14cce26..a6df62acd5f5 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Services.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Services.tsx @@ -62,7 +62,7 @@ import { ListView } from '../../common/ListView/ListView.component'; import NextPrevious from '../../common/NextPrevious/NextPrevious'; import { PagingHandlerParams } from '../../common/NextPrevious/NextPrevious.interface'; import { OwnerLabel } from '../../common/OwnerLabel/OwnerLabel.component'; -import RichTextEditorPreviewer from '../../common/RichTextEditor/RichTextEditorPreviewer'; +import RichTextEditorPreviewerV1 from '../../common/RichTextEditor/RichTextEditorPreviewerV1'; import ButtonSkeleton from '../../common/Skeleton/CommonSkeletons/ControlElements/ControlElements.component'; import { ColumnFilter } from '../../Database/ColumnFilter/ColumnFilter.component'; import PageHeader from '../../PageHeader/PageHeader.component'; @@ -326,7 +326,7 @@ const Services = ({ serviceName }: ServicesProps) => { width: 200, render: (description) => description ? ( - { className="p-t-xs text-grey-body break-all description-text" data-testid="service-description"> {service.description ? ( - ( - + ), }, { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Team/TeamDetails/TeamDetailsV1.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Team/TeamDetails/TeamDetailsV1.tsx index ba4dee350f5e..680d6e732227 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Team/TeamDetails/TeamDetailsV1.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Team/TeamDetails/TeamDetailsV1.tsx @@ -210,11 +210,8 @@ const TeamDetailsV1 = ({ ); const teamCount = useMemo( - () => - isOrganization && currentTeam && currentTeam.childrenCount - ? currentTeam.childrenCount + 1 - : childTeamList.length, - [childTeamList, isOrganization, currentTeam.childrenCount] + () => currentTeam.childrenCount ?? childTeamList.length, + [childTeamList, currentTeam.childrenCount] ); const updateActiveTab = (key: string) => { history.push({ search: Qs.stringify({ activeTab: key }) }); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Team/TeamDetails/TeamHierarchy.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Team/TeamDetails/TeamHierarchy.tsx index d4e27a3ae18c..16c981780649 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Team/TeamDetails/TeamHierarchy.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Team/TeamDetails/TeamHierarchy.tsx @@ -40,7 +40,7 @@ import { isDropRestricted } from '../../../../utils/TeamUtils'; import { showErrorToast, showSuccessToast } from '../../../../utils/ToastUtils'; import { DraggableBodyRowProps } from '../../../common/Draggable/DraggableBodyRowProps.interface'; import FilterTablePlaceHolder from '../../../common/ErrorWithPlaceholder/FilterTablePlaceHolder'; -import RichTextEditorPreviewer from '../../../common/RichTextEditor/RichTextEditorPreviewer'; +import RichTextEditorPreviewerV1 from '../../../common/RichTextEditor/RichTextEditorPreviewerV1'; import Table from '../../../common/Table/Table'; import { MovedTeamProps, TeamHierarchyProps } from './team.interface'; import './teams.less'; @@ -136,7 +136,7 @@ const TeamHierarchy: FC = ({ {NO_DATA_PLACEHOLDER} ) : ( - permission.EditAll || permission.EditUsers, + [permission.EditAll, permission.EditUsers] + ); + /** * Make API call to fetch current team user data */ @@ -213,13 +218,13 @@ export const UserTab = ({ - {users.length > 0 && permission.EditAll && ( + {users.length > 0 && editUserPermission && ( { return ( - - {description || '--'} - + ); }, }, diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Team/TeamsSelectable/TeamsSelectable.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Team/TeamsSelectable/TeamsSelectable.tsx index a19eb666e482..ee0b4bda77cb 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Team/TeamsSelectable/TeamsSelectable.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Team/TeamsSelectable/TeamsSelectable.tsx @@ -56,7 +56,13 @@ const TeamsSelectable = ({ try { setIsLoading(true); const { data } = await getTeamsHierarchy(filterJoinable); - setTeams(data); + const sortedData = [...data].sort((a, b) => { + const nameA = a.fullyQualifiedName ?? ''; + const nameB = b.fullyQualifiedName ?? ''; + + return nameA.localeCompare(nameB); + }); + setTeams(sortedData); showTeamsAlert && setNoTeam(isEmpty(data)); } catch (error) { showErrorToast(error as AxiosError); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Team/UserImportResult/UserImportResult.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Team/UserImportResult/UserImportResult.component.tsx index 046bb0f7f2ae..2bfba84e8b74 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Team/UserImportResult/UserImportResult.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Team/UserImportResult/UserImportResult.component.tsx @@ -19,6 +19,7 @@ import { ReactComponent as FailBadgeIcon } from '../../../../assets/svg/fail-bad import { ReactComponent as SuccessBadgeIcon } from '../../../../assets/svg/success-badge.svg'; import { Status } from '../../../../generated/type/csvImportResult'; import { parseCSV } from '../../../../utils/EntityImport/EntityImportUtils'; +import RichTextEditorPreviewerV1 from '../../../common/RichTextEditor/RichTextEditorPreviewerV1'; import Table from '../../../common/Table/Table'; import { UserCSVRecord, @@ -102,14 +103,12 @@ export const UserImportResult = ({ width: 300, render: (description: string) => { return ( - - {description || '--'} - + ); }, }, diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Users/CreateUser/CreateUser.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Users/CreateUser/CreateUser.component.tsx index e8fca9ec4565..0d21bd811296 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Users/CreateUser/CreateUser.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Users/CreateUser/CreateUser.component.tsx @@ -60,7 +60,6 @@ import CopyToClipboardButton from '../../../common/CopyToClipboardButton/CopyToC import { DomainLabel } from '../../../common/DomainLabel/DomainLabel.component'; import InlineAlert from '../../../common/InlineAlert/InlineAlert'; import Loader from '../../../common/Loader/Loader'; -import RichTextEditor from '../../../common/RichTextEditor/RichTextEditor'; import TeamsSelectable from '../../Team/TeamsSelectable/TeamsSelectable'; import { CreateUserProps } from './CreateUser.interface'; @@ -195,6 +194,21 @@ const CreateUser = ({ onSave(userProfile); }; + const descriptionField: FieldProp = useMemo( + () => ({ + name: 'description', + required: false, + label: t('label.description'), + id: 'root/description', + type: FieldTypes.DESCRIPTION, + props: { + 'data-testid': 'description', + initialValue: '', + }, + }), + [] + ); + useEffect(() => { generateRandomPassword(); }, []); @@ -272,13 +286,8 @@ const CreateUser = ({ )} - - - + + {getField(descriptionField)} {!forceBot && ( <> diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Users/UsersProfile/UserProfileTeams/UserProfileTeams.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Users/UsersProfile/UserProfileTeams/UserProfileTeams.component.tsx index c1d65a3611df..2ee31d3a34fd 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Users/UsersProfile/UserProfileTeams/UserProfileTeams.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Users/UsersProfile/UserProfileTeams/UserProfileTeams.component.tsx @@ -112,7 +112,7 @@ const UserProfileTeams = ({ onCancel={handleCloseEditTeam} onSave={handleTeamsSave}> ({ })); describe('SuggestionsAlert', () => { + beforeEach(() => { + jest.useFakeTimers(); + }); + + afterEach(() => { + jest.runOnlyPendingTimers(); + jest.useRealTimers(); + }); + const mockSuggestion: Suggestion = { id: '1', description: 'Test suggestion', @@ -54,6 +63,8 @@ describe('SuggestionsAlert', () => { ); + // Fast-forward until all timers have been executed + jest.runAllTimers(); expect(screen.getByText(/Test suggestion/i)).toBeInTheDocument(); expect(screen.getByText(/Test User/i)).toBeInTheDocument(); @@ -66,6 +77,9 @@ describe('SuggestionsAlert', () => { ); + // Fast-forward until all timers have been executed + jest.runAllTimers(); + expect(screen.getByText(/Test suggestion/i)).toBeInTheDocument(); expect(screen.getByText(/Test User/i)).toBeInTheDocument(); expect(screen.getByTestId('reject-suggestion')).toBeInTheDocument(); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Suggestions/SuggestionsAlert/SuggestionsAlert.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Suggestions/SuggestionsAlert/SuggestionsAlert.tsx index 2cc7eca98d38..0ba977f3f9c1 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Suggestions/SuggestionsAlert/SuggestionsAlert.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Suggestions/SuggestionsAlert/SuggestionsAlert.tsx @@ -17,7 +17,7 @@ import { useTranslation } from 'react-i18next'; import { ReactComponent as StarIcon } from '../../../assets/svg/ic-suggestions-coloured.svg'; import UserPopOverCard from '../../common/PopOverCard/UserPopOverCard'; import ProfilePicture from '../../common/ProfilePicture/ProfilePicture'; -import RichTextEditorPreviewer from '../../common/RichTextEditor/RichTextEditorPreviewer'; +import RichTextEditorPreviewerV1 from '../../common/RichTextEditor/RichTextEditorPreviewerV1'; import { useSuggestionsContext } from '../SuggestionsProvider/SuggestionsProvider'; import { SuggestionAction } from '../SuggestionsProvider/SuggestionsProvider.interface'; import './suggestions-alert.less'; @@ -40,7 +40,7 @@ const SuggestionsAlert = ({ return (
- diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Tag/TagsV1/TagsV1.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Tag/TagsV1/TagsV1.component.tsx index 1eba4884903e..450782471a3d 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Tag/TagsV1/TagsV1.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Tag/TagsV1/TagsV1.component.tsx @@ -41,6 +41,7 @@ const TagsV1 = ({ tooltipOverride, tagType, size, + isEditTags, }: TagsV1Props) => { const color = useMemo( () => (isVersionPage ? undefined : tag.style?.color), @@ -144,7 +145,8 @@ const TagsV1 = ({ ), }, 'tag-chip tag-chip-content', - size + size, + 'cursor-pointer' )} data-testid="tags" style={ @@ -185,14 +187,19 @@ const TagsV1 = ({ } return ( - - {tagChip} - + <> + {isEditTags ? ( + tagChip + ) : ( + + {tagChip} + + )} + ); }; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Tag/TagsV1/TagsV1.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/Tag/TagsV1/TagsV1.interface.ts index e66686e9c7fe..d73d707f8caf 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Tag/TagsV1/TagsV1.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/Tag/TagsV1/TagsV1.interface.ts @@ -31,4 +31,5 @@ export type TagsV1Props = { tooltipOverride?: string; tagType?: TagSource; size?: SelectProps['size']; + isEditTags?: boolean; }; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Topic/TopicDetails/TopicDetails.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Topic/TopicDetails/TopicDetails.component.tsx index 72ed8ec8785c..5f296e6f77e9 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Topic/TopicDetails/TopicDetails.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Topic/TopicDetails/TopicDetails.component.tsx @@ -304,7 +304,14 @@ const TopicDetails: React.FC = ({ const tabs = useMemo( () => [ { - label: , + label: ( + + ), key: EntityTabs.SCHEMA, children: ( diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Topic/TopicDetails/TopicDetails.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Topic/TopicDetails/TopicDetails.test.tsx index e4704ba657ec..706a22220b35 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Topic/TopicDetails/TopicDetails.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Topic/TopicDetails/TopicDetails.test.tsx @@ -92,7 +92,7 @@ jest.mock('../../PageLayoutV1/PageLayoutV1', () => { return jest.fn().mockImplementation(({ children }) =>
{children}
); }); -jest.mock('../../common/RichTextEditor/RichTextEditorPreviewer', () => { +jest.mock('../../common/RichTextEditor/RichTextEditorPreviewerV1', () => { return jest.fn().mockReturnValue(

RichTextEditorPreviwer

); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Topic/TopicSchema/TopicSchema.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Topic/TopicSchema/TopicSchema.test.tsx index 6c94f7e53151..48d2a748cb99 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Topic/TopicSchema/TopicSchema.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Topic/TopicSchema/TopicSchema.test.tsx @@ -50,7 +50,7 @@ jest.mock('../../../utils/GlossaryUtils', () => ({ getGlossaryTermsList: jest.fn().mockImplementation(() => Promise.resolve([])), })); -jest.mock('../../common/RichTextEditor/RichTextEditorPreviewer', () => +jest.mock('../../common/RichTextEditor/RichTextEditorPreviewerV1', () => jest .fn() .mockReturnValue( diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Topic/TopicSchema/TopicSchema.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Topic/TopicSchema/TopicSchema.tsx index cd7d72b1f5fa..94078e18f770 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Topic/TopicSchema/TopicSchema.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Topic/TopicSchema/TopicSchema.tsx @@ -46,7 +46,7 @@ import { updateFieldTags, } from '../../../utils/TableUtils'; import ErrorPlaceHolder from '../../common/ErrorWithPlaceholder/ErrorPlaceHolder'; -import RichTextEditorPreviewer from '../../common/RichTextEditor/RichTextEditorPreviewer'; +import RichTextEditorPreviewerV1 from '../../common/RichTextEditor/RichTextEditorPreviewerV1'; import ToggleExpandButton from '../../common/ToggleExpandButton/ToggleExpandButton'; import { ColumnFilter } from '../../Database/ColumnFilter/ColumnFilter.component'; import SchemaEditor from '../../Database/SchemaEditor/SchemaEditor'; @@ -134,7 +134,7 @@ const TopicSchemaFields: FC = ({ {isVersionView ? ( - + ) : ( getEntityName(record) )} @@ -149,7 +149,7 @@ const TopicSchemaFields: FC = ({ (dataType: DataTypeTopic, record: Field) => ( {isVersionView ? ( - ) : ( diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Visualisations/Chart/OperationDateBarChart.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Visualisations/Chart/OperationDateBarChart.tsx index 2d055e16816e..da3d27d2be15 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Visualisations/Chart/OperationDateBarChart.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Visualisations/Chart/OperationDateBarChart.tsx @@ -73,6 +73,7 @@ const OperationDateBarChart = ({ tooltipFormatter(value)} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/AirflowMessageBanner/AirflowMessageBanner.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/AirflowMessageBanner/AirflowMessageBanner.tsx index c457865a22cf..508cf5fa3c3a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/AirflowMessageBanner/AirflowMessageBanner.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/AirflowMessageBanner/AirflowMessageBanner.tsx @@ -16,7 +16,7 @@ import { isEmpty } from 'lodash'; import React, { FC } from 'react'; import { ReactComponent as IconRetry } from '../../../assets/svg/ic-retry-icon.svg'; import { useAirflowStatus } from '../../../hooks/useAirflowStatus'; -import RichTextEditorPreviewer from '../../common/RichTextEditor/RichTextEditorPreviewer'; +import RichTextEditorPreviewerV1 from '../RichTextEditor/RichTextEditorPreviewerV1'; import './airflow-message-banner.less'; const AirflowMessageBanner: FC = ({ className }) => { @@ -33,7 +33,7 @@ const AirflowMessageBanner: FC = ({ className }) => { data-testid="no-airflow-placeholder" size={16}> - diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/AsyncSelectList/AsyncSelectList.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/AsyncSelectList/AsyncSelectList.tsx index c10ba5908001..3bd1621e228d 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/AsyncSelectList/AsyncSelectList.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/AsyncSelectList/AsyncSelectList.tsx @@ -248,6 +248,7 @@ const AsyncSelectList: FC = ({ return ( > = ({ return ( > = ({ if (lastSelectedMap.has(value)) { return lastSelectedMap.get(value) as SelectOption; } - const initialData = findGlossaryTermByFqn( + const initialData = findItemByFqn( [ ...glossaries, ...(isNull(searchOptions) ? [] : searchOptions), diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/ExtensionTable.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/ExtensionTable.tsx index 374bc4549cf2..4e25677a768b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/ExtensionTable.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/ExtensionTable.tsx @@ -12,11 +12,10 @@ */ import { Table, Typography } from 'antd'; import { ColumnsType } from 'antd/lib/table'; -import classNames from 'classnames'; import { isObject, isString, map } from 'lodash'; import React, { useMemo } from 'react'; import { useTranslation } from 'react-i18next'; -import RichTextEditorPreviewer from '../RichTextEditor/RichTextEditorPreviewer'; +import RichTextEditorPreviewerV1 from '../RichTextEditor/RichTextEditorPreviewerV1'; import { ExtentionEntities, ExtentionEntitiesKeys, @@ -29,10 +28,8 @@ interface ExtensionDataSource { export const ExtensionTable = ({ extension, - tableClassName, }: { extension: ExtentionEntities[ExtentionEntitiesKeys]['extension']; - tableClassName?: string; }) => { const { t } = useTranslation(); const dataSource: ExtensionDataSource[] = useMemo(() => { @@ -56,7 +53,7 @@ export const ExtensionTable = ({ const isStringValue = isString(value); if (isStringValue) { - return ; + return ; } return ( @@ -72,7 +69,7 @@ export const ExtensionTable = ({ return (
= ({ { +jest.mock('../../common/RichTextEditor/RichTextEditorPreviewerV1', () => { return jest .fn() .mockReturnValue( diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/PropertyValue.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/PropertyValue.tsx index a2dd1a0f15e7..970ea37b1ecf 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/PropertyValue.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/PropertyValue.tsx @@ -37,7 +37,6 @@ import { noop, omitBy, toNumber, - toUpper, } from 'lodash'; import moment, { Moment } from 'moment'; import React, { @@ -65,6 +64,7 @@ import { CSMode } from '../../../enums/codemirror.enum'; import { SearchIndex } from '../../../enums/search.enum'; import { EntityReference } from '../../../generated/entity/type'; import { Config } from '../../../generated/type/customProperty'; +import { getCustomPropertyMomentFormat } from '../../../utils/CustomProperty.utils'; import { calculateInterval } from '../../../utils/date-time/DateTimeUtils'; import entityUtilClassBase from '../../../utils/EntityUtilClassBase'; import { getEntityName } from '../../../utils/EntityUtils'; @@ -76,7 +76,7 @@ import SchemaEditor from '../../Database/SchemaEditor/SchemaEditor'; import { ModalWithMarkdownEditor } from '../../Modals/ModalWithMarkdownEditor/ModalWithMarkdownEditor'; import InlineEdit from '../InlineEdit/InlineEdit.component'; import ProfilePicture from '../ProfilePicture/ProfilePicture'; -import RichTextEditorPreviewer from '../RichTextEditor/RichTextEditorPreviewer'; +import RichTextEditorPreviewerV1 from '../RichTextEditor/RichTextEditorPreviewerV1'; import { PropertyValueProps, PropertyValueType, @@ -187,7 +187,7 @@ export const PropertyValue: FC = ({ const getPropertyInput = () => { const commonStyle: CSSProperties = { marginBottom: '0px', - minWidth: '250px', + width: '100%', }; switch (propertyType.name) { case 'string': @@ -278,9 +278,9 @@ export const PropertyValue: FC = ({ case 'date-cp': case 'dateTime-cp': { - // Default format is 'yyyy-mm-dd' - const format = toUpper( - (property.customPropertyConfig?.config as string) ?? 'yyyy-mm-dd' + const format = getCustomPropertyMomentFormat( + propertyType.name, + property.customPropertyConfig?.config ); const initialValues = { @@ -314,11 +314,11 @@ export const PropertyValue: FC = ({ @@ -327,8 +327,11 @@ export const PropertyValue: FC = ({ } case 'time-cp': { - const format = - (property.customPropertyConfig?.config as string) ?? 'HH:mm:ss'; + const format = getCustomPropertyMomentFormat( + propertyType.name, + property.customPropertyConfig?.config + ); + const initialValues = { time: value ? moment(value, format) : undefined, }; @@ -359,10 +362,10 @@ export const PropertyValue: FC = ({ @@ -755,7 +758,7 @@ export const PropertyValue: FC = ({ const isKeyAdded = versionDataKeys?.includes(propertyName); return ( - @@ -763,7 +766,7 @@ export const PropertyValue: FC = ({ } switch (propertyType.name) { case 'markdown': - return ; + return ; case 'enum': return ( @@ -905,7 +908,7 @@ export const PropertyValue: FC = ({ return (
@@ -1034,7 +1037,7 @@ export const PropertyValue: FC = ({ {!isRenderedInRightPanel && (
- []; + onCancel: () => void; + onSave: (data: TableTypePropertyValueType) => Promise; +} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/TableTypeProperty/EditTableTypePropertyModal.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/TableTypeProperty/EditTableTypePropertyModal.test.tsx new file mode 100644 index 000000000000..7c3a94dddc53 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/TableTypeProperty/EditTableTypePropertyModal.test.tsx @@ -0,0 +1,95 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import '@testing-library/jest-dom/extend-expect'; +import { fireEvent, render, waitFor } from '@testing-library/react'; +import React from 'react'; +import { CustomProperty } from '../../../../generated/type/customProperty'; +import EditTableTypePropertyModal from './EditTableTypePropertyModal'; +import { EditTableTypePropertyModalProps } from './EditTableTypePropertyModal.interface'; + +jest.mock('./TableTypePropertyEditTable', () => + jest.fn().mockImplementation(() =>
TableTypePropertyEditTable
) +); + +jest.mock('./TableTypePropertyView', () => + jest.fn().mockImplementation(() =>
TableTypePropertyView
) +); + +const mockOnCancel = jest.fn(); +const mockOnSave = jest.fn().mockResolvedValue(undefined); + +const defaultProps: EditTableTypePropertyModalProps = { + isVisible: true, + isUpdating: false, + property: {} as CustomProperty, + columns: ['column1', 'column2'], + rows: [{ column1: 'value1', column2: 'value2' }], + onCancel: mockOnCancel, + onSave: mockOnSave, +}; + +describe('EditTableTypePropertyModal', () => { + it('should render the modal with the correct title', () => { + const { getByText } = render( + + ); + + expect(getByText('label.edit-entity-name')).toBeInTheDocument(); + }); + + it('should call onCancel when the cancel button is clicked', () => { + const { getByTestId } = render( + + ); + fireEvent.click(getByTestId('cancel-update-table-type-property')); + + expect(mockOnCancel).toHaveBeenCalled(); + }); + + it('should call onSave with the correct data when the update button is clicked', async () => { + const { getByTestId } = render( + + ); + fireEvent.click(getByTestId('update-table-type-property')); + await waitFor(() => + expect(mockOnSave).toHaveBeenCalledWith({ + rows: [{ column1: 'value1', column2: 'value2' }], + columns: ['column1', 'column2'], + }) + ); + }); + + it('should add a new row when the add new row button is clicked', () => { + const { getByTestId, getByText } = render( + + ); + fireEvent.click(getByTestId('add-new-row')); + + expect(getByText('TableTypePropertyEditTable')).toBeInTheDocument(); + }); + + it('should render TableTypePropertyView when dataSource is empty', () => { + const props = { ...defaultProps, rows: [] }; + const { getByText } = render(); + + expect(getByText('TableTypePropertyView')).toBeInTheDocument(); + }); + + it('should render TableTypePropertyEditTable when dataSource is not empty', () => { + const { getByText } = render( + + ); + + expect(getByText('TableTypePropertyEditTable')).toBeInTheDocument(); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/TableTypeProperty/EditTableTypePropertyModal.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/TableTypeProperty/EditTableTypePropertyModal.tsx index 19088331bd10..779ae871bf16 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/TableTypeProperty/EditTableTypePropertyModal.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/TableTypeProperty/EditTableTypePropertyModal.tsx @@ -10,31 +10,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import ReactDataGrid from '@inovua/reactdatagrid-community'; import '@inovua/reactdatagrid-community/index.css'; import { TypeComputedProps } from '@inovua/reactdatagrid-community/types'; import { Button, Modal, Typography } from 'antd'; import { isEmpty, omit } from 'lodash'; import React, { FC, MutableRefObject, useCallback, useState } from 'react'; import { useTranslation } from 'react-i18next'; -import { CustomProperty } from '../../../../generated/type/customProperty'; import { getEntityName } from '../../../../utils/EntityUtils'; import { TableTypePropertyValueType } from '../CustomPropertyTable.interface'; import './edit-table-type-property.less'; +import { EditTableTypePropertyModalProps } from './EditTableTypePropertyModal.interface'; +import TableTypePropertyEditTable from './TableTypePropertyEditTable'; import TableTypePropertyView from './TableTypePropertyView'; -interface EditTableTypePropertyModalProps { - isVisible: boolean; - isUpdating: boolean; - property: CustomProperty; - columns: string[]; - rows: Record[]; - onCancel: () => void; - onSave: (data: TableTypePropertyValueType) => Promise; -} - -let inEdit = false; - const EditTableTypePropertyModal: FC = ({ isVisible, isUpdating, @@ -54,91 +42,16 @@ const EditTableTypePropertyModal: FC = ({ MutableRefObject >({ current: null }); - const filterColumns = columns.map((column) => ({ - name: column, - header: column, - defaultFlex: 1, - sortable: false, - minWidth: 180, - })); - - const onEditComplete = useCallback( - ({ value, columnId, rowId }) => { - const data = [...dataSource]; - - data[rowId][columnId] = value; - - setDataSource(data); + const handleEditGridRef = useCallback( + (ref: MutableRefObject) => { + setGridRef(ref); }, - [dataSource] + [] ); - const onEditStart = () => { - inEdit = true; - }; - - const onEditStop = () => { - requestAnimationFrame(() => { - inEdit = false; - gridRef.current?.focus(); - }); - }; - - const onKeyDown = (event: KeyboardEvent) => { - if (inEdit) { - if (event.key === 'Escape') { - const [rowIndex, colIndex] = gridRef.current?.computedActiveCell ?? [ - 0, 0, - ]; - const column = gridRef.current?.getColumnBy(colIndex); - - gridRef.current?.cancelEdit?.({ - rowIndex, - columnId: column?.name ?? '', - }); - } - - return; - } - const grid = gridRef.current; - if (!grid) { - return; - } - let [rowIndex, colIndex] = grid.computedActiveCell ?? [0, 0]; - - if (event.key === ' ' || event.key === 'Enter') { - const column = grid.getColumnBy(colIndex); - grid.startEdit?.({ columnId: column.name ?? '', rowIndex }); - event.preventDefault(); - - return; - } - if (event.key !== 'Tab') { - return; - } - event.preventDefault(); - event.stopPropagation(); - - const direction = event.shiftKey ? -1 : 1; - - const columns = grid.visibleColumns; - const rowCount = grid.count; - - colIndex += direction; - if (colIndex === -1) { - colIndex = columns.length - 1; - rowIndex -= 1; - } - if (colIndex === columns.length) { - rowIndex += 1; - colIndex = 0; - } - if (rowIndex < 0 || rowIndex === rowCount) { - return; - } - - grid?.setActiveCell([rowIndex, colIndex]); - }; + const handleEditDataSource = useCallback((data: Record[]) => { + setDataSource(data); + }, []); const handleAddRow = useCallback(() => { setDataSource((data) => { @@ -207,20 +120,12 @@ const EditTableTypePropertyModal: FC = ({ {isEmpty(dataSource) ? ( ) : ( - )} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/TableTypeProperty/TableTypePropertyEditTable.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/TableTypeProperty/TableTypePropertyEditTable.interface.ts new file mode 100644 index 000000000000..5ca123f1d31c --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/TableTypeProperty/TableTypePropertyEditTable.interface.ts @@ -0,0 +1,22 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { TypeComputedProps } from '@inovua/reactdatagrid-community/types'; +import { MutableRefObject } from 'react'; + +export interface TableTypePropertyEditTableProps { + columns: string[]; + dataSource: Record[]; + gridRef: MutableRefObject; + handleEditGridRef: (ref: MutableRefObject) => void; + handleEditDataSource: (data: Record[]) => void; +} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/TableTypeProperty/TableTypePropertyEditTable.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/TableTypeProperty/TableTypePropertyEditTable.test.tsx new file mode 100644 index 000000000000..efadad047f76 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/TableTypeProperty/TableTypePropertyEditTable.test.tsx @@ -0,0 +1,60 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { TypeComputedProps } from '@inovua/reactdatagrid-community/types'; +import { act, render, screen } from '@testing-library/react'; +import React, { MutableRefObject } from 'react'; +import TableTypePropertyEditTable from './TableTypePropertyEditTable'; +import { TableTypePropertyEditTableProps } from './TableTypePropertyEditTable.interface'; + +const mockDataSource: { value: Record[] } = { + value: [ + { + name: 'Property 1', + value: 'Value 1', + id: '0', + }, + ], +}; + +const mockDataGridRefObj: { + value: MutableRefObject; +} = { + value: { current: null }, +}; + +const props: TableTypePropertyEditTableProps = { + columns: ['name', 'value'], + dataSource: mockDataSource.value, + gridRef: mockDataGridRefObj.value, + handleEditGridRef: jest + .fn() + .mockImplementation((data: Record[]) => { + mockDataSource.value = data; + }), + handleEditDataSource: jest + .fn() + .mockImplementation((data: MutableRefObject) => { + mockDataGridRefObj.value = data; + }), +}; + +describe('TableTypePropertyEditTable', () => { + it('should render the table with given columns and dataSource', async () => { + await act(async () => { + render(); + }); + + expect(screen.getByText('Property 1')).toBeInTheDocument(); + expect(screen.getByText('Value 1')).toBeInTheDocument(); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/TableTypeProperty/TableTypePropertyEditTable.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/TableTypeProperty/TableTypePropertyEditTable.tsx new file mode 100644 index 000000000000..90e224d4f908 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/TableTypeProperty/TableTypePropertyEditTable.tsx @@ -0,0 +1,131 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import ReactDataGrid from '@inovua/reactdatagrid-community'; +import React, { useCallback } from 'react'; +import { TableTypePropertyEditTableProps } from './TableTypePropertyEditTable.interface'; + +let inEdit = false; + +const TableTypePropertyEditTable = ({ + dataSource, + columns, + gridRef, + handleEditGridRef, + handleEditDataSource, +}: TableTypePropertyEditTableProps) => { + const filterColumns = columns.map((column) => ({ + name: column, + header: column, + defaultFlex: 1, + sortable: false, + minWidth: 180, + })); + + const onEditComplete = useCallback( + ({ value, columnId, rowId }) => { + const data = [...dataSource]; + + data[rowId][columnId] = value; + + handleEditDataSource(data); + }, + [dataSource] + ); + + const onEditStart = () => { + inEdit = true; + }; + + const onEditStop = () => { + requestAnimationFrame(() => { + inEdit = false; + gridRef.current?.focus(); + }); + }; + + const onKeyDown = (event: KeyboardEvent) => { + if (inEdit) { + if (event.key === 'Escape') { + const [rowIndex, colIndex] = gridRef.current?.computedActiveCell ?? [ + 0, 0, + ]; + const column = gridRef.current?.getColumnBy(colIndex); + + gridRef.current?.cancelEdit?.({ + rowIndex, + columnId: column?.name ?? '', + }); + } + + return; + } + const grid = gridRef.current; + if (!grid) { + return; + } + let [rowIndex, colIndex] = grid.computedActiveCell ?? [0, 0]; + + if (event.key === ' ' || event.key === 'Enter') { + const column = grid.getColumnBy(colIndex); + grid.startEdit?.({ columnId: column.name ?? '', rowIndex }); + event.preventDefault(); + + return; + } + if (event.key !== 'Tab') { + return; + } + event.preventDefault(); + event.stopPropagation(); + + const direction = event.shiftKey ? -1 : 1; + + const columns = grid.visibleColumns; + const rowCount = grid.count; + + colIndex += direction; + if (colIndex === -1) { + colIndex = columns.length - 1; + rowIndex -= 1; + } + if (colIndex === columns.length) { + rowIndex += 1; + colIndex = 0; + } + if (rowIndex < 0 || rowIndex === rowCount) { + return; + } + + grid?.setActiveCell([rowIndex, colIndex]); + }; + + return ( + + ); +}; + +export default TableTypePropertyEditTable; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/property-value.less b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/property-value.less index fc786b0bdae7..7de5af548f1b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/property-value.less +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/property-value.less @@ -94,5 +94,11 @@ } .custom-property-inline-edit-container { + width: 100%; + flex-wrap: wrap; overflow-x: scroll; + + .ant-space-item:first-child { + width: inherit; + } } diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/DomainSelectableList/DomainSelectableList.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/DomainSelectableList/DomainSelectableList.component.tsx index aecba1071cf3..02ec15d89a30 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/DomainSelectableList/DomainSelectableList.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/DomainSelectableList/DomainSelectableList.component.tsx @@ -15,26 +15,12 @@ import React, { useCallback, useMemo, useState } from 'react'; import { useTranslation } from 'react-i18next'; import { ReactComponent as EditIcon } from '../../../assets/svg/edit-new.svg'; import { ReactComponent as DomainIcon } from '../../../assets/svg/ic-domain.svg'; -import { - DE_ACTIVE_COLOR, - PAGE_SIZE_MEDIUM, -} from '../../../constants/constants'; +import { DE_ACTIVE_COLOR } from '../../../constants/constants'; import { NO_PERMISSION_FOR_ACTION } from '../../../constants/HelperTextUtil'; -import { EntityType } from '../../../enums/entity.enum'; -import { SearchIndex } from '../../../enums/search.enum'; import { EntityReference } from '../../../generated/entity/type'; -import { useApplicationStore } from '../../../hooks/useApplicationStore'; -import { getDomainList } from '../../../rest/domainAPI'; -import { searchData } from '../../../rest/miscAPI'; -import { formatDomainsResponse } from '../../../utils/APIUtils'; -import { Transi18next } from '../../../utils/CommonUtils'; -import { - getEntityName, - getEntityReferenceListFromEntities, -} from '../../../utils/EntityUtils'; +import { getEntityName } from '../../../utils/EntityUtils'; import Fqn from '../../../utils/Fqn'; -import { getDomainPath } from '../../../utils/RouterUtils'; -import { SelectableList } from '../SelectableList/SelectableList.component'; +import DomainSelectablTree from '../DomainSelectableTree/DomainSelectableTree'; import './domain-select-dropdown.less'; import { DomainSelectableListProps } from './DomainSelectableList.interface'; @@ -72,58 +58,28 @@ const DomainSelectableList = ({ popoverProps, selectedDomain, multiple = false, + onCancel, }: DomainSelectableListProps) => { const { t } = useTranslation(); - const { theme } = useApplicationStore(); const [popupVisible, setPopupVisible] = useState(false); const selectedDomainsList = useMemo(() => { if (selectedDomain) { - return Array.isArray(selectedDomain) ? selectedDomain : [selectedDomain]; + return Array.isArray(selectedDomain) + ? selectedDomain.map((item) => item.fullyQualifiedName) + : [selectedDomain.fullyQualifiedName]; } return []; }, [selectedDomain]); - const fetchOptions = async (searchText: string, after?: string) => { - if (searchText) { - try { - const res = await searchData( - searchText, - 1, - PAGE_SIZE_MEDIUM, - '', - '', - '', - SearchIndex.DOMAIN - ); - - const data = getEntityReferenceListFromEntities( - formatDomainsResponse(res.data.hits.hits), - EntityType.DOMAIN - ); - - return { data, paging: { total: res.data.hits.total.value } }; - } catch (error) { - return { data: [], paging: { total: 0 } }; - } - } else { - try { - const { data, paging } = await getDomainList({ - limit: PAGE_SIZE_MEDIUM, - after: after ?? undefined, - }); - const filterData = getEntityReferenceListFromEntities( - data, - EntityType.DOMAIN - ); - - return { data: filterData, paging }; - } catch (error) { - return { data: [], paging: { total: 0 } }; - } + const initialDomains = useMemo(() => { + if (selectedDomain) { + return Array.isArray(selectedDomain) ? selectedDomain : [selectedDomain]; } - }; + + return []; + }, [selectedDomain]); const handleUpdate = useCallback( async (domains: EntityReference[]) => { @@ -132,11 +88,17 @@ const DomainSelectableList = ({ } else { await onUpdate(domains[0]); } + setPopupVisible(false); }, [onUpdate, multiple] ); + const handleCancel = useCallback(() => { + setPopupVisible(false); + onCancel?.(); + }, [onCancel]); + return ( // Used Button to stop click propagation event anywhere in the component to parent // TeamDetailV1 collapsible panel @@ -146,39 +108,17 @@ const DomainSelectableList = ({ - } - values={{ - link: t('label.domain-plural'), - }} - /> - } - fetchOptions={fetchOptions} - multiSelect={multiple} - removeIconTooltipLabel={t('label.remove-entity', { - entity: t('label.domain-lowercase'), - })} - searchPlaceholder={t('label.search-for-type', { - type: t('label.domain'), - })} - selectedItems={selectedDomainsList} - onCancel={() => setPopupVisible(false)} - onUpdate={handleUpdate} + } open={popupVisible} - overlayClassName="domain-select-popover" + overlayClassName="domain-select-popover w-400" placement="bottomRight" showArrow={false} trigger="click" diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/DomainSelectableList/DomainSelectableList.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/common/DomainSelectableList/DomainSelectableList.interface.ts index 9962a54c8a3f..0fcd98a5055b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/DomainSelectableList/DomainSelectableList.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/DomainSelectableList/DomainSelectableList.interface.ts @@ -21,4 +21,5 @@ export interface DomainSelectableListProps { popoverProps?: PopoverProps; selectedDomain?: EntityReference | EntityReference[]; multiple?: boolean; + onCancel?: () => void; } diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/DomainSelectableList/domain-select-dropdown.less b/openmetadata-ui/src/main/resources/ui/src/components/common/DomainSelectableList/domain-select-dropdown.less index 3b8694212385..13aae85efc6a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/DomainSelectableList/domain-select-dropdown.less +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/DomainSelectableList/domain-select-dropdown.less @@ -15,7 +15,11 @@ .domain-select-popover { min-width: 275px; - padding-top: 0; + + &.ant-popover { + padding-top: 0; + } + .ant-popover-inner-content { padding: 0; } diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/DomainSelectableTree/DomainSelectableTree.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/common/DomainSelectableTree/DomainSelectableTree.interface.ts new file mode 100644 index 000000000000..076576357ee8 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/DomainSelectableTree/DomainSelectableTree.interface.ts @@ -0,0 +1,25 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { DefaultOptionType } from 'antd/lib/select'; +import { EntityReference } from '../../../generated/entity/type'; + +export interface DomainSelectableTreeProps { + value?: string[]; // array of fqn + onSubmit: (option: EntityReference[]) => Promise; + visible: boolean; + onCancel: () => void; + isMultiple?: boolean; + initialDomains?: EntityReference[]; +} + +export type TreeListItem = Omit; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/DomainSelectableTree/DomainSelectableTree.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/DomainSelectableTree/DomainSelectableTree.tsx new file mode 100644 index 000000000000..950242adf193 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/DomainSelectableTree/DomainSelectableTree.tsx @@ -0,0 +1,265 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Button, Empty, Space, Tree } from 'antd'; +import Search from 'antd/lib/input/Search'; +import { AxiosError } from 'axios'; +import { debounce } from 'lodash'; +import React, { + FC, + Key, + useCallback, + useEffect, + useMemo, + useState, +} from 'react'; +import { useTranslation } from 'react-i18next'; +import { ReactComponent as IconDown } from '../../../assets/svg/ic-arrow-down.svg'; +import { ReactComponent as IconRight } from '../../../assets/svg/ic-arrow-right.svg'; +import { EntityType } from '../../../enums/entity.enum'; +import { Domain } from '../../../generated/entity/domains/domain'; +import { EntityReference } from '../../../generated/tests/testCase'; +import { listDomainHierarchy, searchDomains } from '../../../rest/domainAPI'; +import { + convertDomainsToTreeOptions, + isDomainExist, +} from '../../../utils/DomainUtils'; +import { getEntityReferenceFromEntity } from '../../../utils/EntityUtils'; +import { findItemByFqn } from '../../../utils/GlossaryUtils'; +import { + escapeESReservedCharacters, + getEncodedFqn, +} from '../../../utils/StringsUtils'; +import { showErrorToast } from '../../../utils/ToastUtils'; +import Loader from '../Loader/Loader'; +import './domain-selectable.less'; +import { + DomainSelectableTreeProps, + TreeListItem, +} from './DomainSelectableTree.interface'; + +const DomainSelectablTree: FC = ({ + onSubmit, + value, + visible, + onCancel, + isMultiple = false, + initialDomains, +}) => { + const { t } = useTranslation(); + const [treeData, setTreeData] = useState([]); + const [domains, setDomains] = useState([]); + const [isLoading, setIsLoading] = useState(false); + const [isSubmitLoading, setIsSubmitLoading] = useState(false); + const [selectedDomains, setSelectedDomains] = useState([]); + const [searchTerm, setSearchTerm] = useState(''); + + const handleMultiDomainSave = async () => { + const selectedFqns = selectedDomains + .map((domain) => domain.fullyQualifiedName) + .sort((a, b) => (a ?? '').localeCompare(b ?? '')); + const initialFqns = (value as string[]).sort((a, b) => a.localeCompare(b)); + + if (JSON.stringify(selectedFqns) !== JSON.stringify(initialFqns)) { + setIsSubmitLoading(true); + const domains1 = selectedDomains.map((item) => + getEntityReferenceFromEntity(item, EntityType.DOMAIN) + ); + await onSubmit(domains1); + setIsSubmitLoading(false); + } else { + onCancel(); + } + }; + + const handleSingleDomainSave = async () => { + const selectedFqn = selectedDomains[0]?.fullyQualifiedName; + const initialFqn = value?.[0]; + + if (selectedFqn !== initialFqn) { + setIsSubmitLoading(true); + let retn: EntityReference[] = []; + if (selectedDomains.length > 0) { + const domain = getEntityReferenceFromEntity( + selectedDomains[0], + EntityType.DOMAIN + ); + retn = [domain]; + } + await onSubmit(retn); + setIsSubmitLoading(false); + } else { + onCancel(); + } + }; + + const fetchAPI = useCallback(async () => { + try { + setIsLoading(true); + const data = await listDomainHierarchy({ limit: 100 }); + + const combinedData = [...data.data]; + initialDomains?.forEach((selectedDomain) => { + const exists = combinedData.some((domain: Domain) => + isDomainExist(domain, selectedDomain.fullyQualifiedName ?? '') + ); + if (!exists) { + combinedData.push(selectedDomain as unknown as Domain); + } + }); + + setTreeData(convertDomainsToTreeOptions(combinedData, 0, isMultiple)); + setDomains(combinedData); + } catch (error) { + showErrorToast(error as AxiosError); + } finally { + setIsLoading(false); + } + }, [initialDomains]); + + const onSelect = (selectedKeys: React.Key[]) => { + if (!isMultiple) { + const selectedData = []; + for (const item of selectedKeys) { + selectedData.push( + findItemByFqn(domains, item as string, false) as Domain + ); + } + + setSelectedDomains(selectedData); + } + }; + + const onCheck = ( + checked: Key[] | { checked: Key[]; halfChecked: Key[] } + ): void => { + if (Array.isArray(checked)) { + const selectedData = []; + for (const item of checked) { + selectedData.push( + findItemByFqn(domains, item as string, false) as Domain + ); + } + + setSelectedDomains(selectedData); + } else { + const selected = checked.checked.map( + (item) => findItemByFqn(domains, item as string, false) as Domain + ); + + setSelectedDomains(selected); + } + }; + + const onSearch = debounce(async (value: string) => { + setSearchTerm(value); + if (value) { + try { + setIsLoading(true); + const encodedValue = getEncodedFqn(escapeESReservedCharacters(value)); + const results: Domain[] = await searchDomains(encodedValue); + const updatedTreeData = convertDomainsToTreeOptions( + results, + 0, + isMultiple + ); + setTreeData(updatedTreeData); + setDomains(results); + } finally { + setIsLoading(false); + } + } else { + fetchAPI(); + } + }, 300); + + const switcherIcon = useCallback(({ expanded }) => { + return expanded ? : ; + }, []); + + const treeContent = useMemo(() => { + if (isLoading) { + return ; + } else if (treeData.length === 0) { + return ( + + ); + } else { + return ( + + ); + } + }, [isLoading, treeData, value, onSelect, isMultiple, searchTerm]); + + useEffect(() => { + if (visible) { + setSearchTerm(''); + fetchAPI(); + } + }, [visible]); + + return ( +
+ onSearch(e.target.value)} + /> + + {treeContent} + + + + + +
+ ); +}; + +export default DomainSelectablTree; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/DomainSelectableTree/domain-selectable.less b/openmetadata-ui/src/main/resources/ui/src/components/common/DomainSelectableTree/domain-selectable.less new file mode 100644 index 000000000000..bcf651e2e812 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/DomainSelectableTree/domain-selectable.less @@ -0,0 +1,26 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@import (reference) url('../../../styles/variables.less'); + +.domain-selectable-tree { + overflow: auto; + height: 270px; + + .ant-tree-treenode { + .ant-tree-node-content-wrapper { + &.ant-tree-node-selected { + background-color: @radio-button-checked-bg; + } + } + } +} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/DraggableTabs/DraggableTabs.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/DraggableTabs/DraggableTabs.tsx deleted file mode 100644 index 580ab4fb468f..000000000000 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/DraggableTabs/DraggableTabs.tsx +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Copyright 2024 Collate. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import type { TabsProps } from 'antd'; -import { Tabs } from 'antd'; -import React, { useRef, useState } from 'react'; -import { DndProvider, useDrag, useDrop } from 'react-dnd'; -import { HTML5Backend } from 'react-dnd-html5-backend'; -import { sortTabs } from '../../../utils/CustomizePage/CustomizePageUtils'; - -const type = 'DraggableTabNode'; -interface DraggableTabPaneProps extends React.HTMLAttributes { - index: React.Key; - moveNode: (dragIndex: React.Key, hoverIndex: React.Key) => void; -} - -export const DraggableTabNode = ({ - index, - children, - moveNode, -}: DraggableTabPaneProps) => { - const ref = useRef(null); - const [{ isOver, dropClassName }, drop] = useDrop({ - accept: type, - collect: (monitor) => { - const { index: dragIndex } = monitor.getItem<{ index: string }>() || {}; - if (dragIndex === index) { - return {}; - } - - return { - isOver: monitor.isOver(), - dropClassName: 'dropping', - }; - }, - drop: (item: { index: React.Key }) => { - moveNode(item.index, index); - }, - }); - const [, drag] = useDrag({ - type, - item: { index }, - collect: (monitor) => ({ - isDragging: monitor.isDragging(), - }), - }); - drop(drag(ref)); - - return ( -
- {children} -
- ); -}; - -export const DraggableTabs: React.FC< - TabsProps & { onTabChange?: (newKeys: React.Key[]) => void } -> = (props) => { - const { items = [] } = props; - const [order, setOrder] = useState([]); - - const moveTabNode = (dragKey: React.Key, hoverKey: React.Key) => { - const newOrder = order.slice(); - - items.forEach((item) => { - if (item.key && newOrder.indexOf(item.key) === -1) { - newOrder.push(item.key); - } - }); - - const dragIndex = newOrder.indexOf(dragKey); - const hoverIndex = newOrder.indexOf(hoverKey); - - newOrder.splice(dragIndex, 1); - newOrder.splice(hoverIndex, 0, dragKey); - - props.onTabChange?.(newOrder); - setOrder(newOrder); - }; - - const renderTabBar: TabsProps['renderTabBar'] = ( - tabBarProps, - DefaultTabBar - ) => ( - - {(node) => ( - - {node} - - )} - - ); - - const orderItems = sortTabs(items, order as string[]); - - return ( - - - - ); -}; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/EntityDescription/DescriptionV1.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/EntityDescription/DescriptionV1.tsx index 25fb5e368db1..fb7e03adbb47 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/EntityDescription/DescriptionV1.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/EntityDescription/DescriptionV1.tsx @@ -23,6 +23,7 @@ import { ReactComponent as RequestIcon } from '../../../assets/svg/request-icon. import { DE_ACTIVE_COLOR } from '../../../constants/constants'; import { EntityField } from '../../../constants/Feeds.constants'; import { EntityType } from '../../../enums/entity.enum'; +import { isDescriptionContentEmpty } from '../../../utils/BlockEditorUtils'; import { getEntityFeedLink } from '../../../utils/EntityUtils'; import { getRequestDescriptionPath, @@ -33,7 +34,7 @@ import { ModalWithMarkdownEditor } from '../../Modals/ModalWithMarkdownEditor/Mo import SuggestionsAlert from '../../Suggestions/SuggestionsAlert/SuggestionsAlert'; import { useSuggestionsContext } from '../../Suggestions/SuggestionsProvider/SuggestionsProvider'; import SuggestionsSlider from '../../Suggestions/SuggestionsSlider/SuggestionsSlider'; -import RichTextEditorPreviewer from '../RichTextEditor/RichTextEditorPreviewer'; +import RichTextEditorPreviewerV1 from '../RichTextEditor/RichTextEditorPreviewerV1'; import { DescriptionProps } from './Description.interface'; const { Text } = Typography; @@ -90,7 +91,7 @@ const DescriptionV1 = ({ }, [entityType, entityFqn]); const taskActionButton = useMemo(() => { - const hasDescription = Boolean(description.trim()); + const hasDescription = !isDescriptionContentEmpty(description.trim()); const isTaskEntity = TASK_ENTITIES.includes(entityType as EntityType); @@ -188,15 +189,13 @@ const DescriptionV1 = ({ if (suggestionData) { return suggestionData; } else { - return description.trim() ? ( - - ) : ( - {t('label.no-description')} ); } }, [description, suggestionData, isDescriptionExpanded]); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/EntityPageInfos/AnnouncementCard/AnnouncementCard.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/EntityPageInfos/AnnouncementCard/AnnouncementCard.tsx index 7bb9e81da9d4..19c2d12d7f06 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/EntityPageInfos/AnnouncementCard/AnnouncementCard.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/EntityPageInfos/AnnouncementCard/AnnouncementCard.tsx @@ -15,6 +15,7 @@ import { Card, Space, Typography } from 'antd'; import React, { FC, useMemo } from 'react'; import { ReactComponent as AnnouncementIcon } from '../../../../assets/svg/announcements-v1.svg'; import { Thread } from '../../../../generated/entity/feed/thread'; +import RichTextEditorPreviewerV1 from '../../RichTextEditor/RichTextEditorPreviewerV1'; import './AnnouncementCard.less'; interface Props { @@ -50,12 +51,12 @@ const AnnouncementCard: FC = ({ onClick, announcement }) => { {message && ( - - {message} - + )} ); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/EntityPageInfos/ManageButton/ManageButton.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/EntityPageInfos/ManageButton/ManageButton.tsx index 2b79d0347cd7..a5f4a6d1bbf5 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/EntityPageInfos/ManageButton/ManageButton.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/EntityPageInfos/ManageButton/ManageButton.tsx @@ -13,13 +13,14 @@ import { Button, Dropdown, Modal, Tooltip, Typography } from 'antd'; import { ItemType } from 'antd/lib/menu/hooks/useItems'; -import { AxiosError } from 'axios'; +import axios, { AxiosError } from 'axios'; import classNames from 'classnames'; import { capitalize, isUndefined } from 'lodash'; import React, { FC, useCallback, useMemo, useState } from 'react'; import { useTranslation } from 'react-i18next'; import { ReactComponent as IconAnnouncementsBlack } from '../../../../assets/svg/announcements-black.svg'; import { ReactComponent as EditIcon } from '../../../../assets/svg/edit-new.svg'; +import { ReactComponent as TableIcon } from '../../../../assets/svg/plus-outlined.svg'; import { ReactComponent as IconDelete } from '../../../../assets/svg/ic-delete.svg'; import { ReactComponent as IconRestore } from '../../../../assets/svg/ic-restore.svg'; import { ReactComponent as IconSetting } from '../../../../assets/svg/ic-settings-gray.svg'; @@ -34,6 +35,9 @@ import DeleteWidgetModal from '../../DeleteWidget/DeleteWidgetModal'; import { ManageButtonItemLabel } from '../../ManageButtonContentItem/ManageButtonContentItem.component'; import { ManageButtonProps } from './ManageButton.interface'; import './ManageButton.less'; +import { useHistory } from 'react-router-dom'; +import { getOidcToken } from '../../../../utils/LocalStorageUtils'; +import AddTableModal from '../../../Modals/AddTableModal/AddTableModal.component'; const ManageButton: FC = ({ allowSoftDelete, @@ -43,6 +47,7 @@ const ManageButton: FC = ({ softDeleteMessagePostFix, hardDeleteMessagePostFix, entityName, + entityFQN, displayName, entityType, canDelete, @@ -63,9 +68,11 @@ const ManageButton: FC = ({ }) => { const { t } = useTranslation(); const [isDelete, setIsDelete] = useState(false); + const [isAdd, setIsAdd] = useState(false); const [isEntityRestoring, setIsEntityRestoring] = useState(false); const [showReactiveModal, setShowReactiveModal] = useState(false); const [isDisplayNameEditing, setIsDisplayNameEditing] = useState(false); + const history = useHistory(); const isProfilerSupported = useMemo( () => @@ -98,6 +105,38 @@ const ManageButton: FC = ({ setIsDisplayNameEditing(false); } }; + + const handleNewTable = async (data: EntityName) => { + + if (data.displayName){ + const token = getOidcToken() + + // Create Headers + const headers = { + headers: { Authorization: `Bearer ${token}` } + }; + + const name = data.displayName.trim().replace(' ', '_').replace(/[-:,\(\)]/g, '').normalize("NFD").replace(/[\u0300-\u036f]/g, "") + + // Create Table + const table = { + "columns": [], + "databaseSchema": entityFQN, + "name": name, + "displayName" : data.displayName + } + + // Make request + await axios.put('/api/v1/tables', table, headers); + setIsAdd(false); + + // Success redirect + history.push("/table/" + entityFQN + '.' + name); + }else{ + setIsAdd(false); + } + + }; const showAnnouncementOption = useMemo( () => @@ -185,6 +224,27 @@ const ManageButton: FC = ({ ] as ItemType[]) : []), ...(extraDropdownContent ?? []), + ...(isProfilerSupported && entityType == 'databaseSchema' + ? ([ + { + label: ( + + ), + onClick: (e) => { + if (canDelete) { + e.domEvent.stopPropagation(); + setIsAdd(true); + } + }, + key: 'add-button', + }, + ] as ItemType[]) + : []), ...(isProfilerSupported ? ([ { @@ -286,6 +346,18 @@ const ManageButton: FC = ({ onCancel={() => setIsDelete(false)} /> )} + {isAdd && ( + setIsAdd(false)} + onSave={handleNewTable} + /> + )} {onEditDisplayName && isDisplayNameEditing && ( { @@ -39,26 +41,8 @@ const FeedsFilterPopover = ({ useState(defaultFilter); const items = useMemo( - () => [ - { - title: t('label.all'), - key: currentUser?.isAdmin - ? FeedFilter.ALL - : FeedFilter.OWNER_OR_FOLLOWS, - description: t('message.feed-filter-all'), - }, - { - title: t('label.my-data'), - key: FeedFilter.OWNER, - description: t('message.feed-filter-owner'), - }, - { - title: t('label.following'), - key: FeedFilter.FOLLOWS, - description: t('message.feed-filter-following'), - }, - ], - [currentUser] + () => getFeedFilterWidgets(feedTab, currentUser?.isAdmin), + [currentUser?.isAdmin, feedTab] ); const onFilterUpdate = useCallback(() => { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/FeedsFilterPopover/FeedsFilterPopover.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/common/FeedsFilterPopover/FeedsFilterPopover.interface.ts index bb1d41f156f0..66690980d7ce 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/FeedsFilterPopover/FeedsFilterPopover.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/FeedsFilterPopover/FeedsFilterPopover.interface.ts @@ -11,8 +11,10 @@ * limitations under the License. */ import { FeedFilter } from '../../../enums/mydata.enum'; +import { ActivityFeedTabs } from '../../ActivityFeed/ActivityFeedTab/ActivityFeedTab.interface'; export interface FeedsFilterPopoverProps { + feedTab: ActivityFeedTabs; defaultFilter: FeedFilter; onUpdate: (value: FeedFilter) => void; } diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/FeedsFilterPopover/FeedsFilterPopover.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/FeedsFilterPopover/FeedsFilterPopover.test.tsx index 77c737e766f1..98de8a954e73 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/FeedsFilterPopover/FeedsFilterPopover.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/FeedsFilterPopover/FeedsFilterPopover.test.tsx @@ -13,10 +13,12 @@ import { act, fireEvent, render, screen } from '@testing-library/react'; import React from 'react'; import { FeedFilter } from '../../../enums/mydata.enum'; +import { ActivityFeedTabs } from '../../ActivityFeed/ActivityFeedTab/ActivityFeedTab.interface'; import FeedsFilterPopover from './FeedsFilterPopover.component'; const onUpdateMock = jest.fn(); const mockProps = { + feedTab: ActivityFeedTabs.ALL, defaultFilter: FeedFilter.ALL, onUpdate: onUpdateMock, }; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/Form/JSONSchema/JSONSchemaTemplate/ObjectFieldTemplate.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/Form/JSONSchema/JSONSchemaTemplate/ObjectFieldTemplate.tsx index 3cc716657fd2..409eb20cfd22 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/Form/JSONSchema/JSONSchemaTemplate/ObjectFieldTemplate.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/Form/JSONSchema/JSONSchemaTemplate/ObjectFieldTemplate.tsx @@ -64,7 +64,7 @@ export const ObjectFieldTemplate: FunctionComponent = const fieldElement = ( - + ); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/Form/JSONSchema/JsonSchemaWidgets/SelectWidget.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/Form/JSONSchema/JsonSchemaWidgets/SelectWidget.tsx index 23d6cdbe4ae7..c424561e20d9 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/Form/JSONSchema/JsonSchemaWidgets/SelectWidget.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/Form/JSONSchema/JsonSchemaWidgets/SelectWidget.tsx @@ -32,6 +32,7 @@ const SelectWidget: FC = (props) => { disabled={rest.disabled} id={rest.id} mode={rest.multiple ? 'multiple' : undefined} + open={props.readonly ? false : undefined} placeholder={rest.placeholder} value={rest.value} onBlur={() => onBlur(rest.id, rest.value)} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/Form/JSONSchema/JsonSchemaWidgets/TreeSelectWidget.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/Form/JSONSchema/JsonSchemaWidgets/TreeSelectWidget.tsx index 44e3baf96acd..bbc08d7ac77e 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/Form/JSONSchema/JsonSchemaWidgets/TreeSelectWidget.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/Form/JSONSchema/JsonSchemaWidgets/TreeSelectWidget.tsx @@ -49,6 +49,7 @@ const TreeSelectWidget: FC = ({ treeDefaultExpandAll data-testid="tree-select-widget" disabled={rest.disabled} + open={rest.readonly ? false : undefined} showCheckedStrategy={TreeSelect.SHOW_PARENT} style={{ width: '100%', diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/FormBuilder/FormBuilder.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/FormBuilder/FormBuilder.tsx index 8f4cb2b608f6..2d035c23c085 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/FormBuilder/FormBuilder.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/FormBuilder/FormBuilder.tsx @@ -16,7 +16,7 @@ import Form, { FormProps, IChangeEvent } from '@rjsf/core'; import { Button } from 'antd'; import classNames from 'classnames'; import { LoadingState } from 'Models'; -import React, { forwardRef, FunctionComponent, useState } from 'react'; +import React, { forwardRef, FunctionComponent, useMemo, useState } from 'react'; import { ServiceCategory } from '../../../enums/service.enum'; import { ConfigData } from '../../../interface/service.interface'; import { transformErrors } from '../../../utils/formUtils'; @@ -64,6 +64,10 @@ const FormBuilder: FunctionComponent = forwardRef( }, ref ) => { + const isReadOnlyForm = useMemo(() => { + return !!props.readonly; + }, [props.readonly]); + const [localFormData, setLocalFormData] = useState( formatFormDataForRender(formData ?? {}) ); @@ -87,6 +91,39 @@ const FormBuilder: FunctionComponent = forwardRef( props.onChange && props.onChange(e); }; + const submitButton = useMemo(() => { + if (status === 'waiting') { + return ( + + ); + } else if (status === 'success') { + return ( + + ); + } else { + return ( + + ); + } + }, [status, isLoading, okText]); + return (
= forwardRef( )} - {status === 'waiting' ? ( - - ) : status === 'success' ? ( - - ) : ( - - )} + {!isReadOnlyForm && submitButton} ); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/InlineEdit/InlineEdit.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/InlineEdit/InlineEdit.component.tsx index e7e97214b088..f34f188d4417 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/InlineEdit/InlineEdit.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/InlineEdit/InlineEdit.component.tsx @@ -14,6 +14,7 @@ import { CheckOutlined, CloseOutlined } from '@ant-design/icons'; import { Button, Space } from 'antd'; import classNames from 'classnames'; import React from 'react'; +import './inline-edit.less'; import { InlineEditProps } from './InlineEdit.interface'; const InlineEdit = ({ @@ -28,7 +29,7 @@ const InlineEdit = ({ }: InlineEditProps) => { return ( { - const button = document.createElement('button'); - - button.className = 'toastui-editor-toolbar-icons markdown-icon'; - button.style.backgroundImage = 'none'; - button.type = 'button'; - button.style.margin = '0'; - button.style.marginTop = '4px'; - button.innerHTML = ` - - markdown-icon - `; - - return button; -}; - -export const EDITOR_TOOLBAR_ITEMS = [ - 'heading', - 'bold', - 'italic', - 'strike', - 'ul', - 'ol', - 'link', - 'hr', - 'quote', - 'code', - 'codeblock', - { - name: i18n.t('label.markdown-guide'), - el: markdownButton(), - tooltip: i18n.t('label.markdown-guide'), - }, -]; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/RichTextEditor/RichTextEditor.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/common/RichTextEditor/RichTextEditor.interface.ts index 9dddec599d89..ee03c664f158 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/RichTextEditor/RichTextEditor.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/RichTextEditor/RichTextEditor.interface.ts @@ -47,19 +47,11 @@ export type EditorType = 'markdown' | 'wysiwyg'; export interface RichTextEditorProp extends HTMLAttributes { autofocus?: boolean; initialValue?: string; - placeHolder?: string; - previewStyle?: PreviewStyle; - editorType?: EditorType; - previewHighlight?: boolean; - extendedAutolinks?: boolean; - hideModeSwitch?: boolean; - useCommandShortcut?: boolean; readonly?: boolean; - height?: string; onTextChange?: (value: string) => void; + placeHolder?: string; } export interface EditorContentRef { getEditorContent: () => string; - clearEditorContent: () => void; } diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/RichTextEditor/RichTextEditor.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/RichTextEditor/RichTextEditor.test.tsx index a1c1e8952b17..aa709f698b0d 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/RichTextEditor/RichTextEditor.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/RichTextEditor/RichTextEditor.test.tsx @@ -1,5 +1,5 @@ /* - * Copyright 2022 Collate. + * Copyright 2025 Collate. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -10,68 +10,82 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -import { findByTestId, queryByTestId, render } from '@testing-library/react'; -import React, { Component } from 'react'; -import { MemoryRouter } from 'react-router-dom'; +import { fireEvent, render, screen } from '@testing-library/react'; +import React from 'react'; +import { EditorContentRef } from '../../Modals/ModalWithMarkdownEditor/ModalWithMarkdownEditor.interface'; import RichTextEditor from './RichTextEditor'; -jest.mock('@toast-ui/react-editor', () => { - class Editor extends Component { - // eslint-disable-next-line @typescript-eslint/no-empty-function - getInstance() {} +jest.mock('../../BlockEditor/BlockEditor', () => { + return jest.fn().mockImplementation(({ content, onChange, ref }: any) => { + if (ref && ref.current) { + ref.current = { editor: { getHTML: jest.fn().mockReturnValue(content) } }; // mock the editor object + } - // eslint-disable-next-line @typescript-eslint/no-empty-function - getRootElement() {} + return ( +