Create table

.github/workflows/auto-cherry-pick-labeled-prs.yaml

@@ -5,13 +5,14 @@ run-name: OpenMetadata release cherry-pick PR #${{ github.event.pull_request.num
 
 # yamllint disable-line rule:truthy
 on:
-  pull_request:
+  pull_request_target:
     types: [closed]
     branches:
       - main
 permissions:
   contents: write
   pull-requests: write
+  issues: write
 env:
   CURRENT_RELEASE_ENDPOINT: ${{ vars.CURRENT_RELEASE_ENDPOINT }}  # Endpoint that returns the current release version in json format
 jobs:

.github/workflows/playwright-mysql-e2e-skip.yml

@@ -31,9 +31,9 @@ jobs:
   playwright-ci-mysql:
     runs-on: ubuntu-latest
     strategy:
-    fail-fast: false
-    matrix:
-      shardIndex: [1, 2]
-      shardTotal: [2]
+      fail-fast: false
+      matrix:
+        shardIndex: [1, 2]
+        shardTotal: [2]
     steps:
       - run: 'echo "Step is not required"'

.github/workflows/playwright-mysql-e2e.yml

@@ -36,7 +36,7 @@ concurrency:
 
 jobs:
   playwright-ci-mysql:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     environment: test
     strategy:
       fail-fast: false

.github/workflows/playwright-postgresql-e2e-skip.yml

@@ -31,9 +31,9 @@ jobs:
   playwright-ci-postgresql:
     runs-on: ubuntu-latest
     strategy:
-    fail-fast: false
-    matrix:
-      shardIndex: [1, 2]
-      shardTotal: [2]
+      fail-fast: false
+      matrix:
+        shardIndex: [1, 2]
+        shardTotal: [2]
     steps:
       - run: 'echo "Step is not required"'

.github/workflows/playwright-postgresql-e2e.yml

@@ -36,7 +36,7 @@ concurrency:
 
 jobs:
   playwright-ci-postgresql:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     environment: test
     strategy:
       fail-fast: false

.github/workflows/py-cli-e2e-tests.yml

@@ -65,7 +65,7 @@ jobs:
 
     - name: configure aws credentials
       if: contains('quicksight', matrix.e2e-test) || contains('datalake_s3', matrix.e2e-test) || contains('athena', matrix.e2e-test)
-      uses: aws-actions/configure-aws-credentials@v1
+      uses: aws-actions/configure-aws-credentials@v4
       with:
         role-to-assume: ${{ secrets.E2E_AWS_IAM_ROLE_ARN }}
         role-session-name: github-ci-aws-e2e-tests

.github/workflows/py-tests.yml

@@ -55,7 +55,7 @@ jobs:
         docker-images: false
 
     - name: Wait for the labeler
-      uses: lewagon/wait-on-check-action@v1.3.3
+      uses: lewagon/wait-on-check-action@v1.3.4
       if: ${{ github.event_name == 'pull_request_target' }}
       with:
         ref: ${{ github.event.pull_request.head.sha }}

bootstrap/sql/migrations/native/1.5.15/mysql/schemaChanges.sql

@@ -0,0 +1,5 @@
+-- Make domain policy and role non-system
+UPDATE policy_entity SET json = JSON_SET(json, '$.provider', 'user') where name = 'DomainOnlyAccessPolicy';
+UPDATE policy_entity SET json = JSON_SET(json, '$.allowDelete', true) where name = 'DomainOnlyAccessPolicy';
+UPDATE role_entity SET json = JSON_SET(json, '$.provider', 'user') where name = 'DomainOnlyAccessRole';
+UPDATE role_entity SET json = JSON_SET(json, '$.allowDelete', true) where name = 'DomainOnlyAccessRole';

bootstrap/sql/migrations/native/1.5.15/postgres/schemaChanges.sql

@@ -0,0 +1,5 @@
+-- Make domain policy and role non-system
+UPDATE policy_entity SET json = JSONB_SET(json::jsonb, '{provider}', '"user"', true) where name = 'DomainOnlyAccessPolicy';
+UPDATE policy_entity SET json = JSONB_SET(json::jsonb, '{allowDelete}', 'true', true) WHERE name = 'DomainOnlyAccessPolicy';
+UPDATE role_entity SET json = JSONB_SET(json::jsonb, '{provider}', '"user"', true) where name = 'DomainOnlyAccessRole';
+UPDATE role_entity SET json = JSONB_SET(json::jsonb, '{allowDelete}', 'true', true) WHERE name = 'DomainOnlyAccessRole';

bootstrap/sql/migrations/native/1.6.2/mysql/schemaChanges.sql

@@ -0,0 +1,44 @@
+-- add timestamp index for test case result reindex performance
+ALTER TABLE data_quality_data_time_series ADD INDEX `idx_timestamp_desc` (timestamp DESC);
+
+CREATE TABLE background_jobs (
+  id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY,
+  jobType VARCHAR(256) NOT NULL,
+  methodName VARCHAR(256) NOT NULL,
+  jobArgs JSON NOT NULL,
+  status VARCHAR(50) NOT NULL DEFAULT 'PENDING',
+  createdBy VARCHAR(256) NOT NULL,
+  createdAt BIGINT UNSIGNED NOT NULL DEFAULT (UNIX_TIMESTAMP(NOW(3)) * 1000),
+  updatedAt BIGINT UNSIGNED NOT NULL DEFAULT (UNIX_TIMESTAMP(NOW(3)) * 1000)
+);
+
+CREATE INDEX idx_status_createdAt ON background_jobs (status, createdAt);
+CREATE INDEX idx_createdBy ON background_jobs (createdBy);
+CREATE INDEX idx_status ON background_jobs (status);
+CREATE INDEX idx_jobType ON background_jobs (jobType);
+CREATE INDEX idx_updatedAt ON background_jobs (updatedAt);
+
+-- rename executable -> basic for test suites
+UPDATE test_suite
+SET json = JSON_INSERT(
+    JSON_REMOVE(json, '$.executable'),
+    '$.basic',
+    JSON_EXTRACT(json, '$.executable')
+)
+WHERE JSON_EXTRACT(json, '$.executable') IS NOT NULL;
+
+-- rename executableEntityReference -> basicEntityReference for test suites
+UPDATE test_suite
+SET json = JSON_INSERT(
+    JSON_REMOVE(json, '$.executableEntityReference'),
+    '$.basicEntityReference',
+    JSON_EXTRACT(json, '$.executableEntityReference')
+)
+WHERE JSON_EXTRACT(json, '$.executableEntityReference') IS NOT NULL;
+
+-- clean up the testSuites
+UPDATE test_case SET json = json_remove(json, '$.testSuites');
+
+-- clean up the testSuites in the version history too
+UPDATE entity_extension SET json = json_remove(json, '$.testSuites') WHERE jsonSchema = 'testCase';
+

bootstrap/sql/migrations/native/1.6.2/postgres/schemaChanges.sql

@@ -0,0 +1,46 @@
+-- add timestamp index for test case result reindex performance
+CREATE INDEX idx_timestamp_desc ON data_quality_data_time_series (timestamp DESC);
+
+CREATE TABLE background_jobs (
+  id BIGSERIAL PRIMARY KEY,
+  jobType VARCHAR(256) NOT NULL,
+  methodName VARCHAR(256) NOT NULL,
+  jobArgs JSONB NOT NULL,
+  status VARCHAR(50) NOT NULL DEFAULT 'PENDING',
+  createdBy VARCHAR(256) NOT NULL,
+  createdAt BIGINT NOT NULL DEFAULT (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT,
+  updatedAt BIGINT NOT NULL DEFAULT (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT
+);
+
+CREATE INDEX idx_status_createdAt ON background_jobs (status, createdAt);
+CREATE INDEX idx_createdBy ON background_jobs (createdBy);
+CREATE INDEX idx_status ON background_jobs (status);
+CREATE INDEX idx_jobType ON background_jobs (jobType);
+CREATE INDEX idx_updatedAt ON background_jobs (updatedAt);
+
+-- rename executable -> basic for test suites
+UPDATE test_suite
+SET json = jsonb_set(
+  json::jsonb #- '{executable}',
+  '{basic}',
+  (json #> '{executable}')::jsonb,
+  true
+)
+WHERE json #>> '{executable}' IS NOT NULL;
+
+-- rename executableEntityReference -> basicEntityReference for test suites
+UPDATE test_suite
+SET json = jsonb_set(
+  json::jsonb #- '{executableEntityReference}',
+  '{basicEntityReference}',
+  (json #> '{executableEntityReference}')::jsonb,
+  true
+)
+WHERE json #>> '{executableEntityReference}' IS NOT NULL;
+
+-- clean up the testSuites
+UPDATE test_case SET json = json::jsonb #- '{testSuites}';
+
+-- clean up the testSuites in the version history too
+UPDATE entity_extension SET json = json::jsonb #- '{testSuites}' WHERE jsonSchema = 'testCase';
+

bootstrap/sql/migrations/native/1.6.3/mysql/schemaChanges.sql

@@ -0,0 +1,6 @@
+-- Add constraint to apps_data_store
+ALTER TABLE apps_data_store ADD CONSTRAINT entity_relationship_pky PRIMARY KEY (identifier, type);
+UPDATE user_entity SET json = JSON_SET(json, '$.isBot', false) WHERE JSON_EXTRACT(json, '$.isBot') IS NULL;
+ALTER TABLE user_entity  ADD COLUMN isBot BOOLEAN GENERATED ALWAYS AS (json -> '$.isBot') NOT NULL;
+CREATE INDEX idx_isBot ON user_entity (isBot);
+DELETE from apps_data_store;

bootstrap/sql/migrations/native/1.6.3/postgres/schemaChanges.sql

@@ -0,0 +1,6 @@
+-- Add constraint to apps_data_store
+ALTER TABLE apps_data_store ADD CONSTRAINT entity_relationship_pky PRIMARY KEY (identifier, type);
+UPDATE user_entity SET json = jsonb_set(json::jsonb, '{isBot}', 'false'::jsonb, true) WHERE NOT (json ?? 'isBot');
+ALTER TABLE user_entity ADD COLUMN isBot BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED NOT NULL;
+CREATE INDEX idx_isBot ON user_entity (isBot);
+DELETE from apps_data_store;

common/pom.xml

@@ -18,7 +18,7 @@
   <parent>
     <artifactId>platform</artifactId>
     <groupId>org.open-metadata</groupId>
-    <version>1.6.0-SNAPSHOT</version>
+    <version>1.6.4</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <properties>

common/src/main/java/org/openmetadata/annotations/DeprecatedAnnotator.java

@@ -0,0 +1,76 @@
+package org.openmetadata.annotations;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.sun.codemodel.JAnnotationUse;
+import com.sun.codemodel.JClass;
+import com.sun.codemodel.JDefinedClass;
+import com.sun.codemodel.JFieldVar;
+import com.sun.codemodel.JMethod;
+import java.lang.reflect.Field;
+import java.util.TreeMap;
+import org.jsonschema2pojo.AbstractAnnotator;
+
+/** Add {@link Deprecated} annotation to generated Java classes */
+public class DeprecatedAnnotator extends AbstractAnnotator {
+
+  /** Add {@link Deprecated} annotation to property fields */
+  @Override
+  public void propertyField(
+      JFieldVar field, JDefinedClass clazz, String propertyName, JsonNode propertyNode) {
+    super.propertyField(field, clazz, propertyName, propertyNode);
+    if (propertyNode.get("deprecated") != null && propertyNode.get("deprecated").asBoolean()) {
+      field.annotate(Deprecated.class);
+    }
+  }
+
+  /** Add {@link Deprecated} annotation to getter methods */
+  @Override
+  public void propertyGetter(JMethod getter, JDefinedClass clazz, String propertyName) {
+    super.propertyGetter(getter, clazz, propertyName);
+    addDeprecatedAnnotationIfApplies(getter, propertyName);
+  }
+
+  /** Add {@link Deprecated} annotation to setter methods */
+  @Override
+  public void propertySetter(JMethod setter, JDefinedClass clazz, String propertyName) {
+    super.propertySetter(setter, clazz, propertyName);
+    addDeprecatedAnnotationIfApplies(setter, propertyName);
+  }
+
+  /**
+   * Use reflection methods to access the {@link JDefinedClass} of the {@link JMethod} object. If
+   * the {@link JMethod} is pointing to a field annotated with {@link Deprecated} then annotates
+   * the {@link JMethod} object with {@link Deprecated}
+   */
+  private void addDeprecatedAnnotationIfApplies(JMethod jMethod, String propertyName) {
+    try {
+      Field outerClassField = JMethod.class.getDeclaredField("outer");
+      outerClassField.setAccessible(true);
+      JDefinedClass outerClass = (JDefinedClass) outerClassField.get(jMethod);
+
+      TreeMap<String, JFieldVar> insensitiveFieldsMap =
+          new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
+      insensitiveFieldsMap.putAll(outerClass.fields());
+
+      if (insensitiveFieldsMap.containsKey(propertyName)
+          && insensitiveFieldsMap.get(propertyName).annotations().stream()
+              .anyMatch(
+                  annotation ->
+                      Deprecated.class.getName().equals(getAnnotationClassName(annotation)))) {
+        jMethod.annotate(Deprecated.class);
+      }
+    } catch (NoSuchFieldException | IllegalAccessException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  private String getAnnotationClassName(JAnnotationUse annotation) {
+    try {
+      Field clazzField = JAnnotationUse.class.getDeclaredField("clazz");
+      clazzField.setAccessible(true);
+      return ((JClass) clazzField.get(annotation)).fullName();
+    } catch (NoSuchFieldException | IllegalAccessException e) {
+      throw new RuntimeException(e);
+    }
+  }
+}

common/src/main/java/org/openmetadata/annotations/OpenMetadataAnnotator.java

@@ -19,6 +19,10 @@ public class OpenMetadataAnnotator extends CompositeAnnotator {
 
   public OpenMetadataAnnotator() {
     // we can add multiple annotators
-    super(new ExposedAnnotator(), new MaskedAnnotator(), new PasswordAnnotator());
+    super(
+        new ExposedAnnotator(),
+        new MaskedAnnotator(),
+        new PasswordAnnotator(),
+        new DeprecatedAnnotator());
   }
 }

conf/openmetadata.yaml

@@ -180,6 +180,7 @@ authenticationConfiguration:
   # This will only be valid when provider type specified is customOidc
   providerName: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
   publicKeyUrls: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
+  tokenValidationAlgorithm: ${AUTHENTICATION_TOKEN_VALIDATION_ALGORITHM:-"RS256"}
   authority: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
   clientId: ${AUTHENTICATION_CLIENT_ID:-""}
   callbackUrl: ${AUTHENTICATION_CALLBACK_URL:-""}
@@ -295,6 +296,7 @@ eventMonitoringConfiguration:
   batchSize: ${EVENT_MONITOR_BATCH_SIZE:-10}
   pathPattern: ${EVENT_MONITOR_PATH_PATTERN:-["/api/v1/tables/*", "/api/v1/health-check"]}
   latency: ${EVENT_MONITOR_LATENCY:-[0.99, 0.90]} # For value p99=0.99, p90=0.90, p50=0.50 etc.
+  servicesHealthCheckInterval: ${EVENT_MONITOR_SERVICES_HEALTH_CHECK_INTERVAL:-300}
   # it will use the default auth provider for AWS services if parameters are not set
   # parameters:
   #   region: ${OM_MONITOR_REGION:-""}

docker/development/docker-compose.yml

@@ -492,7 +492,7 @@ services:
       DB_HOST: ${AIRFLOW_DB_HOST:-mysql}
       DB_PORT: ${AIRFLOW_DB_PORT:-3306}
       AIRFLOW_DB: ${AIRFLOW_DB:-airflow_db}
-      DB_SCHEME: ${AIRFLOW_DB_SCHEME:-mysql+pymysql}
+      DB_SCHEME: ${AIRFLOW_DB_SCHEME:-mysql+mysqldb}
       DB_USER: ${AIRFLOW_DB_USER:-airflow_user}
       DB_PASSWORD: ${AIRFLOW_DB_PASSWORD:-airflow_pass}
 

docker/docker-compose-ingestion/docker-compose-ingestion.yml

@@ -18,15 +18,15 @@ volumes:
 services:  
   ingestion:
     container_name: openmetadata_ingestion
-    image: docker.getcollate.io/openmetadata/ingestion:1.5.0-SNAPSHOT
+    image: docker.getcollate.io/openmetadata/ingestion:1.6.4
     environment:
       AIRFLOW__API__AUTH_BACKENDS: "airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session"
       AIRFLOW__CORE__EXECUTOR: LocalExecutor
       AIRFLOW__OPENMETADATA_AIRFLOW_APIS__DAG_GENERATED_CONFIGS: "/opt/airflow/dag_generated_configs"
       DB_HOST: ${AIRFLOW_DB_HOST:-mysql}
       DB_PORT: ${AIRFLOW_DB_PORT:-3306}
       AIRFLOW_DB: ${AIRFLOW_DB:-airflow_db}
-      DB_SCHEME: ${AIRFLOW_DB_SCHEME:-mysql+pymysql}
+      DB_SCHEME: ${AIRFLOW_DB_SCHEME:-mysql+mysqldb}
       DB_USER: ${AIRFLOW_DB_USER:-airflow_user}
       DB_PASSWORD: ${AIRFLOW_DB_PASSWORD:-airflow_pass}
       # extra connection-string properties for the database

docker/docker-compose-openmetadata/docker-compose-openmetadata.yml

@@ -14,7 +14,7 @@ services:
   execute-migrate-all:
     container_name: execute_migrate_all
     command: "./bootstrap/openmetadata-ops.sh migrate"
-    image: docker.getcollate.io/openmetadata/server:1.5.0-SNAPSHOT
+    image: docker.getcollate.io/openmetadata/server:1.6.4
     environment:
       OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata}
       SERVER_PORT: ${SERVER_PORT:-8585}
@@ -227,7 +227,7 @@ services:
   openmetadata-server:
     container_name: openmetadata_server
     restart: always
-    image: docker.getcollate.io/openmetadata/server:1.5.0-SNAPSHOT
+    image: docker.getcollate.io/openmetadata/server:1.6.4
     environment:
       OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata}
       SERVER_PORT: ${SERVER_PORT:-8585}

docker/docker-compose-quickstart/Dockerfile

@@ -11,7 +11,7 @@
 
 # Build stage
 FROM alpine:3 AS build
-ARG RI_VERSION="1.6.0-SNAPSHOT"
+ARG RI_VERSION="1.6.4"
 ENV RELEASE_URL="https://github.com/open-metadata/OpenMetadata/releases/download/${RI_VERSION}-release/openmetadata-${RI_VERSION}.tar.gz"
 
 RUN mkdir -p /opt/openmetadata && \
@@ -21,7 +21,7 @@ RUN mkdir -p /opt/openmetadata && \
 
 # Final stage
 FROM alpine:3
-ARG RI_VERSION="1.6.0-SNAPSHOT"
+ARG RI_VERSION="1.6.4"
 ARG BUILD_DATE
 ARG COMMIT_ID
 LABEL maintainer="OpenMetadata"