From 3346cfdcde44c8a2dadad70450bef90c4cc31790 Mon Sep 17 00:00:00 2001 From: devtmaz Date: Mon, 22 Jun 2026 11:21:42 +0200 Subject: [PATCH 1/5] =?UTF-8?q?=E2=9C=A8=20add=20setup=20scripts=20for=20u?= =?UTF-8?q?c=20catalog=20and=20schemas?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../00_create_catalog_and_schemas.ipynb | 60 +++++++++++++++++++ .../00_init/00_create_catalog_and_schemas.py | 18 ++++++ 2 files changed, 78 insertions(+) create mode 100644 citibike_etl/notebooks/00_init/00_create_catalog_and_schemas.ipynb create mode 100644 citibike_etl/scripts/00_init/00_create_catalog_and_schemas.py diff --git a/citibike_etl/notebooks/00_init/00_create_catalog_and_schemas.ipynb b/citibike_etl/notebooks/00_init/00_create_catalog_and_schemas.ipynb new file mode 100644 index 0000000..6cee5db --- /dev/null +++ b/citibike_etl/notebooks/00_init/00_create_catalog_and_schemas.ipynb @@ -0,0 +1,60 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "a9b84ddb", + "metadata": {}, + "outputs": [], + "source": [ + "catalog = dbutils.widgets.get(\"catalog\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a35cb495", + "metadata": {}, + "outputs": [], + "source": [ + "spark.sql(f\"CREATE CATALOG IF NOT EXISTS {catalog}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "86a47623", + "metadata": {}, + "outputs": [], + "source": [ + "spark.sql(f\"CREATE SCHEMA IF NOT EXISTS {catalog}.00_landing\")\n", + "spark.sql(f\"CREATE SCHEMA IF NOT EXISTS {catalog}.01_bronze\")\n", + "spark.sql(f\"CREATE SCHEMA IF NOT EXISTS {catalog}.02_silver\")\n", + "spark.sql(f\"CREATE SCHEMA IF NOT EXISTS {catalog}.03_gold\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5f838c1a", + "metadata": {}, + "outputs": [], + "source": [ + "spark.sql(f\"CREATE VOLUME IF NOT EXISTS {catalog}.00_landing.source_citibike_data\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv_dbc (3.12.3)", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/citibike_etl/scripts/00_init/00_create_catalog_and_schemas.py b/citibike_etl/scripts/00_init/00_create_catalog_and_schemas.py new file mode 100644 index 0000000..0440981 --- /dev/null +++ b/citibike_etl/scripts/00_init/00_create_catalog_and_schemas.py @@ -0,0 +1,18 @@ +import sys + +from pyspark.sql import SparkSession + +spark = SparkSession.builder.getOrCreate() +catalog = sys.argv[1] + + +spark.sql(f"CREATE CATALOG IF NOT EXISTS {catalog}") + +spark.sql(f"CREATE SCHEMA IF NOT EXISTS {catalog}.00_landing") +spark.sql(f"CREATE SCHEMA IF NOT EXISTS {catalog}.01_bronze") +spark.sql(f"CREATE SCHEMA IF NOT EXISTS {catalog}.02_silver") +spark.sql(f"CREATE SCHEMA IF NOT EXISTS {catalog}.03_gold") + +spark.sql( + f"CREATE VOLUME IF NOT EXISTS {catalog}.00_landing.source_citibike_data" +) From b650ead767908f2bb9ed7c567d63b2e599af4a78 Mon Sep 17 00:00:00 2001 From: devtmaz Date: Mon, 22 Jun 2026 11:22:10 +0200 Subject: [PATCH 2/5] =?UTF-8?q?=F0=9F=94=A7=20add=20setup=20task=20to=20jo?= =?UTF-8?q?b=20definitions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- resources/jobs/citibike_etl_pipeline_nb.job.yml | 10 ++++++++++ resources/jobs/citibike_etl_pipeline_py.job.yml | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/resources/jobs/citibike_etl_pipeline_nb.job.yml b/resources/jobs/citibike_etl_pipeline_nb.job.yml index 0bf9312..aca61e3 100644 --- a/resources/jobs/citibike_etl_pipeline_nb.job.yml +++ b/resources/jobs/citibike_etl_pipeline_nb.job.yml @@ -3,7 +3,17 @@ resources: citibike_etl_pipeline_nb: name: citibike_etl_pipeline_nb tasks: + - task_key: 00_init + notebook_task: + notebook_path: ../../citibike_etl/notebooks/00_init/00_create_catalog_and_schemas.ipynb + source: WORKSPACE + base_parameters: + # Pass value of variables + catalog: "${var.catalog}" + environment_key: Default - task_key: 01_bronze_citibike + depends_on: + - task_key: 00_init notebook_task: notebook_path: ../../citibike_etl/notebooks/01_bronze/01_bronze_citibike.ipynb source: WORKSPACE diff --git a/resources/jobs/citibike_etl_pipeline_py.job.yml b/resources/jobs/citibike_etl_pipeline_py.job.yml index 6e98a9b..415824b 100644 --- a/resources/jobs/citibike_etl_pipeline_py.job.yml +++ b/resources/jobs/citibike_etl_pipeline_py.job.yml @@ -3,7 +3,17 @@ resources: citibike_etl_pipeline_py: name: citibike_etl_pipeline_py tasks: + - task_key: 00_init + spark_python_task: + python_file: ../../citibike_etl/scripts/00_init/00_create_catalog_and_schemas.py + source: WORKSPACE + parameters: + # These parameters could be obtained by creating them in Jobs UI and copying from job YAML. + - "${var.catalog}" + environment_key: Default - task_key: 01_bronze_citibike + depends_on: + - task_key: 00_init spark_python_task: python_file: ../../citibike_etl/scripts/01_bronze/01_bronze_citibike.py source: WORKSPACE From 306c1a4931f0bc3e2bc191f4fbe1c3a15db20928 Mon Sep 17 00:00:00 2001 From: devtmaz Date: Mon, 22 Jun 2026 12:42:03 +0200 Subject: [PATCH 3/5] =?UTF-8?q?=E2=8F=AA=20revert=20task=20creation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- resources/jobs/citibike_etl_pipeline_nb.job.yml | 10 ---------- resources/jobs/citibike_etl_pipeline_py.job.yml | 10 ---------- 2 files changed, 20 deletions(-) diff --git a/resources/jobs/citibike_etl_pipeline_nb.job.yml b/resources/jobs/citibike_etl_pipeline_nb.job.yml index aca61e3..0bf9312 100644 --- a/resources/jobs/citibike_etl_pipeline_nb.job.yml +++ b/resources/jobs/citibike_etl_pipeline_nb.job.yml @@ -3,17 +3,7 @@ resources: citibike_etl_pipeline_nb: name: citibike_etl_pipeline_nb tasks: - - task_key: 00_init - notebook_task: - notebook_path: ../../citibike_etl/notebooks/00_init/00_create_catalog_and_schemas.ipynb - source: WORKSPACE - base_parameters: - # Pass value of variables - catalog: "${var.catalog}" - environment_key: Default - task_key: 01_bronze_citibike - depends_on: - - task_key: 00_init notebook_task: notebook_path: ../../citibike_etl/notebooks/01_bronze/01_bronze_citibike.ipynb source: WORKSPACE diff --git a/resources/jobs/citibike_etl_pipeline_py.job.yml b/resources/jobs/citibike_etl_pipeline_py.job.yml index 415824b..6e98a9b 100644 --- a/resources/jobs/citibike_etl_pipeline_py.job.yml +++ b/resources/jobs/citibike_etl_pipeline_py.job.yml @@ -3,17 +3,7 @@ resources: citibike_etl_pipeline_py: name: citibike_etl_pipeline_py tasks: - - task_key: 00_init - spark_python_task: - python_file: ../../citibike_etl/scripts/00_init/00_create_catalog_and_schemas.py - source: WORKSPACE - parameters: - # These parameters could be obtained by creating them in Jobs UI and copying from job YAML. - - "${var.catalog}" - environment_key: Default - task_key: 01_bronze_citibike - depends_on: - - task_key: 00_init spark_python_task: python_file: ../../citibike_etl/scripts/01_bronze/01_bronze_citibike.py source: WORKSPACE From d2426eb07a1322af289f465c191d904572e52179 Mon Sep 17 00:00:00 2001 From: devtmaz Date: Mon, 22 Jun 2026 12:42:17 +0200 Subject: [PATCH 4/5] =?UTF-8?q?Revert=20"=E2=9C=A8=20add=20setup=20scripts?= =?UTF-8?q?=20for=20uc=20catalog=20and=20schemas"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../00_create_catalog_and_schemas.ipynb | 60 ------------------- .../00_init/00_create_catalog_and_schemas.py | 18 ------ 2 files changed, 78 deletions(-) delete mode 100644 citibike_etl/notebooks/00_init/00_create_catalog_and_schemas.ipynb delete mode 100644 citibike_etl/scripts/00_init/00_create_catalog_and_schemas.py diff --git a/citibike_etl/notebooks/00_init/00_create_catalog_and_schemas.ipynb b/citibike_etl/notebooks/00_init/00_create_catalog_and_schemas.ipynb deleted file mode 100644 index 6cee5db..0000000 --- a/citibike_etl/notebooks/00_init/00_create_catalog_and_schemas.ipynb +++ /dev/null @@ -1,60 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "a9b84ddb", - "metadata": {}, - "outputs": [], - "source": [ - "catalog = dbutils.widgets.get(\"catalog\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a35cb495", - "metadata": {}, - "outputs": [], - "source": [ - "spark.sql(f\"CREATE CATALOG IF NOT EXISTS {catalog}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "86a47623", - "metadata": {}, - "outputs": [], - "source": [ - "spark.sql(f\"CREATE SCHEMA IF NOT EXISTS {catalog}.00_landing\")\n", - "spark.sql(f\"CREATE SCHEMA IF NOT EXISTS {catalog}.01_bronze\")\n", - "spark.sql(f\"CREATE SCHEMA IF NOT EXISTS {catalog}.02_silver\")\n", - "spark.sql(f\"CREATE SCHEMA IF NOT EXISTS {catalog}.03_gold\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5f838c1a", - "metadata": {}, - "outputs": [], - "source": [ - "spark.sql(f\"CREATE VOLUME IF NOT EXISTS {catalog}.00_landing.source_citibike_data\")" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": ".venv_dbc (3.12.3)", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python", - "version": "3.12.3" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/citibike_etl/scripts/00_init/00_create_catalog_and_schemas.py b/citibike_etl/scripts/00_init/00_create_catalog_and_schemas.py deleted file mode 100644 index 0440981..0000000 --- a/citibike_etl/scripts/00_init/00_create_catalog_and_schemas.py +++ /dev/null @@ -1,18 +0,0 @@ -import sys - -from pyspark.sql import SparkSession - -spark = SparkSession.builder.getOrCreate() -catalog = sys.argv[1] - - -spark.sql(f"CREATE CATALOG IF NOT EXISTS {catalog}") - -spark.sql(f"CREATE SCHEMA IF NOT EXISTS {catalog}.00_landing") -spark.sql(f"CREATE SCHEMA IF NOT EXISTS {catalog}.01_bronze") -spark.sql(f"CREATE SCHEMA IF NOT EXISTS {catalog}.02_silver") -spark.sql(f"CREATE SCHEMA IF NOT EXISTS {catalog}.03_gold") - -spark.sql( - f"CREATE VOLUME IF NOT EXISTS {catalog}.00_landing.source_citibike_data" -) From cabadba9ae417d9b89ad14eba2846eb87ad9b9fd Mon Sep 17 00:00:00 2001 From: devtmaz Date: Mon, 22 Jun 2026 12:42:37 +0200 Subject: [PATCH 5/5] =?UTF-8?q?=E2=9C=A8=20add=20script=20to=20setup=20cat?= =?UTF-8?q?alog=20and=20schemas?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- notebooks/create_catalog_and_schemas.ipynb | 263 +++++++++++++++++++++ 1 file changed, 263 insertions(+) create mode 100644 notebooks/create_catalog_and_schemas.ipynb diff --git a/notebooks/create_catalog_and_schemas.ipynb b/notebooks/create_catalog_and_schemas.ipynb new file mode 100644 index 0000000..2c9a915 --- /dev/null +++ b/notebooks/create_catalog_and_schemas.ipynb @@ -0,0 +1,263 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "605f7f4f-ea1c-44d6-a014-7cc018a59757", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "source": [ + "### Notebook Purpose\n", + "\n", + "This notebook is intended to be run by Developer with appropiate priviliges to create catalogs/schemas to setup data layer for citibike project" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "492cab25-63c9-45be-bdcd-74f66972e44b", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "source": [ + "#### Catalog and schema creation" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "finishTime": 1782124349366, + "inputWidgets": {}, + "nuid": "a4e84857-8782-4fa7-9590-c22aadca8084", + "showTitle": false, + "startTime": 1782124348701, + "submitTime": 1782124345888, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "outputs": [], + "source": [ + "catalog = \"citibike_dev\" # change depending on environment" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "finishTime": 1782124352876, + "inputWidgets": {}, + "nuid": "d844b5fc-bb7b-49f9-8c24-dc5b68e70e05", + "showTitle": false, + "startTime": 1782124350222, + "submitTime": 1782124347173, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "DataFrame[]" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "spark.sql(f\"CREATE CATALOG IF NOT EXISTS {catalog}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "finishTime": 1782124356236, + "inputWidgets": {}, + "nuid": "c35380f5-b37a-477a-8830-7b24f3f8c9e7", + "showTitle": false, + "startTime": 1782124353127, + "submitTime": 1782124348862, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "DataFrame[]" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "spark.sql(f\"CREATE SCHEMA IF NOT EXISTS {catalog}.00_landing\")\n", + "spark.sql(f\"CREATE SCHEMA IF NOT EXISTS {catalog}.01_bronze\")\n", + "spark.sql(f\"CREATE SCHEMA IF NOT EXISTS {catalog}.02_silver\")\n", + "spark.sql(f\"CREATE SCHEMA IF NOT EXISTS {catalog}.03_gold\")" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "finishTime": 1782124357263, + "inputWidgets": {}, + "nuid": "1c24aa93-6b55-4fc9-b690-0b92f2c84bbe", + "showTitle": false, + "startTime": 1782124356291, + "submitTime": 1782124350186, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "DataFrame[]" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "spark.sql(f\"CREATE VOLUME IF NOT EXISTS {catalog}.00_landing.source_citibike_data\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "48e1c7b6-b258-49c1-a82d-6d61897725b7", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "source": [ + "#### Optional Step - prepolulate landing layer\n", + "\n", + "Cell below copies csv file with sample citibike data from current working directory to Databricks Volume\n", + "\n", + "**Prerequisites**: Import CSV file from `fixtures` folder to the same directory as this notebook." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "finishTime": 1782124440055, + "inputWidgets": {}, + "nuid": "e6b58a78-7032-49d2-ad42-c85d59c46c01", + "showTitle": false, + "startTime": 1782124437216, + "submitTime": 1782124437190, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from pathlib import Path\n", + "\n", + "cwd = Path.cwd()\n", + "catalog = \"citibike_dev\"\n", + "filename = 'JC-202503-citibike-tripdata.csv'\n", + "\n", + "fixture_path = f'dbfs:{cwd}/JC-202503-citibike-tripdata.csv'\n", + "volume_path = f'/Volumes/{catalog}/00_landing/source_citibike_data/{filename}'\n", + "\n", + "# Copy file from bundle to volume\n", + "dbutils.fs.cp(fixture_path, volume_path)" + ] + } + ], + "metadata": { + "application/vnd.databricks.v1+notebook": { + "computePreferences": null, + "dashboards": [], + "environmentMetadata": null, + "inputWidgetPreferences": null, + "language": "python", + "notebookMetadata": { + "pythonIndentUnit": 4 + }, + "notebookName": "create_catalog_and_schemas", + "widgets": {} + }, + "kernelspec": { + "display_name": ".venv_dbc (3.12.3)", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} \ No newline at end of file