✨ Add SQL DDLs for UC

notebooks/create_catalog_and_schemas.ipynb

@@ -0,0 +1,263 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {},
+     "inputWidgets": {},
+     "nuid": "605f7f4f-ea1c-44d6-a014-7cc018a59757",
+     "showTitle": false,
+     "tableResultSettingsMap": {},
+     "title": ""
+    }
+   },
+   "source": [
+    "### Notebook Purpose\n",
+    "\n",
+    "This notebook is intended to be run by Developer with appropiate priviliges to create catalogs/schemas to setup data layer for citibike project"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {},
+     "inputWidgets": {},
+     "nuid": "492cab25-63c9-45be-bdcd-74f66972e44b",
+     "showTitle": false,
+     "tableResultSettingsMap": {},
+     "title": ""
+    }
+   },
+   "source": [
+    "#### Catalog and schema creation"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 0,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "rowLimit": 10000
+     },
+     "finishTime": 1782124349366,
+     "inputWidgets": {},
+     "nuid": "a4e84857-8782-4fa7-9590-c22aadca8084",
+     "showTitle": false,
+     "startTime": 1782124348701,
+     "submitTime": 1782124345888,
+     "tableResultSettingsMap": {},
+     "title": ""
+    }
+   },
+   "outputs": [],
+   "source": [
+    "catalog = \"citibike_dev\"    # change depending on environment"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 0,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "rowLimit": 10000
+     },
+     "finishTime": 1782124352876,
+     "inputWidgets": {},
+     "nuid": "d844b5fc-bb7b-49f9-8c24-dc5b68e70e05",
+     "showTitle": false,
+     "startTime": 1782124350222,
+     "submitTime": 1782124347173,
+     "tableResultSettingsMap": {},
+     "title": ""
+    }
+   },
+   "outputs": [
+    {
+     "output_type": "execute_result",
+     "data": {
+      "text/plain": [
+       "DataFrame[]"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "spark.sql(f\"CREATE CATALOG IF NOT EXISTS {catalog}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 0,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "rowLimit": 10000
+     },
+     "finishTime": 1782124356236,
+     "inputWidgets": {},
+     "nuid": "c35380f5-b37a-477a-8830-7b24f3f8c9e7",
+     "showTitle": false,
+     "startTime": 1782124353127,
+     "submitTime": 1782124348862,
+     "tableResultSettingsMap": {},
+     "title": ""
+    }
+   },
+   "outputs": [
+    {
+     "output_type": "execute_result",
+     "data": {
+      "text/plain": [
+       "DataFrame[]"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "spark.sql(f\"CREATE SCHEMA IF NOT EXISTS {catalog}.00_landing\")\n",
+    "spark.sql(f\"CREATE SCHEMA IF NOT EXISTS {catalog}.01_bronze\")\n",
+    "spark.sql(f\"CREATE SCHEMA IF NOT EXISTS {catalog}.02_silver\")\n",
+    "spark.sql(f\"CREATE SCHEMA IF NOT EXISTS {catalog}.03_gold\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 0,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "rowLimit": 10000
+     },
+     "finishTime": 1782124357263,
+     "inputWidgets": {},
+     "nuid": "1c24aa93-6b55-4fc9-b690-0b92f2c84bbe",
+     "showTitle": false,
+     "startTime": 1782124356291,
+     "submitTime": 1782124350186,
+     "tableResultSettingsMap": {},
+     "title": ""
+    }
+   },
+   "outputs": [
+    {
+     "output_type": "execute_result",
+     "data": {
+      "text/plain": [
+       "DataFrame[]"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "spark.sql(f\"CREATE VOLUME IF NOT EXISTS {catalog}.00_landing.source_citibike_data\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {},
+     "inputWidgets": {},
+     "nuid": "48e1c7b6-b258-49c1-a82d-6d61897725b7",
+     "showTitle": false,
+     "tableResultSettingsMap": {},
+     "title": ""
+    }
+   },
+   "source": [
+    "#### Optional Step - prepolulate landing layer\n",
+    "\n",
+    "Cell below copies csv file with sample citibike data from current working directory to Databricks Volume\n",
+    "\n",
+    "**Prerequisites**: Import CSV file from `fixtures` folder to the same directory as this notebook."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 0,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "rowLimit": 10000
+     },
+     "finishTime": 1782124440055,
+     "inputWidgets": {},
+     "nuid": "e6b58a78-7032-49d2-ad42-c85d59c46c01",
+     "showTitle": false,
+     "startTime": 1782124437216,
+     "submitTime": 1782124437190,
+     "tableResultSettingsMap": {},
+     "title": ""
+    }
+   },
+   "outputs": [
+    {
+     "output_type": "execute_result",
+     "data": {
+      "text/plain": [
+       "True"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from pathlib import Path\n",
+    "\n",
+    "cwd = Path.cwd()\n",
+    "catalog = \"citibike_dev\"\n",
+    "filename = 'JC-202503-citibike-tripdata.csv'\n",
+    "\n",
+    "fixture_path = f'dbfs:{cwd}/JC-202503-citibike-tripdata.csv'\n",
+    "volume_path = f'/Volumes/{catalog}/00_landing/source_citibike_data/{filename}'\n",
+    "\n",
+    "# Copy file from bundle to volume\n",
+    "dbutils.fs.cp(fixture_path, volume_path)"
+   ]
+  }
+ ],
+ "metadata": {
+  "application/vnd.databricks.v1+notebook": {
+   "computePreferences": null,
+   "dashboards": [],
+   "environmentMetadata": null,
+   "inputWidgetPreferences": null,
+   "language": "python",
+   "notebookMetadata": {
+    "pythonIndentUnit": 4
+   },
+   "notebookName": "create_catalog_and_schemas",
+   "widgets": {}
+  },
+  "kernelspec": {
+   "display_name": ".venv_dbc (3.12.3)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "name": "python"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}