diff --git a/notebooks/create_catalog_and_schemas.ipynb b/notebooks/create_catalog_and_schemas.ipynb new file mode 100644 index 0000000..2c9a915 --- /dev/null +++ b/notebooks/create_catalog_and_schemas.ipynb @@ -0,0 +1,263 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "605f7f4f-ea1c-44d6-a014-7cc018a59757", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "source": [ + "### Notebook Purpose\n", + "\n", + "This notebook is intended to be run by Developer with appropiate priviliges to create catalogs/schemas to setup data layer for citibike project" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "492cab25-63c9-45be-bdcd-74f66972e44b", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "source": [ + "#### Catalog and schema creation" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "finishTime": 1782124349366, + "inputWidgets": {}, + "nuid": "a4e84857-8782-4fa7-9590-c22aadca8084", + "showTitle": false, + "startTime": 1782124348701, + "submitTime": 1782124345888, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "outputs": [], + "source": [ + "catalog = \"citibike_dev\" # change depending on environment" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "finishTime": 1782124352876, + "inputWidgets": {}, + "nuid": "d844b5fc-bb7b-49f9-8c24-dc5b68e70e05", + "showTitle": false, + "startTime": 1782124350222, + "submitTime": 1782124347173, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "DataFrame[]" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "spark.sql(f\"CREATE CATALOG IF NOT EXISTS {catalog}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "finishTime": 1782124356236, + "inputWidgets": {}, + "nuid": "c35380f5-b37a-477a-8830-7b24f3f8c9e7", + "showTitle": false, + "startTime": 1782124353127, + "submitTime": 1782124348862, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "DataFrame[]" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "spark.sql(f\"CREATE SCHEMA IF NOT EXISTS {catalog}.00_landing\")\n", + "spark.sql(f\"CREATE SCHEMA IF NOT EXISTS {catalog}.01_bronze\")\n", + "spark.sql(f\"CREATE SCHEMA IF NOT EXISTS {catalog}.02_silver\")\n", + "spark.sql(f\"CREATE SCHEMA IF NOT EXISTS {catalog}.03_gold\")" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "finishTime": 1782124357263, + "inputWidgets": {}, + "nuid": "1c24aa93-6b55-4fc9-b690-0b92f2c84bbe", + "showTitle": false, + "startTime": 1782124356291, + "submitTime": 1782124350186, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "DataFrame[]" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "spark.sql(f\"CREATE VOLUME IF NOT EXISTS {catalog}.00_landing.source_citibike_data\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "48e1c7b6-b258-49c1-a82d-6d61897725b7", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "source": [ + "#### Optional Step - prepolulate landing layer\n", + "\n", + "Cell below copies csv file with sample citibike data from current working directory to Databricks Volume\n", + "\n", + "**Prerequisites**: Import CSV file from `fixtures` folder to the same directory as this notebook." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "finishTime": 1782124440055, + "inputWidgets": {}, + "nuid": "e6b58a78-7032-49d2-ad42-c85d59c46c01", + "showTitle": false, + "startTime": 1782124437216, + "submitTime": 1782124437190, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from pathlib import Path\n", + "\n", + "cwd = Path.cwd()\n", + "catalog = \"citibike_dev\"\n", + "filename = 'JC-202503-citibike-tripdata.csv'\n", + "\n", + "fixture_path = f'dbfs:{cwd}/JC-202503-citibike-tripdata.csv'\n", + "volume_path = f'/Volumes/{catalog}/00_landing/source_citibike_data/{filename}'\n", + "\n", + "# Copy file from bundle to volume\n", + "dbutils.fs.cp(fixture_path, volume_path)" + ] + } + ], + "metadata": { + "application/vnd.databricks.v1+notebook": { + "computePreferences": null, + "dashboards": [], + "environmentMetadata": null, + "inputWidgetPreferences": null, + "language": "python", + "notebookMetadata": { + "pythonIndentUnit": 4 + }, + "notebookName": "create_catalog_and_schemas", + "widgets": {} + }, + "kernelspec": { + "display_name": ".venv_dbc (3.12.3)", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} \ No newline at end of file