From 783e14ce020009bdd14186d3ce333bbec09786d0 Mon Sep 17 00:00:00 2001 From: "user1-63761607@lodsprodmca.onmicrosoft.com" Date: Mon, 27 Jul 2026 16:26:35 +0000 Subject: [PATCH 1/3] adding exercise --- Allfiles/01-explore-azure-databricks.ipynb | 192 ++++++++++++++++++--- 1 file changed, 170 insertions(+), 22 deletions(-) diff --git a/Allfiles/01-explore-azure-databricks.ipynb b/Allfiles/01-explore-azure-databricks.ipynb index af6e358..8b46625 100644 --- a/Allfiles/01-explore-azure-databricks.ipynb +++ b/Allfiles/01-explore-azure-databricks.ipynb @@ -2,8 +2,16 @@ "cells": [ { "cell_type": "markdown", - "id": "82c6d5ed", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "7a601857-78c8-4bab-b2a6-4389ddc11845", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "## Exercise 1: Running Python Code\n", "\n", @@ -17,25 +25,80 @@ "2. Prints a formatted message that summarises the route. For example:\n", " `Route: City Express | From: Central Station → To: Airport Terminal`\n", "\n", - "> 💬 **Genie Code tip:** Open Genie Code panel (click the ![assistant-icon](https://raw.githubusercontent.com/MicrosoftLearning/DP-750T00-Implement-Data-Engineering-Solutions-using-Azure-Databricks/refs/heads/main/Allfiles/media/databricks-assistant.svg) icon on the cell) and type:\n", + "> \uD83D\uDCAC **Genie Code tip:** Open Genie Code panel (click the ![assistant-icon](https://raw.githubusercontent.com/MicrosoftLearning/DP-750T00-Implement-Data-Engineering-Solutions-using-Azure-Databricks/refs/heads/main/Allfiles/media/databricks-assistant.svg) icon on the cell) and type:\n", "> *\"Write Python code to define three route variables and print a formatted summary message\"*\n", "> Use the suggestion as a starting point, then personalise the values." ] }, { "cell_type": "code", - "execution_count": null, - "id": "ef6b149a", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "784b33d8-e96a-4ecf-912c-b4c1ea50a4cc", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Cell 2" + } + }, "outputs": [], "source": [ - "# TODO: Define your route variables and print a formatted summary message" + "%python\n", + "\n", + "route_data =[]\n", + "route_id = ['R001' 'R002', 'R003']\n", + "route_name = ['City Express', 'Harbor Line', 'Metro']\n", + "origin = ['Central Station', 'Downtown', 'Airport Terminal']\n", + "destination = ['Airport Terminal', 'Airport Terminal', 'City Center']\n", + "\n", + "for i in range(len(route_id)):\n", + " route_data.append([route_id[i],route_name[i],origin[i],destination[i]])\n", + "\n", + "route_data\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "374888a6-5fa5-4ad1-a90a-6dab15ad7252", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Cell 3" + } + }, + "outputs": [], + "source": [ + "for i in range(len(route_data)):\n", + " print(f\"Route: {route_data[i][1]} | From: {route_data[i][2]} → To: {route_data[i][3]}\")" ] }, { "cell_type": "markdown", - "id": "0d2df199", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "55d3b919-ace8-4586-9f93-ae3b06e46e27", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "## Exercise 2: Running SQL Commands\n", "\n", @@ -50,26 +113,46 @@ "\n", "The result should be a single-row table with at least two columns.\n", "\n", - "> 💬 **Genie Code tip:** Ask Genie Code:\n", + "> \uD83D\uDCAC **Genie Code tip:** Ask Genie Code:\n", "> *\"Write a SQL SELECT statement in a Databricks notebook that returns the current timestamp and a string literal as named columns\"*\n", "> Remember to start your cell with `%sql`." ] }, { "cell_type": "code", - "execution_count": null, - "id": "01a150db", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "57a32746-f97d-44ec-9832-fcb1b9f76ebd", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ - "# TODO: Use the %sql magic command and write a SQL query\n", - "# Hint: start with %sql on the first line, then write your SELECT statement" + "%sql\n", + "SELECT current_timestamp() AS current_datetime, 'CityMoves Transit' AS system_name\n" ] }, { "cell_type": "markdown", - "id": "2a577b30", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "63912b42-24eb-43e2-8b7f-3ad3fc22e35a", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "## Exercise 3: Documenting with Markdown\n", "\n", @@ -83,23 +166,88 @@ "2. A short paragraph (2–3 sentences) describing the purpose of the notebook in the context of CityMoves Transit.\n", "3. A bulleted list (`-`) of at least three things a reader will find in this notebook.\n", "\n", - "> 💬 **Genie Code tip:** Select this markdown cell and ask Genie Code:\n", + "> \uD83D\uDCAC **Genie Code tip:** Select this markdown cell and ask Genie Code:\n", "> *\"Write a Markdown introduction cell for a data engineering notebook about a public transit system called CityMoves Transit\"*\n", "> Refine the generated text as you see fit." ] }, { "cell_type": "markdown", - "id": "974bf85b", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "6d007e6b-f716-477d-9bc1-1c92b36fdd3b", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "source": [ + "# CityMoves Transit — Data Engineering Notebook\n", + "\n", + "This notebook serves as an introductory exploration of the CityMoves Transit data platform built on Azure Databricks. It demonstrates core notebook capabilities that the data engineering team will use daily for route analytics and operational reporting.\n", + "\n", + "- Python code for defining and summarising transit route data\n", + "- SQL queries for retrieving system metadata and timestamps\n", + "- Markdown documentation for clear, shareable notebook structure" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "f398ad50-590e-48ac-874e-bba7e6217801", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "source": [] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "ee66acac-aec1-451c-a96b-c4498421ece1", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [] } ], "metadata": { + "application/vnd.databricks.v1+notebook": { + "computePreferences": null, + "dashboards": [], + "environmentMetadata": { + "base_environment": "", + "environment_version": "5" + }, + "inputWidgetPreferences": null, + "language": "python", + "notebookMetadata": { + "mostRecentlyExecutedCommandWithImplicitDF": { + "commandId": 3202781778147245, + "dataframes": [ + "_sqldf" + ] + }, + "pythonIndentUnit": 4 + }, + "notebookName": "01-explore-azure-databricks (1)", + "widgets": {} + }, "language_info": { "name": "python" } }, "nbformat": 4, - "nbformat_minor": 5 -} + "nbformat_minor": 0 +} \ No newline at end of file From a92adda794aaf3e7b4a891a3399f865f04fdadaa Mon Sep 17 00:00:00 2001 From: "gb.burcea@gmail.com" Date: Thu, 30 Jul 2026 16:49:18 +0000 Subject: [PATCH 2/3] second exercise --- .../02-select-and-configure-compute.ipynb | 196 +++++++++++++++--- 1 file changed, 162 insertions(+), 34 deletions(-) diff --git a/Allfiles/02-select-and-configure-compute.ipynb b/Allfiles/02-select-and-configure-compute.ipynb index 62deb25..516fe49 100644 --- a/Allfiles/02-select-and-configure-compute.ipynb +++ b/Allfiles/02-select-and-configure-compute.ipynb @@ -2,8 +2,16 @@ "cells": [ { "cell_type": "markdown", - "id": "f611b9a4", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "47edc795-82c4-4438-a096-8eaa26cc876a", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "## Exercise 3: Install Libraries Notebook-Scoped\n", "\n", @@ -16,14 +24,22 @@ }, { "cell_type": "markdown", - "id": "47d5e4b7", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "6e2f07ae-d264-45b2-abfc-b9d960754fc3", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 3.1 — Install the `faker` library notebook-scoped\n", "\n", "Your team relies on `faker` to generate synthetic patient records for pipeline testing. Install it as a notebook-scoped library so the package is available to this notebook only and doesn't affect other users sharing the same compute.\n", "\n", - "> 🤖 **Genie Code tip:** Open Genie Code panel (click the ![assistant-icon](https://raw.githubusercontent.com/MicrosoftLearning/DP-750T00-Implement-Data-Engineering-Solutions-using-Azure-Databricks/refs/heads/main/Allfiles/media/genie-code.svg) icon on the cell) and use the prompt below to get started:\n", + "> \uD83E\uDD16 **Genie Code tip:** Open Genie Code panel (click the ![assistant-icon](https://raw.githubusercontent.com/MicrosoftLearning/DP-750T00-Implement-Data-Engineering-Solutions-using-Azure-Databricks/refs/heads/main/Allfiles/media/genie-code.svg) icon on the cell) and use the prompt below to get started:\n", "> *\"How do I install a Python package notebook-scoped in Databricks using a magic command?\"*\n", "\n", "**Hint:** Use a `%pip` magic command to install the package. Pinning an exact version (e.g., `faker==40.8.0`) is a good practice for reproducibility." @@ -31,18 +47,37 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "667ecc3f", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "8ef69aa3-d5a0-44d9-ab99-1bb0cc8aa8a2", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ - "# TODO: Install faker==40.8.0 notebook-scoped" + "%pip install faker==40.8.0" ] }, { "cell_type": "markdown", - "id": "5145c763", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "d90646dc-cf99-442e-852c-dcd3ad8f1ae9", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 3.2 — Verify the installation\n", "\n", @@ -52,24 +87,48 @@ "2. Creating a `Faker` instance.\n", "3. Printing a randomly generated **full name** and **date of birth**.\n", "\n", - "> 🤖 **Genie Code tip:**\n", + "> \uD83E\uDD16 **Genie Code tip:**\n", "> *\"Show me a Python example of generating a random name and date of birth using the Faker library.\"*" ] }, { "cell_type": "code", - "execution_count": null, - "id": "df305ccc", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "916c5cad-4b4b-4946-ac40-11bd951ddea6", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ - "# TODO: Import Faker, create an instance, and print a generated name and date of birth" + "from faker import Faker\n", + "\n", + "fake = Faker()\n", + "\n", + "print(f\"Full Name: {fake.name()}\")\n", + "print(f\"Date of Birth: {fake.date_of_birth(minimum_age=20, maximum_age=80)}\")" ] }, { "cell_type": "markdown", - "id": "537b38a5", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "3f613c1f-763b-44d4-a070-d283dd24b382", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "## Exercise 4: Generate and Analyze Synthetic Patient Data\n", "\n", @@ -80,8 +139,16 @@ }, { "cell_type": "markdown", - "id": "ee221a7b", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "677df061-0d6b-40d3-a2ef-d2de6128f607", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 4.1 — Generate a synthetic patient admissions DataFrame\n", "\n", @@ -97,7 +164,7 @@ "\n", "Display the first 10 rows of the resulting DataFrame.\n", "\n", - "> 🤖 **Genie Code tip:**\n", + "> \uD83E\uDD16 **Genie Code tip:**\n", "> *\"How do I generate a list of Python dictionaries using the Faker library, then create a PySpark DataFrame from that list?\"*\n", ">\n", "> **Hint:** Use `faker.date_of_birth()` with `minimum_age` and `maximum_age` parameters, or `faker.date_between()` with `start_date` and `end_date`. Use `random.choice()` for the diagnosis code." @@ -105,19 +172,53 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "b38ec3d4", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "9d3b3ee7-b41b-477f-844e-e9237abd9be0", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ - "# TODO: Generate 100 synthetic patient records and create a Spark DataFrame\n", - "# Display the first 10 rows" + "from faker import Faker\n", + "import random\n", + "from datetime import date\n", + "fake = Faker()\n", + "diagnosis_codes = [\"I21\", 'J18', 'E11', 'K80', 'N39']\n", + "\n", + "records = []\n", + "for i in range(1, 1001):\n", + " records.append({\n", + " \"patient_id\": i, \n", + " \"full_name\": fake.name(), \n", + " \"date_of_birth\": str(fake.date_between(start_date = date(1940, 1, 1), end_date = date(2005, 12, 31))), \n", + " \"admission_date\":str(fake.date_between(start_date=date(2023, 1, 1), end_date= date(2025, 12, 31))), \n", + " \"diagnosis_code\": random.choice(diagnosis_codes)\n", + " })\n", + "df = spark.createDataFrame(records)\n", + "display(df)" ] }, { "cell_type": "markdown", - "id": "034453d6", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "947491cd-51e9-48c7-8f48-f342f6c12935", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 4.2 — Analyze admissions by diagnosis code\n", "\n", @@ -127,26 +228,53 @@ "2. Order the results from **most to least** admissions.\n", "3. Display the result.\n", "\n", - "> 🤖 **Genie Code tip:**\n", + "> \uD83E\uDD16 **Genie Code tip:**\n", "> *\"How do I group by a column, count occurrences, and sort the result in descending order using PySpark DataFrame API?\"*" ] }, { "cell_type": "code", - "execution_count": null, - "id": "0115d27c", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "cba9771b-976e-40e3-ba2c-11c7d771481b", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ - "# TODO: Count admissions per diagnosis_code, ordered from most to least" + "from pyspark.sql.functions import col \n", + "\n", + "\n", + "diagnosis_counts = df.groupBy(\"diagnosis_code\").count().orderBy(col(\"count\").desc())\n", + "display(diagnosis_counts)" ] } ], "metadata": { + "application/vnd.databricks.v1+notebook": { + "computePreferences": null, + "dashboards": [], + "environmentMetadata": null, + "inputWidgetPreferences": null, + "language": "python", + "notebookMetadata": { + "pythonIndentUnit": 4 + }, + "notebookName": "02-select-and-configure-compute", + "widgets": {} + }, "language_info": { "name": "python" } }, "nbformat": 4, - "nbformat_minor": 5 + "nbformat_minor": 0 } From 73bb60ff350b14451b03798662d4aef871f9b24d Mon Sep 17 00:00:00 2001 From: "user1-63853039@lodsprodmca.onmicrosoft.com" Date: Thu, 30 Jul 2026 19:04:17 +0000 Subject: [PATCH 3/3] adding 3rd exercise --- ...nd-organize-objects-in-unity-catalog.ipynb | 979 +++++++++++++++--- 1 file changed, 814 insertions(+), 165 deletions(-) diff --git a/Allfiles/03-create-and-organize-objects-in-unity-catalog.ipynb b/Allfiles/03-create-and-organize-objects-in-unity-catalog.ipynb index 7e49032..fddca91 100644 --- a/Allfiles/03-create-and-organize-objects-in-unity-catalog.ipynb +++ b/Allfiles/03-create-and-organize-objects-in-unity-catalog.ipynb @@ -2,8 +2,19 @@ "cells": [ { "cell_type": "markdown", - "id": "86081d58", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "d85d70ca-f48a-4fd3-9fd6-d4f3d60167fc", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "## Exercise 1: Set Up the Catalog Structure\n", "\n", @@ -16,7 +27,7 @@ "```txt\n", "\n", "Unity Catalog\n", - "└── edu_dev [catalog]\n", + "└── edu_dev63853039 [catalog]\n", " │ tags: environment=development | university=lakeside | data_classification=internal\n", " │\n", " ├── bronze [schema]\n", @@ -61,18 +72,29 @@ }, { "cell_type": "markdown", - "id": "293e733e", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "bf42b44b-1e8b-4b2a-930f-d5c2f97b88d1", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 1.1 — Create the development catalog\n", "\n", - "Create a catalog called `edu_dev` with a descriptive comment. This catalog serves as the **development environment** for Lakeside University's data platform.\n", + "Create a catalog called `edu_dev63853039` with a descriptive comment. This catalog serves as the **development environment** for Lakeside University's data platform.\n", "\n", "**Naming convention reminders:**\n", "- Use lowercase with underscores\n", "- No periods, spaces, forward slashes, or control characters\n", "\n", - "> 🤖 **Genie Code tip:** Ask Genie Code:\n", + "> \uD83E\uDD16 **Genie Code tip:** Ask Genie Code:\n", "> *\"How do I create a Unity Catalog catalog in Databricks SQL with a comment?\"*\n", "\n", "**Hint:** Use the SQL command `CREATE CATALOG`." @@ -80,9 +102,20 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "9933f5b5", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "56373874-bc35-479c-8712-45bb01547a79", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "# Since no default storage is enabled, we are inheriting the storage path from the default catalog's root.\n", @@ -101,23 +134,48 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "e38aeda3", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "47ea25a1-9872-4855-9ad6-6c41b049141e", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Cell 5" + } + }, "outputs": [], "source": [ "%sql\n", - "-- TODO: Create the edu_dev catalog with a descriptive comment" + "CREATE CATALOG IF NOT EXISTS edu_dev63853039\n", + "MANAGED LOCATION 'abfss://unity-catalog-storage@dbstoragep3bbxu5v55osy.dfs.core.windows.net/7405616271049399'\n", + "COMMENT 'Development catalog for Lakeside University data platform';" ] }, { "cell_type": "markdown", - "id": "3470e517", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "f6b3c97c-ac3b-433b-8114-ecc45c3aefd2", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 1.2 — Create medallion schemas\n", "\n", - "Within the `edu_dev` catalog, create three schemas that implement the **medallion architecture**:\n", + "Within the `edu_dev63853039` catalog, create three schemas that implement the **medallion architecture**:\n", "\n", "| Schema | Purpose |\n", "|--------|---------|\n", @@ -125,7 +183,7 @@ "| `silver` | Cleaned, validated, and enriched data |\n", "| `gold` | Aggregated, analytics-ready data |\n", "\n", - "> 🤖 **Genie Code tip:** Ask:\n", + "> \uD83E\uDD16 **Genie Code tip:** Ask:\n", "> *\"Show me how to create multiple schemas in a Databricks Unity Catalog catalog using SQL.\"*\n", "\n", "**Hint:** Use `CREATE SCHEMA IF NOT EXISTS catalog.schema_name` for each schema." @@ -133,19 +191,49 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "f82b0009", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "d668e74c-1ef6-4a77-b33e-c496f517ee52", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql\n", - "-- TODO: Create the bronze, silver, and gold schemas inside edu_dev" + "CREATE SCHEMA IF NOT EXISTS edu_dev63853039.bronze\n", + "COMMENT 'Raw ingested data - unmodified source files'; \n", + "\n", + "CREATE SCHEMA IF NOT EXISTS edu_dev63853039.silver\n", + "COMMENT 'Cleaned, validated, and enriched data';\n", + "\n", + "CREATE SCHEMA IF NOT EXISTS edu_dev63853039.gold\n", + "COMMENT 'Aggregated and ready for analysis';\n" ] }, { "cell_type": "markdown", - "id": "fc2c333a", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "b14d3471-b8a1-4906-a503-ea9e4ab61181", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "## Exercise 2: Create Tables with Constraints\n", "\n", @@ -154,12 +242,23 @@ }, { "cell_type": "markdown", - "id": "43cc1d95", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "55175eca-57fa-4eda-bd30-203cc34d41ac", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 2.1 — Create the `students` table\n", "\n", - "Create a managed Delta table `edu_dev.silver.students` to store student records with the following columns:\n", + "Create a managed Delta table `edu_dev63853039.silver.students` to store student records with the following columns:\n", "\n", "| Column | Type | Description |\n", "|--------|------|-------------|\n", @@ -170,7 +269,7 @@ "| `enrollment_year` | INT | Year the student enrolled |\n", "| `program` | STRING | Degree program |\n", "\n", - "> 🤖 **Genie Code tip:** Ask:\n", + "> \uD83E\uDD16 **Genie Code tip:** Ask:\n", "> *\"How do I create a managed Delta table with a primary key constraint in Unity Catalog?\"*\n", "\n", "**Hint:** Add `CONSTRAINT students_pk PRIMARY KEY (student_id)` at the end of the column list." @@ -178,23 +277,55 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "c84aa6e8", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "545e9034-88b2-4640-89f1-b511280c027e", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql\n", - "-- TODO: Create edu_dev.silver.students with a primary key constraint" + "CREATE TABLE IF NOT EXISTS edu_dev63853039.silver.students (\n", + "\n", + " student_id BIGINT NOT NULL,\n", + " first_name STRING, \n", + " last_name STRING, \n", + " email STRING, \n", + " enrolled_date TIMESTAMP, \n", + " program STRING, \n", + " CONSTRAINT students_pk PRIMARY KEY(student_id)\n", + ");" ] }, { "cell_type": "markdown", - "id": "62473788", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "2142e2db-871d-442e-a8cb-018e428970cc", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 2.2 — Create the `courses` table\n", "\n", - "Create `edu_dev.silver.courses` with the following columns:\n", + "Create `edu_dev63853039.silver.courses` with the following columns:\n", "\n", "| Column | Type | Description |\n", "|--------|------|-------------|\n", @@ -203,29 +334,58 @@ "| `department` | STRING | Academic department offering the course |\n", "| `credits` | INT | Number of academic credits |\n", "\n", - "> 🤖 **Genie Code tip:** Ask:\n", + "> \uD83E\uDD16 **Genie Code tip:** Ask:\n", "> *\"Write a CREATE TABLE statement for a courses table in Databricks Unity Catalog with a primary key constraint.\"*" ] }, { "cell_type": "code", - "execution_count": null, - "id": "12121b41", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "a684072c-2646-4b8d-91ff-eb898d9411c8", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql\n", - "-- TODO: Create edu_dev.silver.courses with a primary key constraint" + "CREATE TABLE IF NOT EXISTS edu_dev63853039.silver.courses (\n", + " course_id BIGINT NOT NULL, \n", + " course_name STRING, \n", + " department STRING, \n", + " credits INT, \n", + " CONSTRAINT courses_pk PRIMARY KEY(course_id)\n", + "); " ] }, { "cell_type": "markdown", - "id": "1a2238fd", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "b0465fe0-40c2-4322-9d9e-dcf776f63955", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 2.3 — Create the `enrollments` table with foreign keys\n", "\n", - "Create `edu_dev.silver.enrollments` to record which students are enrolled in which courses. This table must reference both `students` and `courses` via foreign key constraints.\n", + "Create `edu_dev63853039.silver.enrollments` to record which students are enrolled in which courses. This table must reference both `students` and `courses` via foreign key constraints.\n", "\n", "| Column | Type | Description |\n", "|--------|------|-------------|\n", @@ -235,7 +395,7 @@ "| `semester` | STRING | Academic semester (e.g., `Spring 2023`) |\n", "| `grade` | DECIMAL(4,2) | Numerical grade on a 0.0–10.0 scale |\n", "\n", - "> 🤖 **Genie Code tip:** Ask:\n", + "> \uD83E\uDD16 **Genie Code tip:** Ask:\n", "> *\"How do I add foreign key constraints referencing other tables when creating a table in Databricks Unity Catalog?\"*\n", "\n", "**Hint:** Use `CONSTRAINT FOREIGN KEY () REFERENCES ..()` for each foreign key." @@ -243,19 +403,51 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "aaf6a330", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "cbed427e-ab8d-41f7-89b7-a5c83975fe0c", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql\n", - "-- TODO: Create edu_dev.silver.enrollments with primary key and two foreign key constraints" + "CREATE TABLE IF NOT EXISTS edu_dev63853039.silver.enrollments (\n", + " enrollment_id BIGINT NOT NULL, \n", + " student_id BIGINT, \n", + " course_id BIGINT, \n", + " semester STRING, \n", + " grade DECIMAL(4,2), \n", + " CONSTRAINT enrollments_pk PRIMARY KEY(enrollment_id),\n", + " CONSTRAINT enrollments_students_fk FOREIGN KEY(student_id) REFERENCES edu_dev63853039.silver.students(student_id),\n", + " CONSTRAINT enrollments_courses_fk FOREIGN KEY(course_id) REFERENCES edu_dev63853039.silver.courses(course_id)\n", + ");" ] }, { "cell_type": "markdown", - "id": "e06d70c9", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "eac7be32-e47e-4ae5-9a7b-b1d802893636", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Sample data — run this cell\n", "\n", @@ -264,14 +456,26 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "3de44411", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "5a612473-233c-4115-a415-2da3a4da9fc4", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql\n", "-- Populate students\n", - "INSERT INTO edu_dev.silver.students VALUES\n", + "INSERT INTO edu_dev63853039.silver.students VALUES\n", "(1001, 'Emma', 'Watson', 'emma.watson@lakeside.edu', 2022, 'Computer Science'),\n", "(1002, 'Liam', 'Ahmed', 'liam.ahmed@lakeside.edu', 2023, 'Data Science'),\n", "(1003, 'Sofia', 'Chen', 'sofia.chen@lakeside.edu', 2021, 'Mathematics'),\n", @@ -284,7 +488,7 @@ "(1010, 'Jake', 'OBrien', 'jake.obrien@lakeside.edu', 2020, 'Computer Science');\n", "\n", "-- Populate courses\n", - "INSERT INTO edu_dev.silver.courses VALUES\n", + "INSERT INTO edu_dev63853039.silver.courses VALUES\n", "(101, 'Introduction to Programming', 'Computer Science', 3),\n", "(102, 'Data Structures', 'Computer Science', 4),\n", "(103, 'Statistics for Data Science', 'Data Science', 3),\n", @@ -293,7 +497,7 @@ "(106, 'Linear Algebra', 'Mathematics', 3);\n", "\n", "-- Populate enrollments\n", - "INSERT INTO edu_dev.silver.enrollments VALUES\n", + "INSERT INTO edu_dev63853039.silver.enrollments VALUES\n", "(1, 1001, 101, 'Spring 2023', 8.50),\n", "(2, 1001, 102, 'Fall 2023', 7.80),\n", "(3, 1002, 103, 'Spring 2023', 9.20),\n", @@ -313,8 +517,19 @@ }, { "cell_type": "markdown", - "id": "4333ac8a", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "497190f2-178f-4375-b070-04575c16b6a7", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "## Exercise 3: Create Views\n", "\n", @@ -323,12 +538,23 @@ }, { "cell_type": "markdown", - "id": "da6bf6d7", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "945011ce-9172-404c-a691-5a918c45884a", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 3.1 — Create a standard view\n", "\n", - "Create a view `edu_dev.silver.vw_student_enrollments` that presents a combined, human-readable view of student enrollment records. It should include:\n", + "Create a view `edu_dev63853039.silver.vw_student_enrollments` that presents a combined, human-readable view of student enrollment records. It should include:\n", "\n", "- Student full name (concatenate `first_name` and `last_name` with a space between them)\n", "- Student `email`\n", @@ -337,7 +563,7 @@ "- `semester`\n", "- `grade`\n", "\n", - "> 🤖 **Genie Code tip:** Ask:\n", + "> \uD83E\uDD16 **Genie Code tip:** Ask:\n", "> *\"Write a SQL view in Azure Databricks that joins three tables: students, courses, and enrollments\"*\n", "\n", "**Hint:** Use `CONCAT(first_name, ' ', last_name)` or the `||` operator to combine names. You need to join all three silver tables." @@ -345,24 +571,70 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "36092d06", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "f708c615-da8a-49c1-a866-3f3e2b3841ea", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Cell 18" + } + }, "outputs": [], "source": [ "%sql\n", - "-- TODO: Create the edu_dev.silver.vw_student_enrollments view\n", - "-- Join students, courses, and enrollments; include full_name, email, course_name, department, semester, grade" + "-- TODO: Create the edu_dev63853039.silver.vw_student_enrollments view\n", + "-- Join students, courses, and enrollments; include full_name, email, course_name, department, semester, grade\n", + "\n", + "DROP TABLE IF EXISTS edu_dev63853039.silver.vw_student_enrollments;\n", + "\n", + "CREATE OR REPLACE VIEW edu_dev63853039.silver.vw_student_enrollments AS (\n", + " SELECT \n", + " CONCAT(s.first_name, ' ', s.last_name) \n", + " AS full_name,\n", + " s.email,\n", + " c.course_name,\n", + " c.department,\n", + " e.semester,\n", + " e.grade\n", + " FROM \n", + " edu_dev63853039.silver.enrollments e \n", + " INNER JOIN \n", + " edu_dev63853039.silver.students s\n", + " ON \n", + " e.student_id = s.student_id\n", + " INNER JOIN \n", + " edu_dev63853039.silver.courses c\n", + " ON \n", + " e.course_id = c.course_id\n", + ");" ] }, { "cell_type": "markdown", - "id": "304bd8cb", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "489032f4-79bb-40b0-8811-6184c425096d", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 3.2 — Create a materialized view for department statistics\n", "\n", - "Create a materialized view `edu_dev.gold.vw_department_enrollment_stats` that pre-computes the following per academic department:\n", + "Create a materialized view `edu_dev63853039.gold.vw_department_enrollment_stats` that pre-computes the following per academic department:\n", "\n", "| Output column | Description |\n", "|--------------|-------------|\n", @@ -371,7 +643,7 @@ "| `avg_grade` | Average grade, rounded to 2 decimal places |\n", "| `distinct_students` | Number of unique students enrolled |\n", "\n", - "> 🤖 **Genie Code tip:** Ask:\n", + "> \uD83E\uDD16 **Genie Code tip:** Ask:\n", "> *\"What is the syntax for creating a materialized view in Azure Databricks that aggregates data from two joined tables?\"*\n", "\n", "**Hint:** Use `CREATE MATERIALIZED VIEW`, join `enrollments` with `courses`, and use `COUNT()`, `ROUND(AVG(...), 2)`, and `COUNT(DISTINCT ...)`.\n", @@ -382,19 +654,58 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "589ad9da", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "b607649d-b674-4d19-b938-02c58e26869c", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Cell 20" + } + }, "outputs": [], "source": [ "%sql\n", - "-- TODO: Create the edu_dev.gold.vw_department_enrollment_stats materialized view" + "-- TODO: Create the edu_dev63853039.gold.vw_department_enrollment_stats materialized view\n", + "\n", + "CREATE MATERIALIZED VIEW IF NOT EXISTS edu_dev63853039.gold.vw_department_enrollment_stats AS (\n", + " SELECT \n", + " c.department,\n", + " COUNT(*) AS total_enrollments,\n", + " ROUND(AVG(e.grade), 2) AS avg_grade,\n", + " COUNT(DISTINCT e.student_id) AS distinct_students\n", + " FROM \n", + " edu_dev63853039.silver.enrollments e\n", + " INNER JOIN \n", + " edu_dev63853039.silver.courses c\n", + " ON \n", + " e.course_id = c.course_id\n", + " GROUP BY \n", + " c.department\n", + ");" ] }, { "cell_type": "markdown", - "id": "4ab4b66e", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "fc765d60-2062-416e-9db2-62509ffdcc14", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "**✅ Test your materialized view** — Run the following query to verify it returns per-department aggregates:\n", "\n", @@ -403,19 +714,42 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "d3b2354e", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "f38a40de-a840-4b78-9af6-9230ae7584cb", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql\n", - "SELECT * FROM edu_dev.gold.vw_department_enrollment_stats;" + "SELECT * FROM edu_dev63853039.gold.vw_department_enrollment_stats;" ] }, { "cell_type": "markdown", - "id": "77ee534b", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "fc9a15af-bfb9-4244-b301-23b82b2fdfc8", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "## Exercise 4: Create a Volume and Read Files\n", "\n", @@ -424,14 +758,25 @@ }, { "cell_type": "markdown", - "id": "a8fca075", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "be2a335a-742a-49e5-8b43-5ebaee21e67c", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 4.1 — Create a managed volume\n", "\n", - "Create a managed volume named `raw_files` in the `edu_dev.bronze` schema. This will serve as the landing area for raw data files.\n", + "Create a managed volume named `raw_files` in the `edu_dev63853039.bronze` schema. This will serve as the landing area for raw data files.\n", "\n", - "> 🤖 **Genie Code tip:** Ask:\n", + "> \uD83E\uDD16 **Genie Code tip:** Ask:\n", "> *\"How do I create a managed volume in Unity Catalog using a SQL statement?\"*\n", "\n", "**Hint:** Use `CREATE VOLUME IF NOT EXISTS ..`." @@ -439,19 +784,43 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "c21a6290", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "c67111e9-a135-47e9-a880-9e643608a798", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql\n", - "-- TODO: Create the raw_files managed volume in edu_dev.bronze" + "-- TODO: Create the raw_files managed volume in edu_dev63853039.bronze\n", + "\n", + "CREATE VOLUME IF NOT EXISTS edu_dev63853039.bronze.raw_files" ] }, { "cell_type": "markdown", - "id": "5252fe12", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "d2132d9c-6c9c-462b-abfd-a6be4afbae5c", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 4.2 — Write a CSV file to the volume\n", "\n", @@ -460,9 +829,20 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "6c348308", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "a047fe3b-0fb4-4ca3-92ac-85789743bd24", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "# Write a sample student CSV file to the volume — run this cell without modification\n", @@ -478,20 +858,31 @@ "1009,Mei,Lindqvist,mei.lindqvist@lakeside.edu,2023,Mathematics\n", "1010,Jake,OBrien,jake.obrien@lakeside.edu,2020,Computer Science\"\"\"\n", "\n", - "dbutils.fs.put(\"/Volumes/edu_dev/bronze/raw_files/students.csv\", csv_content, overwrite=True)\n", - "print(\"students.csv written to /Volumes/edu_dev/bronze/raw_files/\")" + "dbutils.fs.put(\"/Volumes/edu_dev63853039/bronze/raw_files/students.csv\", csv_content, overwrite=True)\n", + "print(\"students.csv written to /Volumes/edu_dev63853039/bronze/raw_files/\")" ] }, { "cell_type": "markdown", - "id": "6823ffb1", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "d7554b95-5c1c-4cd9-8b49-a7d779f40c67", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 4.3 — Read the CSV from the volume into a DataFrame\n", "\n", - "Read the CSV file you just placed in the volume into a Spark DataFrame and display its contents. The file is at `/Volumes/edu_dev/bronze/raw_files/students.csv`.\n", + "Read the CSV file you just placed in the volume into a Spark DataFrame and display its contents. The file is at `/Volumes/edu_dev63853039/bronze/raw_files/students.csv`.\n", "\n", - "> 🤖 **Genie Code tip:** Ask:\n", + "> \uD83E\uDD16 **Genie Code tip:** Ask:\n", "> *\"How do I read a CSV file with a header row from a Unity Catalog volume path in PySpark?\"*\n", "\n", "**Hint:** Use `spark.read.csv(path, header=True, inferSchema=True)` then call `.display()` or `.show()` on the result." @@ -499,18 +890,41 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "4f2b6ef8", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "2e2ecc7c-205e-4959-a66e-ece8c2b5a98c", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ - "# TODO: Read /Volumes/edu_dev/bronze/raw_files/students.csv into a Spark DataFrame and display it" + "# TODO: Read /Volumes/edu_dev63853039/bronze/raw_files/students.csv into a Spark DataFrame and display it\n", + "spark.read.csv('/Volumes/edu_dev63853039/bronze/raw_files/students.csv', header=True, inferSchema=True).display()" ] }, { "cell_type": "markdown", - "id": "d9848d3c", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "f6988ea6-f83f-4725-94ca-e19b2c64a3dc", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "## Exercise 5: Create a Reusable SQL Function\n", "\n", @@ -519,12 +933,23 @@ }, { "cell_type": "markdown", - "id": "efc0eb82", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "cf215b86-565c-4e54-ae30-9db6468e76c2", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 5.1 — Create a grade classification function\n", "\n", - "Create a SQL scalar function `edu_dev.silver.get_grade_classification` that accepts a `grade DECIMAL(4,2)` and returns a `STRING` letter classification based on the scale used at Lakeside University:\n", + "Create a SQL scalar function `edu_dev63853039.silver.get_grade_classification` that accepts a `grade DECIMAL(4,2)` and returns a `STRING` letter classification based on the scale used at Lakeside University:\n", "\n", "| Grade range | Classification |\n", "|-------------|----------------|\n", @@ -534,7 +959,7 @@ "| ≥ 4.0 | `'D'` |\n", "| < 4.0 | `'F'` |\n", "\n", - "> 🤖 **Genie Code tip:** Ask:\n", + "> \uD83E\uDD16 **Genie Code tip:** Ask:\n", "> *\"How do I create a SQL scalar function in Unity Catalog that uses a CASE expression to return a letter grade based on a numeric score?\"*\n", "\n", "**Hint:** Use `CREATE FUNCTION ... RETURNS STRING RETURN CASE WHEN ... END`." @@ -542,61 +967,140 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "abd7299f", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "4e00efc2-8578-4d14-9191-aca2527edc74", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql\n", - "-- TODO: Create the edu_dev.silver.get_grade_classification function" + "-- TODO: Create the edu_dev63853039.silver.get_grade_classification function\n", + "\n", + "CREATE OR REPLACE FUNCTION get_grade_classification(grade\n", + "DECIMAL(4, 2))\n", + "RETURNS STRING\n", + "RETURN CASE\n", + " WHEN grade >= 8.5 THEN 'A'\n", + " WHEN grade >= 7.0 THEN 'B'\n", + " WHEN grade >= 5.5 THEN 'C'\n", + " WHEN grade >= 4.0 THEN 'D'\n", + " ELSE 'F'\n", + "END;" ] }, { "cell_type": "markdown", - "id": "c96c404a", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "a1a7118b-7f8b-4741-a35b-05fcfbca6d46", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 5.2 — Test the function\n", "\n", - "Query `edu_dev.silver.enrollments` and apply your new function to produce a `grade_classification` column for each row. Include: `enrollment_id`, `student_id`, `course_id`, `semester`, `grade`, and `grade_classification`.\n", + "Query `edu_dev63853039.silver.enrollments` and apply your new function to produce a `grade_classification` column for each row. Include: `enrollment_id`, `student_id`, `course_id`, `semester`, `grade`, and `grade_classification`.\n", "\n", - "> 🤖 **Genie Code tip:** Ask:\n", + "> \uD83E\uDD16 **Genie Code tip:** Ask:\n", "> *\"How do I call a Unity Catalog user-defined function in a SELECT statement in Azure Databricks?\"*" ] }, { "cell_type": "code", - "execution_count": null, - "id": "60f10718", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "27ab739a-3b97-4770-b159-26d561be1a1c", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql\n", - "-- TODO: Query edu_dev.silver.enrollments and call get_grade_classification on the grade column" + "-- TODO: Query edu_dev63853039.silver.enrollments and call get_grade_classification on the grade column\n", + "\n", + "SELECT \n", + " enrollment_id, \n", + " student_id, \n", + " course_id,\n", + " semester,\n", + " grade,\n", + " get_grade_classification(grade) AS grade_classification\n", + "FROM \n", + " edu_dev63853039.silver.enrollments;\n", + " \n" ] }, { "cell_type": "markdown", - "id": "ed144885", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "ccef6d90-975b-442d-b5c3-7d92c3c855d5", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "## Exercise 6: DDL Operations\n", "\n", "Unity Catalog objects evolve over time as requirements change. In this exercise, you use `ALTER` statements to:\n", "1. Extend the `students` table with a new column.\n", - "2. Apply governance **tags** to the `edu_dev` catalog." + "2. Apply governance **tags** to the `edu_dev63853039` catalog." ] }, { "cell_type": "markdown", - "id": "c6b30ba9", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "4b8e4332-a4df-4e0e-b126-d0bdae092aec", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 6.1 — Add a column to the students table\n", "\n", - "The student registration system has been updated to capture phone numbers. Add a nullable `phone_number` column of type `STRING` to `edu_dev.silver.students`.\n", + "The student registration system has been updated to capture phone numbers. Add a nullable `phone_number` column of type `STRING` to `edu_dev63853039.silver.students`.\n", "\n", - "> 🤖 **Genie Code tip:** Ask:\n", + "> \uD83E\uDD16 **Genie Code tip:** Ask:\n", "> *\"How do I add a new nullable column to an existing Delta table in Azure Databricks using SQL?\"*\n", "\n", "**Hint:** Use `ALTER TABLE`." @@ -604,23 +1108,49 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "85c7df17", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "461fd0b3-e8fd-40cb-8127-ac459821105e", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql\n", - "-- TODO: Add a phone_number (STRING) column to edu_dev.silver.students" + "-- TODO: Add a phone_number (STRING) column to edu_dev63853039.silver.students\n", + "\n", + "ALTER TABLE edu_dev63853039.silver.students\n", + "ADD COLUMN phone_number STRING" ] }, { "cell_type": "markdown", - "id": "706ba53d", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "d7eb1f67-364e-454f-bfb0-16e35961221d", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 6.2 — Apply tags to the catalog\n", "\n", - "Apply the following tags to the `edu_dev` catalog to support governance and discoverability:\n", + "Apply the following tags to the `edu_dev63853039` catalog to support governance and discoverability:\n", "\n", "| Tag key | Tag value |\n", "|---------|----------|\n", @@ -628,25 +1158,55 @@ "| `university` | `'lakeside'` |\n", "| `data_classification` | `'internal'` |\n", "\n", - "> 🤖 **Genie Code tip:** Ask:\n", + "> \uD83E\uDD16 **Genie Code tip:** Ask:\n", "> *\"How do I set metadata tags on a Unity Catalog catalog using the ALTER CATALOG statement in Databricks?\"*" ] }, { "cell_type": "code", - "execution_count": null, - "id": "bae85a69", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "681f838d-681a-4e71-bd6d-07f6cbf3a46e", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Cell 39" + } + }, "outputs": [], "source": [ "%sql\n", - "-- TODO: Apply the three tags to the edu_dev catalog using ALTER CATALOG ... SET TAGS" + "-- TODO: Apply the three tags to the edu_dev63853039 catalog using ALTER CATALOG ... SET TAGS\n", + "\n", + "ALTER CATALOG edu_dev63853039\n", + "SET TAGS (\n", + " 'environment' = 'education',\n", + " 'university' = 'lakeside',\n", + " 'data_classification' = 'internal'\n", + ")" ] }, { "cell_type": "markdown", - "id": "338f23f8", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "4acd1fc3-97b3-4d32-b730-26a9bd7b5ffa", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "### Task 6.3 — Verify your work\n", "\n", @@ -655,58 +1215,117 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "178ae537", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "f51485bd-a21a-4d69-b815-80ae0a822029", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql\n", "-- Show all tables and views in the silver schema\n", - "SHOW TABLES IN edu_dev.silver;" + "SHOW TABLES IN edu_dev63853039.silver;" ] }, { "cell_type": "code", - "execution_count": null, - "id": "24d433ae", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "68b77758-4e42-48d6-b6e5-4d310a89bf7b", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql\n", "-- Describe the students table — verify the new phone_number column is present\n", - "DESCRIBE TABLE edu_dev.silver.students;" + "DESCRIBE TABLE edu_dev63853039.silver.students;" ] }, { "cell_type": "code", - "execution_count": null, - "id": "44e96056", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "bbadd540-0ecf-467b-aa16-7314ad053899", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql\n", "-- Show catalog details\n", - "DESCRIBE CATALOG EXTENDED edu_dev;" + "DESCRIBE CATALOG EXTENDED edu_dev63853039;" ] }, { "cell_type": "code", - "execution_count": null, - "id": "28dab7de", - "metadata": {}, + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "implicitDf": true, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "47b34975-b8fb-4526-ba4f-8325f5c218c2", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "%sql \n", "-- Query tags via information schema (structured way)\n", "SELECT *\n", "FROM system.information_schema.catalog_tags\n", - "WHERE catalog_name = 'edu_dev';" + "WHERE catalog_name = 'edu_dev63853039';" ] }, { "cell_type": "markdown", - "id": "5b9e4d0e", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "c58cba83-b616-4c8f-b8fe-6fd2bdb0217f", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ "---\n", "\n", @@ -719,10 +1338,40 @@ } ], "metadata": { + "application/vnd.databricks.v1+notebook": { + "computePreferences": { + "hardware": { + "accelerator": null, + "gpuPoolId": null, + "memory": null + }, + "software": { + "pinSparkToX86": null + } + }, + "dashboards": [], + "environmentMetadata": { + "base_environment": "", + "environment_version": "5" + }, + "inputWidgetPreferences": null, + "language": "python", + "notebookMetadata": { + "mostRecentlyExecutedCommandWithImplicitDF": { + "commandId": -1, + "dataframes": [ + "_sqldf" + ] + }, + "pythonIndentUnit": 4 + }, + "notebookName": "03-create-and-organize-objects-in-unity-catalog", + "widgets": {} + }, "language_info": { "name": "python" } }, "nbformat": 4, - "nbformat_minor": 5 -} + "nbformat_minor": 0 +} \ No newline at end of file