From 23a0d70a0dcabcbbbc7e8d2c76d11cace534aa9b Mon Sep 17 00:00:00 2001
From: Tommy Skaue <toskaue@microsoft.com>
Date: Thu, 18 Jun 2026 18:08:34 +0200
Subject: [PATCH 1/2] Fixing broken link in python script

---
 .../DVFabricLinkUtil.ipynb                    | 575 +++++++++++++++++-
 1 file changed, 574 insertions(+), 1 deletion(-)

diff --git a/Administration/Analytics/DataverseLink/FabricLink_SQLAnalyticsEndpoint/DVFabricLinkUtil.ipynb b/Administration/Analytics/DataverseLink/FabricLink_SQLAnalyticsEndpoint/DVFabricLinkUtil.ipynb
index 6d5e06c0..684cf9a8 100644
--- a/Administration/Analytics/DataverseLink/FabricLink_SQLAnalyticsEndpoint/DVFabricLinkUtil.ipynb
+++ b/Administration/Analytics/DataverseLink/FabricLink_SQLAnalyticsEndpoint/DVFabricLinkUtil.ipynb
@@ -1 +1,574 @@
-{"cells":[{"cell_type":"markdown","source":["# Notebook does following \n","\n","- 🏗️ Step 1: Create case-insensitive Data Warehouse.\n","- 🔹 Step 2: Connect to Synapse Serverless Database and get table list and schema information.\n","- 🔁 Step 3: Get parent and child tables information from local file.\n","- 🛠️ Step 4: Connect to Fabric Link lakehouse get table metadata and generate view ddl statements.\n","- 🏃‍♂️ Step 5: Connect to Fabric case in-senstive data warehouse and create views.\n","- 🚀 Step 6: Connect to Synaspe serverless virtual datawarehouse and collect views and dependencies.\n","- 🚀 Step 7: Connect to Fabric datawarehouse and deploy create views\n"],"metadata":{"microsoft":{"language":"python","language_group":"jupyter_python"},"nteract":{"transient":{"deleting":false}}},"id":"97ea5c26-d6c2-4e71-a58c-217d10d63b4f"},{"cell_type":"code","source":["# --- Configuration Constants ---\n","WORKSPACE_ID        = \"cca1475b-c4fe-417f-844a-f3e8a061e55d\"\n","FABRIC_LH_DATABASE  = \"dataverse_jjunoenvfre\"\n","FABRIC_WH_DATABASE  = \"Sales_DW\"\n","\n","#Optional parameters to connect to your Synapse Serveress to get table and schema info\n","DRIVER              = \"{ODBC Driver 18 for SQL Server}\"\n","SYNAPSE_SERVER      = None # update to example value \"d365analyticsfabricsynapse-ondemand.sql.azuresynapse.net\"  \n","SYNAPSE_EDL_DATABASE= None #update to example value \"analytics.sandbox.operations.dynamics.com\" \n","SYNAPSE_EDL_SCHEMA  = None  #\"dbo\" \n","SYNAPSE_EDL_CONST_COLUMN_NAME = None #=\"_SysRowId\" \n","\n","\n","#Optional parameters to connect to your Synapse Serveress Datawarehouse to get views and dependencies\n","SYNAPSE_DW_DATABASE = None # example value \"Dynamics365_DW\"\n","SYNAPSE_DW_SCHEMA   = None # example value \"dbo\"\n","SYNAPSE_DW_VIEWS    = None # example value \"CustomerDim,SalesFact,SupplierDim\"\n","FABRIC_WH_SCHEMA    = None # example value \"edw\"\n","\n","#Fixed parameter - No need to change\n","GITHUB_RAW_BASE_URL = \"https://raw.githubusercontent.com/microsoft/Dynamics-365-FastTrack-Implementation-Assets/refs/heads/master/Analytics/DataverseLink/FabricLink_SQLAnalyticsEndpoint/DVFabricLinkUtil/\"\n","\n","DERIVED_TABLE_MAP_PATH = \"./builtin/resources/derived_table_map.json\"\n","LH_DDL_TEMPLATE_PATH = \"./builtin/resources/get_lh_ddl_as_view.sql\"\n","VIEW_DEPENDENCY_TEMPLATE_PATH = \"./builtin/resources/get_view_dependency.sql\"\n","REQUIRED_FILES = [\n","    (\"derived_table_map.json\", DERIVED_TABLE_MAP_PATH),\n","    (\"get_lh_ddl_as_view.sql\", LH_DDL_TEMPLATE_PATH),\n","    (\"get_view_dependency.sql\", VIEW_DEPENDENCY_TEMPLATE_PATH)]\n","\n","WAREHOUSE_VIEW_DDL_PARAMETERS = {\n","            \"source_schema\": \"dbo\",\n","            \"target_schema\": \"dbo\",\n","            \"only_fno_tables\": 1,\n","            \"tables_to_include\":\"*\",\n","            \"tables_to_exclude\": \"*\",\n","            \"filter_deleted_rows\": 1,\n","            \"join_derived_tables\": 1,\n","            \"change_collation\": 1,\n","            \"translate_enums\": 0,\n","            \"schema_map\": '[]',\n","            \"derived_table_map\": '[]'\n","        }\n","   \n","\n"],"outputs":[{"output_type":"display_data","data":{"application/vnd.jupyter.statement-meta+json":{"session_id":"801eb0aa-9a3a-4c30-b877-75c0402afbf2","normalized_state":"finished","queued_time":"2025-07-29T22:36:44.0741704Z","session_start_time":null,"execution_start_time":"2025-07-29T22:36:44.0752036Z","execution_finish_time":"2025-07-29T22:36:44.4572043Z","parent_msg_id":"7deffeec-b256-4647-bd8f-893cbaccb677"}},"metadata":{}}],"execution_count":3,"metadata":{"microsoft":{"language":"python","language_group":"jupyter_python"}},"id":"50733e90-a4fc-4424-896b-16399342f769"},{"cell_type":"code","source":["# --- Imports ---\n","import requests\n","import json\n","import logging\n","import time\n","import struct\n","import urllib.parse\n","import pandas as pd\n","from IPython.display import display, JSON\n","from sqlalchemy import create_engine, text, event\n","from requests.exceptions import HTTPError\n","import sys\n","import os\n","import re\n","\n","synapse_edl_engine = None\n","synapse_dw_engine = None\n","\n","def download_file_if_not_exists(url, local_path):\n","    \"\"\"Download a file from GitHub if it doesn't exist locally.\"\"\"\n","    if not notebookutils.fs.exists(local_path):\n","        notebookutils.fs.mkdirs(os.path.dirname(local_path))\n","        logger.info(f\"⬇️ Downloading {local_path} ...\")\n","        response = requests.get(url)\n","        response.raise_for_status()  # Fail if not 200 OK\n","        notebookutils.fs.put(local_path, response.content.decode('utf-8'))  # <-- decode bytes to string\n","    else:\n","        logger.info(f\"📄 File already exists locally: {local_path}\")\n","\n","def is_not_none_and_empty(s):\n","    return s is not None and s != ''\n","\n","\n","# --- Database Connections ---\n","def create_synapse_engine(server, database):\n","    connection_string = f\"DRIVER={DRIVER};SERVER={server};DATABASE={database};Encrypt=yes;TrustServerCertificate=no;\"\n","    odbc_conn_str = f\"mssql+pyodbc:///?odbc_connect={urllib.parse.quote_plus(connection_string)}\"\n","    engine = create_engine(odbc_conn_str)\n","    return engine\n","\n","synapse_edl_engine = create_synapse_engine(SYNAPSE_SERVER, SYNAPSE_EDL_DATABASE)\n","synapse_dw_engine = create_synapse_engine(SYNAPSE_SERVER, SYNAPSE_DW_DATABASE)\n","\n","@event.listens_for(synapse_edl_engine, \"do_connect\")\n","@event.listens_for(synapse_dw_engine, \"do_connect\")\n","def inject_access_token(dialect, conn_rec, cargs, cparams):\n","    token = notebookutils.credentials.getToken(\"https://database.windows.net/\")\n","    token_bytes = token.encode(\"utf-16-le\")\n","    token_struct = struct.pack(f\"<I{len(token_bytes)}s\", len(token_bytes), token_bytes)\n","    cparams[\"attrs_before\"] = {1256: token_struct}\n","\n","# --- Utility Functions ---\n","def load_builtin_file(path):\n","    \"\"\"Load a built-in SQL or JSON file.\"\"\"\n","    return notebookutils.fs.head(path)\n","\n","def connect_to_fabric_artifact(artifact_name, workspace_id):\n","    return notebookutils.data.connect_to_artifact(artifact_name, workspace_id)\n","\n","# --- Warehouse Management ---\n","def warehouse_exists(workspace_id, warehouse_name):\n","    try:\n","        connect_to_fabric_artifact(warehouse_name, workspace_id)\n","        return True\n","    except Exception as e:\n","        if \"ArtifactNotFoundException\" in str(type(e)):\n","            logger.info(f\"🔍 Warehouse not found:{warehouse_name}\")\n","            return False\n","        raise\n","\n","def create_case_insensitive_warehouse(workspace_id, warehouse_name, retries=5, delay=10):\n","    if warehouse_exists(workspace_id, warehouse_name):\n","        logger.info(f\"✅ Warehouse '{warehouse_name}' already exists.\")\n","        return True\n","\n","    logger.info(f\"🏗️ Creating new warehouse:{warehouse_name}\")\n","    token = notebookutils.credentials.getToken(\"https://api.fabric.microsoft.com/\")\n","    headers = {\"Authorization\": f\"Bearer {token}\", \"Content-Type\": \"application/json\"}\n","    payload = {\n","        \"type\": \"Warehouse\",\n","        \"displayName\": warehouse_name,\n","        \"description\": \"New warehouse with case-insensitive collation\",\n","        \"creationPayload\": {\"defaultCollation\": \"Latin1_General_100_CI_AS_KS_WS_SC_UTF8\"}\n","    }\n","\n","    response = requests.post(\n","        f\"https://api.fabric.microsoft.com/v1/workspaces/{workspace_id}/items\",\n","        headers=headers, data=json.dumps(payload)\n","    )\n","    logger.info(f\"📨 Warehouse creation response: {response.text}\", )\n","\n","    for attempt in range(retries):\n","        time.sleep(delay)\n","        if warehouse_exists(workspace_id, warehouse_name):\n","            logger.info(\"✅ Warehouse is ready.\")\n","            return True\n","        logger.info(f\"🔄 Retry {attempt + 1}/{retries}: Warehouse not yet available.\")\n","\n","    logger.error(\"❌ Failed to create warehouse after retries.\")\n","    return False\n","\n","# --- Schema Retrieval ---\n","def fetch_tables_and_schema_map(engine):\n","    \"\"\"Fetch table list and schema mapping.\"\"\"\n","    table_list_query = f\"\"\"\n","        SELECT STRING_AGG(CONVERT(NVARCHAR(MAX), TABLE_NAME), ',') AS tablelist\n","        FROM (\n","            SELECT DISTINCT LOWER(TABLE_NAME) as TABLE_NAME\n","            FROM INFORMATION_SCHEMA.COLUMNS\n","            WHERE TABLE_SCHEMA = '{SYNAPSE_EDL_SCHEMA}' \n","              AND TABLE_NAME IN (\n","                  SELECT DISTINCT TABLE_NAME \n","                  FROM INFORMATION_SCHEMA.COLUMNS\n","                  WHERE TABLE_SCHEMA = '{SYNAPSE_EDL_SCHEMA}' \n","                    AND COLUMN_NAME = '{SYNAPSE_EDL_CONST_COLUMN_NAME}'\n","              )\n","        ) AS tbl\n","    \"\"\"\n","    schema_map_query = f\"\"\"\n","        SELECT TABLE_NAME as tablename, COLUMN_NAME as columnname,\n","            DATA_TYPE + \n","            CASE \n","                WHEN CHARACTER_MAXIMUM_LENGTH IS NOT NULL THEN '(' + CAST(CHARACTER_MAXIMUM_LENGTH AS VARCHAR(10)) + ')'\n","                WHEN CHARACTER_MAXIMUM_LENGTH = -1 THEN '(max)'\n","                WHEN DATA_TYPE = 'decimal' THEN '(' + CAST(NUMERIC_PRECISION AS VARCHAR(10)) + ',' + CAST(NUMERIC_SCALE AS VARCHAR(10)) + ')'\n","                ELSE '' \n","            END AS datatype\n","        FROM INFORMATION_SCHEMA.COLUMNS\n","        WHERE TABLE_SCHEMA = '{SYNAPSE_EDL_SCHEMA}'\n","          AND TABLE_NAME IN (\n","              SELECT DISTINCT TABLE_NAME \n","              FROM INFORMATION_SCHEMA.COLUMNS\n","              WHERE TABLE_SCHEMA = '{SYNAPSE_EDL_SCHEMA}' AND COLUMN_NAME = '{SYNAPSE_EDL_CONST_COLUMN_NAME}'\n","          )\n","    \"\"\"\n","    with engine.connect() as conn:\n","        table_list = conn.execute(text(table_list_query)).scalar()\n","    schema_map_df = pd.read_sql(schema_map_query, engine)\n","    schema_map_json = schema_map_df.to_json(orient=\"records\")\n","\n","    return table_list, schema_map_json\n","\n","# --- DDL and View Creation ---\n","def generate_lh_view_ddl(lakehouse_name, params):\n","    \"\"\"Generate view DDL based on Lakehouse tables.\"\"\"\n","    conn = connect_to_fabric_artifact(lakehouse_name, WORKSPACE_ID)\n","\n","    variables_script = f\"\"\"\n","        DECLARE \n","            @source_database_name VARCHAR(200) = '{lakehouse_name}',\n","            @source_table_schema NVARCHAR(10) = '{params[\"source_schema\"]}',\n","            @target_table_schema NVARCHAR(10) = '{params[\"target_schema\"]}',\n","            @TablesToInclude_FnOOnly INT = {params[\"only_fno_tables\"]},\n","            @TablesToIncluce NVARCHAR(MAX) = '{params[\"tables_to_include\"]}',\n","            @TablesToExcluce NVARCHAR(MAX) = '{params[\"tables_to_exclude\"]}',\n","            @filter_deleted_rows INT = {params[\"filter_deleted_rows\"]},\n","            @join_derived_tables INT = {params[\"join_derived_tables\"]},\n","            @change_column_collation INT = {params[\"change_collation\"]},\n","            @translate_enums INT = {params[\"translate_enums\"]},\n","            @schema_map VARCHAR(MAX) = {params[\"schema_map\"]},\n","            @tableinheritance NVARCHAR(MAX) = LOWER('{params[\"derived_table_map\"]}');\n","    \"\"\"\n","    sql_template = load_builtin_file(LH_DDL_TEMPLATE_PATH)\n","    \n","    sqlquery = f\"{variables_script} {sql_template}\"\n","\n","    logger.debug(f\"Debug: generate_lh_view_ddl query {sqlquery}\")\n","    df = conn.query(f\"{sqlquery}\")\n","    ddl_statement = df.iloc[0, 0]\n","    logger.debug(f\"Debug: generate_lh_view_ddl statement {sqlquery}\")\n","\n","    return ddl_statement\n","\n","def execute_ddl_on_warehouse(warehouse_name, ddl_query):\n","    \"\"\"Execute DDL in warehouse.\"\"\"\n","    conn = connect_to_fabric_artifact(warehouse_name, WORKSPACE_ID)\n","    conn.query(ddl_query)\n","\n","# --- View Dependency Management ---\n","def fetch_view_dependencies(engine, root_entities, old_db, new_db, old_schema, new_schema):\n","    sql_template = load_builtin_file(VIEW_DEPENDENCY_TEMPLATE_PATH)\n","    sql_script = f\"\"\"\n","        SET NOCOUNT ON;\n","        DROP TABLE IF EXISTS #myEntitiestree;\n","        DECLARE @entities NVARCHAR(MAX) = '{root_entities}';\n","        DECLARE @old_schema VARCHAR(10) = '{old_schema}';\n","        DECLARE @new_schema VARCHAR(10) = '{new_schema}';\n","        {sql_template}\n","    \"\"\"\n","    with engine.connect() as conn:\n","        result = conn.execute(text(sql_script))\n","        df = pd.DataFrame(result.fetchall(), columns=result.keys())\n","    df[\"definition\"] = df[\"definition\"].str.replace(old_db, new_db, case=False, regex=False)\n","    return df.to_json(orient=\"records\")\n","\n","def deploy_views(warehouse_name, schema, views_json):\n","    execute_ddl_on_warehouse(warehouse_name, f\"IF NOT EXISTS (SELECT 1 FROM sys.schemas WHERE name = '{schema}') BEGIN EXEC('CREATE SCHEMA {schema}'); END;\")\n","    views = json.loads(views_json)\n","    for view in sorted(views, key=lambda x: x[\"depth\"], reverse=True):\n","        if view.get(\"definition\"):\n","            try:\n","                entityname = view[\"entityName\"]\n","                execute_ddl_on_warehouse(warehouse_name, view[\"definition\"])\n","                logger.info(f\"✅ Deployed view: {entityname}\")\n","            except Exception as e:\n","                logger.error(f\"❌ Failed to deploy view {entityname}: {e}\")\n","\n","for handler in logging.root.handlers[:]:  # Clear existing handlers\n","    logging.root.removeHandler(handler)\n","\n","logging.basicConfig(\n","    level=logging.INFO,\n","    format=\"[%(asctime)s] %(levelname)s - %(message)s\",\n","    datefmt=\"%Y-%m-%d %H:%M:%S\",\n","    handlers=[logging.StreamHandler(sys.stdout)]  # Explicitly use stdout\n",")\n","\n","logger = logging.getLogger()  \n","\n","# --- Main Execution ---\n","def main():\n","    \n","    logger.info(\"🏗️ Step 1: Download template files if does not exists.\")\n","    # --- Download if not exists ---\n","    for filename, local_path in REQUIRED_FILES:\n","        file_url = f\"{GITHUB_RAW_BASE_URL}/{filename}\"\n","        download_file_if_not_exists(file_url, local_path)\n","        \n","    logger.info(\"🏗️ Step 2/7: Ensure case-insensitive warehouse exists.\")\n","    \n","    if create_case_insensitive_warehouse(WORKSPACE_ID, FABRIC_WH_DATABASE):\n","\n","        logger.info(\"🔹 Step 3/7: Fetch tables and schema map from Synapse.\")\n","        tables_to_include = '*'\n","        schema_map = '[]'\n","        if is_not_none_and_empty(SYNAPSE_SERVER) and is_not_none_and_empty(SYNAPSE_EDL_DATABASE) :\n","          \n","            tables_to_include, schema_map = fetch_tables_and_schema_map(synapse_edl_engine)\n","        else:\n","            logger.info(\"🔹 Step 3/7: Skipped\")\n","\n","        logger.info(\"🔁 Step 4/7: Load derived table map.\")\n","        derived_table_map = load_builtin_file(DERIVED_TABLE_MAP_PATH)\n","\n","        logger.info(f\"🛠️ Step 5/7: Generating view DDLfrom {FABRIC_LH_DATABASE}...\")\n","        \n","        # Create a copy of the parameters and update the dynamic fields\n","        params = WAREHOUSE_VIEW_DDL_PARAMETERS.copy()\n","        params[\"schema_map\"] = f\"'{schema_map}'\"\n","        params[\"derived_table_map\"] = derived_table_map\n","        params[\"tables_to_include\"] = tables_to_include\n","\n","\n","        views_ddl = generate_lh_view_ddl(FABRIC_LH_DATABASE, params)\n","      \n","      \n","        logger.info(f\"🛠️ Step 5/7: Executing views DDL on Fabric DW {FABRIC_WH_DATABASE}.\")\n","\n","        view_ddls = views_ddl.split(';')\n","        view_count = len(view_ddls)\n","        logger.info(f\"{view_count} views to deploy.\")\n","        # Loop through each item\n","        counter= 1\n","        for view_ddl in view_ddls:\n","            match = re.search(r\"CREATE\\s+OR\\s+ALTER\\s+VIEW\\s+([\\w\\.]+)\", view_ddl, re.IGNORECASE)\n","            if match:\n","                view_name = match.group(1)\n","                logger.info(f\"{view_count}/{counter} CREATE OR ALTER VIEW {view_name}\")\n","                execute_ddl_on_warehouse(FABRIC_WH_DATABASE, view_ddl)\n","                counter += 1\n","\n","        if is_not_none_and_empty(SYNAPSE_SERVER) and is_not_none_and_empty(SYNAPSE_DW_DATABASE) :\n"," \n","            logger.info(\"🚀 Step 6/7: Fetch view dependencies.\")\n","            views_json = fetch_view_dependencies(\n","                synapse_dw_engine, SYNAPSE_DW_VIEWS,\n","                SYNAPSE_EDL_DATABASE, FABRIC_WH_DATABASE,\n","                SYNAPSE_DW_SCHEMA, FABRIC_WH_SCHEMA\n","            )\n","\n","            logger.info(\"🚀 Step 7/7: Deploy views to warehouse.\")\n","            deploy_views(FABRIC_WH_DATABASE, FABRIC_WH_SCHEMA, views_json)\n","        else:\n","            logger.info(\"🔹 Step 6 and 7: Skipped\")\n","\n","        logger.info(\"🎉 Deployment complete.\")\n","    else:\n","        logger.error(\"❌ Warehouse setup failed.\")\n","    \n","\n","if __name__ == \"__main__\":\n","    main()\n"],"outputs":[{"output_type":"display_data","data":{"application/vnd.jupyter.statement-meta+json":{"session_id":"801eb0aa-9a3a-4c30-b877-75c0402afbf2","normalized_state":"finished","queued_time":"2025-07-29T22:36:44.133805Z","session_start_time":null,"execution_start_time":"2025-07-29T22:36:44.4586243Z","execution_finish_time":"2025-07-29T22:37:29.0235544Z","parent_msg_id":"3785be1d-73c9-43c0-901e-5ad3b7920f19"}},"metadata":{}},{"output_type":"stream","name":"stdout","text":["[2025-07-29 22:36:44] INFO - 🏗️ Step 1: Download template files if does not exists.\n[2025-07-29 22:36:44] INFO - 📄 File already exists locally: ./builtin/resources/derived_table_map.json\n[2025-07-29 22:36:44] INFO - 📄 File already exists locally: ./builtin/resources/get_lh_ddl_as_view.sql\n[2025-07-29 22:36:44] INFO - 📄 File already exists locally: ./builtin/resources/get_view_dependency.sql\n[2025-07-29 22:36:44] INFO - 🏗️ Step 2/7: Ensure case-insensitive warehouse exists.\n[2025-07-29 22:36:45] INFO - ✅ Warehouse 'Sales_DW' already exists.\n[2025-07-29 22:36:45] INFO - 🔹 Step 3/7: Fetch tables and schema map from Synapse.\n[2025-07-29 22:36:45] INFO - 🔹 Step 3/7: Skipped\n[2025-07-29 22:36:45] INFO - 🔁 Step 4/7: Load derived table map.\n[2025-07-29 22:36:45] INFO - 🛠️ Step 5/7: Generating view DDLfrom {FABRIC_LH_DATABASE}...\n[2025-07-29 22:37:06] INFO - 🛠️ Step 5/7: Executing views DDL on Fabric DW Sales_DW.\n[2025-07-29 22:37:06] INFO - 65 views to deploy.\n[2025-07-29 22:37:06] INFO - 65/1 CREATE OR ALTER VIEW dbo.commissionsalesgroup\n[2025-07-29 22:37:06] INFO - 65/2 CREATE OR ALTER VIEW dbo.companyinfo\n[2025-07-29 22:37:06] INFO - 65/3 CREATE OR ALTER VIEW dbo.custgroup\n[2025-07-29 22:37:06] INFO - 65/4 CREATE OR ALTER VIEW dbo.custinvoicejour\n[2025-07-29 22:37:07] INFO - 65/5 CREATE OR ALTER VIEW dbo.custinvoicetrans\n[2025-07-29 22:37:08] INFO - 65/6 CREATE OR ALTER VIEW dbo.custpackingslipjour\n[2025-07-29 22:37:08] INFO - 65/7 CREATE OR ALTER VIEW dbo.custpackingsliptrans\n[2025-07-29 22:37:08] INFO - 65/8 CREATE OR ALTER VIEW dbo.custtable\n[2025-07-29 22:37:10] INFO - 65/9 CREATE OR ALTER VIEW dbo.custtrans\n[2025-07-29 22:37:10] INFO - 65/10 CREATE OR ALTER VIEW dbo.custtransopen\n[2025-07-29 22:37:10] INFO - 65/11 CREATE OR ALTER VIEW dbo.dimensionattribute\n[2025-07-29 22:37:11] INFO - 65/12 CREATE OR ALTER VIEW dbo.dimensionattributevalue\n[2025-07-29 22:37:11] INFO - 65/13 CREATE OR ALTER VIEW dbo.dimensionattributevaluesetitem\n[2025-07-29 22:37:11] INFO - 65/14 CREATE OR ALTER VIEW dbo.dirdunsnumber\n[2025-07-29 22:37:11] INFO - 65/15 CREATE OR ALTER VIEW dbo.dirnameaffix\n[2025-07-29 22:37:11] INFO - 65/16 CREATE OR ALTER VIEW dbo.dirpartylocation\n[2025-07-29 22:37:11] INFO - 65/17 CREATE OR ALTER VIEW dbo.dirpartytable\n[2025-07-29 22:37:11] INFO - 65/18 CREATE OR ALTER VIEW dbo.dirpersonname\n[2025-07-29 22:37:11] INFO - 65/19 CREATE OR ALTER VIEW dbo.ecoresproduct\n[2025-07-29 22:37:11] INFO - 65/20 CREATE OR ALTER VIEW dbo.ecoresproducttranslation\n[2025-07-29 22:37:11] INFO - 65/21 CREATE OR ALTER VIEW dbo.generaljournalaccountentry\n[2025-07-29 22:37:11] INFO - 65/22 CREATE OR ALTER VIEW dbo.generaljournalentry\n[2025-07-29 22:37:11] INFO - 65/23 CREATE OR ALTER VIEW dbo.hcmworker\n[2025-07-29 22:37:11] INFO - 65/24 CREATE OR ALTER VIEW dbo.inventdim\n[2025-07-29 22:37:11] INFO - 65/25 CREATE OR ALTER VIEW dbo.inventitemgroup\n[2025-07-29 22:37:11] INFO - 65/26 CREATE OR ALTER VIEW dbo.inventitemgroupitem\n[2025-07-29 22:37:11] INFO - 65/27 CREATE OR ALTER VIEW dbo.inventiteminventsetup\n[2025-07-29 22:37:11] INFO - 65/28 CREATE OR ALTER VIEW dbo.inventitemlocation\n[2025-07-29 22:37:12] INFO - 65/29 CREATE OR ALTER VIEW dbo.inventitemprice\n[2025-07-29 22:37:12] INFO - 65/30 CREATE OR ALTER VIEW dbo.inventitempurchsetup\n[2025-07-29 22:37:12] INFO - 65/31 CREATE OR ALTER VIEW dbo.inventitemsalessetup\n[2025-07-29 22:37:12] INFO - 65/32 CREATE OR ALTER VIEW dbo.inventitemsampling\n[2025-07-29 22:37:12] INFO - 65/33 CREATE OR ALTER VIEW dbo.inventjournaltable\n[2025-07-29 22:37:12] INFO - 65/34 CREATE OR ALTER VIEW dbo.inventjournaltrans\n[2025-07-29 22:37:12] INFO - 65/35 CREATE OR ALTER VIEW dbo.inventsettlement\n[2025-07-29 22:37:12] INFO - 65/36 CREATE OR ALTER VIEW dbo.inventsum\n[2025-07-29 22:37:12] INFO - 65/37 CREATE OR ALTER VIEW dbo.inventtable\n[2025-07-29 22:37:13] INFO - 65/38 CREATE OR ALTER VIEW dbo.inventtrans\n[2025-07-29 22:37:14] INFO - 65/39 CREATE OR ALTER VIEW dbo.inventtransorigin\n[2025-07-29 22:37:14] INFO - 65/40 CREATE OR ALTER VIEW dbo.inventtransposting\n[2025-07-29 22:37:14] INFO - 65/41 CREATE OR ALTER VIEW dbo.ledgerjournaltable\n[2025-07-29 22:37:14] INFO - 65/42 CREATE OR ALTER VIEW dbo.ledgerjournaltrans\n[2025-07-29 22:37:17] INFO - 65/43 CREATE OR ALTER VIEW dbo.logisticsaddresscountryregion\n[2025-07-29 22:37:17] INFO - 65/44 CREATE OR ALTER VIEW dbo.logisticsaddresscountryregiontranslation\n[2025-07-29 22:37:17] INFO - 65/45 CREATE OR ALTER VIEW dbo.logisticselectronicaddress\n[2025-07-29 22:37:17] INFO - 65/46 CREATE OR ALTER VIEW dbo.logisticslocation\n[2025-07-29 22:37:17] INFO - 65/47 CREATE OR ALTER VIEW dbo.logisticslocationext\n[2025-07-29 22:37:17] INFO - 65/48 CREATE OR ALTER VIEW dbo.logisticspostaladdress\n[2025-07-29 22:37:17] INFO - 65/49 CREATE OR ALTER VIEW dbo.omteammembershipcriterion\n[2025-07-29 22:37:17] INFO - 65/50 CREATE OR ALTER VIEW dbo.pdscatchweightitem\n[2025-07-29 22:37:17] INFO - 65/51 CREATE OR ALTER VIEW dbo.purchline\n[2025-07-29 22:37:19] INFO - 65/52 CREATE OR ALTER VIEW dbo.purchtable\n[2025-07-29 22:37:20] INFO - 65/53 CREATE OR ALTER VIEW dbo.reqtrans\n[2025-07-29 22:37:20] INFO - 65/54 CREATE OR ALTER VIEW dbo.salesline\n[2025-07-29 22:37:22] INFO - 65/55 CREATE OR ALTER VIEW dbo.salesline_w\n[2025-07-29 22:37:22] INFO - 65/56 CREATE OR ALTER VIEW dbo.salestable\n[2025-07-29 22:37:24] INFO - 65/57 CREATE OR ALTER VIEW dbo.systemparameters\n[2025-07-29 22:37:24] INFO - 65/58 CREATE OR ALTER VIEW dbo.taxtrans\n[2025-07-29 22:37:24] INFO - 65/59 CREATE OR ALTER VIEW dbo.vendgroup\n[2025-07-29 22:37:24] INFO - 65/60 CREATE OR ALTER VIEW dbo.vendtable\n[2025-07-29 22:37:26] INFO - 65/61 CREATE OR ALTER VIEW dbo.vendtrans\n[2025-07-29 22:37:26] INFO - 65/62 CREATE OR ALTER VIEW dbo.whsinventenabled\n[2025-07-29 22:37:26] INFO - 65/63 CREATE OR ALTER VIEW dbo.whsinventreserve\n[2025-07-29 22:37:26] INFO - 65/64 CREATE OR ALTER VIEW dbo.whsreservationhierarchyelement\n[2025-07-29 22:37:26] INFO - 65/65 CREATE OR ALTER VIEW dbo.dirpartytable\n[2025-07-29 22:37:27] INFO - 🔹 Step 6 and 7: Skipped\n[2025-07-29 22:37:27] INFO - 🎉 Deployment complete.\n"]}],"execution_count":4,"metadata":{"jupyter":{"outputs_hidden":false,"source_hidden":false},"microsoft":{"language":"python","language_group":"jupyter_python"},"nteract":{"transient":{"deleting":false}}},"id":"8c2d3a9a-f5c8-4078-a36f-b540e44e6bfd"}],"metadata":{"kernel_info":{"jupyter_kernel_name":"python3.11","name":"jupyter"},"kernelspec":{"name":"jupyter","display_name":"Jupyter"},"language_info":{"name":"python"},"microsoft":{"language":"python","language_group":"jupyter_python","ms_spell_check":{"ms_spell_check_language":"en"}},"nteract":{"version":"nteract-front-end@1.0.0"},"widgets":{},"synapse_widget":{"state":{},"version":"0.1"},"spark_compute":{"compute_id":"/trident/default","session_options":{"conf":{"spark.synapse.nbs.session.timeout":"1200000"}}},"dependencies":{"lakehouse":{"known_lakehouses":[{"id":"70459bf8-2d31-42b6-947f-9eea8f2e6c8c"}],"default_lakehouse":"70459bf8-2d31-42b6-947f-9eea8f2e6c8c","default_lakehouse_name":"dataverse_jjunoenvfre","default_lakehouse_workspace_id":"cca1475b-c4fe-417f-844a-f3e8a061e55d"},"warehouse":{"known_warehouses":[]}}},"nbformat":4,"nbformat_minor":5}
\ No newline at end of file
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "97ea5c26-d6c2-4e71-a58c-217d10d63b4f",
+   "metadata": {
+    "microsoft": {
+     "language": "python",
+     "language_group": "jupyter_python"
+    },
+    "nteract": {
+     "transient": {
+      "deleting": false
+     }
+    }
+   },
+   "source": [
+    "# Notebook does following \n",
+    "\n",
+    "- 🏗️ Step 1: Create case-insensitive Data Warehouse.\n",
+    "- 🔹 Step 2: Connect to Synapse Serverless Database and get table list and schema information.\n",
+    "- 🔁 Step 3: Get parent and child tables information from local file.\n",
+    "- 🛠️ Step 4: Connect to Fabric Link lakehouse get table metadata and generate view ddl statements.\n",
+    "- 🏃‍♂️ Step 5: Connect to Fabric case in-senstive data warehouse and create views.\n",
+    "- 🚀 Step 6: Connect to Synaspe serverless virtual datawarehouse and collect views and dependencies.\n",
+    "- 🚀 Step 7: Connect to Fabric datawarehouse and deploy create views\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "50733e90-a4fc-4424-896b-16399342f769",
+   "metadata": {
+    "microsoft": {
+     "language": "python",
+     "language_group": "jupyter_python"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.statement-meta+json": {
+       "execution_finish_time": "2025-07-29T22:36:44.4572043Z",
+       "execution_start_time": "2025-07-29T22:36:44.0752036Z",
+       "normalized_state": "finished",
+       "parent_msg_id": "7deffeec-b256-4647-bd8f-893cbaccb677",
+       "queued_time": "2025-07-29T22:36:44.0741704Z",
+       "session_id": "801eb0aa-9a3a-4c30-b877-75c0402afbf2",
+       "session_start_time": null
+      }
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "# --- Configuration Constants ---\n",
+    "WORKSPACE_ID        = \"cca1475b-c4fe-417f-844a-f3e8a061e55d\"\n",
+    "FABRIC_LH_DATABASE  = \"dataverse_jjunoenvfre\"\n",
+    "FABRIC_WH_DATABASE  = \"Sales_DW\"\n",
+    "\n",
+    "#Optional parameters to connect to your Synapse Serveress to get table and schema info\n",
+    "DRIVER              = \"{ODBC Driver 18 for SQL Server}\"\n",
+    "SYNAPSE_SERVER      = None # update to example value \"d365analyticsfabricsynapse-ondemand.sql.azuresynapse.net\"  \n",
+    "SYNAPSE_EDL_DATABASE= None #update to example value \"analytics.sandbox.operations.dynamics.com\" \n",
+    "SYNAPSE_EDL_SCHEMA  = None  #\"dbo\" \n",
+    "SYNAPSE_EDL_CONST_COLUMN_NAME = None #=\"_SysRowId\" \n",
+    "\n",
+    "\n",
+    "#Optional parameters to connect to your Synapse Serveress Datawarehouse to get views and dependencies\n",
+    "SYNAPSE_DW_DATABASE = None # example value \"Dynamics365_DW\"\n",
+    "SYNAPSE_DW_SCHEMA   = None # example value \"dbo\"\n",
+    "SYNAPSE_DW_VIEWS    = None # example value \"CustomerDim,SalesFact,SupplierDim\"\n",
+    "FABRIC_WH_SCHEMA    = None # example value \"edw\"\n",
+    "\n",
+    "#Fixed parameter - No need to change\n",
+    "GITHUB_RAW_BASE_URL = \"https://raw.githubusercontent.com/microsoft/Dynamics-365-FastTrack-Implementation-Assets/refs/heads/master/Administration/Analytics/DataverseLink/FabricLink_SQLAnalyticsEndpoint/DVFabricLinkUtil/\"\n",
+    "\n",
+    "DERIVED_TABLE_MAP_PATH = \"./builtin/resources/derived_table_map.json\"\n",
+    "LH_DDL_TEMPLATE_PATH = \"./builtin/resources/get_lh_ddl_as_view.sql\"\n",
+    "VIEW_DEPENDENCY_TEMPLATE_PATH = \"./builtin/resources/get_view_dependency.sql\"\n",
+    "REQUIRED_FILES = [\n",
+    "    (\"derived_table_map.json\", DERIVED_TABLE_MAP_PATH),\n",
+    "    (\"get_lh_ddl_as_view.sql\", LH_DDL_TEMPLATE_PATH),\n",
+    "    (\"get_view_dependency.sql\", VIEW_DEPENDENCY_TEMPLATE_PATH)]\n",
+    "\n",
+    "WAREHOUSE_VIEW_DDL_PARAMETERS = {\n",
+    "            \"source_schema\": \"dbo\",\n",
+    "            \"target_schema\": \"dbo\",\n",
+    "            \"only_fno_tables\": 1,\n",
+    "            \"tables_to_include\":\"*\",\n",
+    "            \"tables_to_exclude\": \"*\",\n",
+    "            \"filter_deleted_rows\": 1,\n",
+    "            \"join_derived_tables\": 1,\n",
+    "            \"change_collation\": 1,\n",
+    "            \"translate_enums\": 0,\n",
+    "            \"schema_map\": '[]',\n",
+    "            \"derived_table_map\": '[]'\n",
+    "        }\n",
+    "   \n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "8c2d3a9a-f5c8-4078-a36f-b540e44e6bfd",
+   "metadata": {
+    "jupyter": {
+     "outputs_hidden": false,
+     "source_hidden": false
+    },
+    "microsoft": {
+     "language": "python",
+     "language_group": "jupyter_python"
+    },
+    "nteract": {
+     "transient": {
+      "deleting": false
+     }
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.statement-meta+json": {
+       "execution_finish_time": "2025-07-29T22:37:29.0235544Z",
+       "execution_start_time": "2025-07-29T22:36:44.4586243Z",
+       "normalized_state": "finished",
+       "parent_msg_id": "3785be1d-73c9-43c0-901e-5ad3b7920f19",
+       "queued_time": "2025-07-29T22:36:44.133805Z",
+       "session_id": "801eb0aa-9a3a-4c30-b877-75c0402afbf2",
+       "session_start_time": null
+      }
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[2025-07-29 22:36:44] INFO - 🏗️ Step 1: Download template files if does not exists.\n",
+      "[2025-07-29 22:36:44] INFO - 📄 File already exists locally: ./builtin/resources/derived_table_map.json\n",
+      "[2025-07-29 22:36:44] INFO - 📄 File already exists locally: ./builtin/resources/get_lh_ddl_as_view.sql\n",
+      "[2025-07-29 22:36:44] INFO - 📄 File already exists locally: ./builtin/resources/get_view_dependency.sql\n",
+      "[2025-07-29 22:36:44] INFO - 🏗️ Step 2/7: Ensure case-insensitive warehouse exists.\n",
+      "[2025-07-29 22:36:45] INFO - ✅ Warehouse 'Sales_DW' already exists.\n",
+      "[2025-07-29 22:36:45] INFO - 🔹 Step 3/7: Fetch tables and schema map from Synapse.\n",
+      "[2025-07-29 22:36:45] INFO - 🔹 Step 3/7: Skipped\n",
+      "[2025-07-29 22:36:45] INFO - 🔁 Step 4/7: Load derived table map.\n",
+      "[2025-07-29 22:36:45] INFO - 🛠️ Step 5/7: Generating view DDLfrom {FABRIC_LH_DATABASE}...\n",
+      "[2025-07-29 22:37:06] INFO - 🛠️ Step 5/7: Executing views DDL on Fabric DW Sales_DW.\n",
+      "[2025-07-29 22:37:06] INFO - 65 views to deploy.\n",
+      "[2025-07-29 22:37:06] INFO - 65/1 CREATE OR ALTER VIEW dbo.commissionsalesgroup\n",
+      "[2025-07-29 22:37:06] INFO - 65/2 CREATE OR ALTER VIEW dbo.companyinfo\n",
+      "[2025-07-29 22:37:06] INFO - 65/3 CREATE OR ALTER VIEW dbo.custgroup\n",
+      "[2025-07-29 22:37:06] INFO - 65/4 CREATE OR ALTER VIEW dbo.custinvoicejour\n",
+      "[2025-07-29 22:37:07] INFO - 65/5 CREATE OR ALTER VIEW dbo.custinvoicetrans\n",
+      "[2025-07-29 22:37:08] INFO - 65/6 CREATE OR ALTER VIEW dbo.custpackingslipjour\n",
+      "[2025-07-29 22:37:08] INFO - 65/7 CREATE OR ALTER VIEW dbo.custpackingsliptrans\n",
+      "[2025-07-29 22:37:08] INFO - 65/8 CREATE OR ALTER VIEW dbo.custtable\n",
+      "[2025-07-29 22:37:10] INFO - 65/9 CREATE OR ALTER VIEW dbo.custtrans\n",
+      "[2025-07-29 22:37:10] INFO - 65/10 CREATE OR ALTER VIEW dbo.custtransopen\n",
+      "[2025-07-29 22:37:10] INFO - 65/11 CREATE OR ALTER VIEW dbo.dimensionattribute\n",
+      "[2025-07-29 22:37:11] INFO - 65/12 CREATE OR ALTER VIEW dbo.dimensionattributevalue\n",
+      "[2025-07-29 22:37:11] INFO - 65/13 CREATE OR ALTER VIEW dbo.dimensionattributevaluesetitem\n",
+      "[2025-07-29 22:37:11] INFO - 65/14 CREATE OR ALTER VIEW dbo.dirdunsnumber\n",
+      "[2025-07-29 22:37:11] INFO - 65/15 CREATE OR ALTER VIEW dbo.dirnameaffix\n",
+      "[2025-07-29 22:37:11] INFO - 65/16 CREATE OR ALTER VIEW dbo.dirpartylocation\n",
+      "[2025-07-29 22:37:11] INFO - 65/17 CREATE OR ALTER VIEW dbo.dirpartytable\n",
+      "[2025-07-29 22:37:11] INFO - 65/18 CREATE OR ALTER VIEW dbo.dirpersonname\n",
+      "[2025-07-29 22:37:11] INFO - 65/19 CREATE OR ALTER VIEW dbo.ecoresproduct\n",
+      "[2025-07-29 22:37:11] INFO - 65/20 CREATE OR ALTER VIEW dbo.ecoresproducttranslation\n",
+      "[2025-07-29 22:37:11] INFO - 65/21 CREATE OR ALTER VIEW dbo.generaljournalaccountentry\n",
+      "[2025-07-29 22:37:11] INFO - 65/22 CREATE OR ALTER VIEW dbo.generaljournalentry\n",
+      "[2025-07-29 22:37:11] INFO - 65/23 CREATE OR ALTER VIEW dbo.hcmworker\n",
+      "[2025-07-29 22:37:11] INFO - 65/24 CREATE OR ALTER VIEW dbo.inventdim\n",
+      "[2025-07-29 22:37:11] INFO - 65/25 CREATE OR ALTER VIEW dbo.inventitemgroup\n",
+      "[2025-07-29 22:37:11] INFO - 65/26 CREATE OR ALTER VIEW dbo.inventitemgroupitem\n",
+      "[2025-07-29 22:37:11] INFO - 65/27 CREATE OR ALTER VIEW dbo.inventiteminventsetup\n",
+      "[2025-07-29 22:37:11] INFO - 65/28 CREATE OR ALTER VIEW dbo.inventitemlocation\n",
+      "[2025-07-29 22:37:12] INFO - 65/29 CREATE OR ALTER VIEW dbo.inventitemprice\n",
+      "[2025-07-29 22:37:12] INFO - 65/30 CREATE OR ALTER VIEW dbo.inventitempurchsetup\n",
+      "[2025-07-29 22:37:12] INFO - 65/31 CREATE OR ALTER VIEW dbo.inventitemsalessetup\n",
+      "[2025-07-29 22:37:12] INFO - 65/32 CREATE OR ALTER VIEW dbo.inventitemsampling\n",
+      "[2025-07-29 22:37:12] INFO - 65/33 CREATE OR ALTER VIEW dbo.inventjournaltable\n",
+      "[2025-07-29 22:37:12] INFO - 65/34 CREATE OR ALTER VIEW dbo.inventjournaltrans\n",
+      "[2025-07-29 22:37:12] INFO - 65/35 CREATE OR ALTER VIEW dbo.inventsettlement\n",
+      "[2025-07-29 22:37:12] INFO - 65/36 CREATE OR ALTER VIEW dbo.inventsum\n",
+      "[2025-07-29 22:37:12] INFO - 65/37 CREATE OR ALTER VIEW dbo.inventtable\n",
+      "[2025-07-29 22:37:13] INFO - 65/38 CREATE OR ALTER VIEW dbo.inventtrans\n",
+      "[2025-07-29 22:37:14] INFO - 65/39 CREATE OR ALTER VIEW dbo.inventtransorigin\n",
+      "[2025-07-29 22:37:14] INFO - 65/40 CREATE OR ALTER VIEW dbo.inventtransposting\n",
+      "[2025-07-29 22:37:14] INFO - 65/41 CREATE OR ALTER VIEW dbo.ledgerjournaltable\n",
+      "[2025-07-29 22:37:14] INFO - 65/42 CREATE OR ALTER VIEW dbo.ledgerjournaltrans\n",
+      "[2025-07-29 22:37:17] INFO - 65/43 CREATE OR ALTER VIEW dbo.logisticsaddresscountryregion\n",
+      "[2025-07-29 22:37:17] INFO - 65/44 CREATE OR ALTER VIEW dbo.logisticsaddresscountryregiontranslation\n",
+      "[2025-07-29 22:37:17] INFO - 65/45 CREATE OR ALTER VIEW dbo.logisticselectronicaddress\n",
+      "[2025-07-29 22:37:17] INFO - 65/46 CREATE OR ALTER VIEW dbo.logisticslocation\n",
+      "[2025-07-29 22:37:17] INFO - 65/47 CREATE OR ALTER VIEW dbo.logisticslocationext\n",
+      "[2025-07-29 22:37:17] INFO - 65/48 CREATE OR ALTER VIEW dbo.logisticspostaladdress\n",
+      "[2025-07-29 22:37:17] INFO - 65/49 CREATE OR ALTER VIEW dbo.omteammembershipcriterion\n",
+      "[2025-07-29 22:37:17] INFO - 65/50 CREATE OR ALTER VIEW dbo.pdscatchweightitem\n",
+      "[2025-07-29 22:37:17] INFO - 65/51 CREATE OR ALTER VIEW dbo.purchline\n",
+      "[2025-07-29 22:37:19] INFO - 65/52 CREATE OR ALTER VIEW dbo.purchtable\n",
+      "[2025-07-29 22:37:20] INFO - 65/53 CREATE OR ALTER VIEW dbo.reqtrans\n",
+      "[2025-07-29 22:37:20] INFO - 65/54 CREATE OR ALTER VIEW dbo.salesline\n",
+      "[2025-07-29 22:37:22] INFO - 65/55 CREATE OR ALTER VIEW dbo.salesline_w\n",
+      "[2025-07-29 22:37:22] INFO - 65/56 CREATE OR ALTER VIEW dbo.salestable\n",
+      "[2025-07-29 22:37:24] INFO - 65/57 CREATE OR ALTER VIEW dbo.systemparameters\n",
+      "[2025-07-29 22:37:24] INFO - 65/58 CREATE OR ALTER VIEW dbo.taxtrans\n",
+      "[2025-07-29 22:37:24] INFO - 65/59 CREATE OR ALTER VIEW dbo.vendgroup\n",
+      "[2025-07-29 22:37:24] INFO - 65/60 CREATE OR ALTER VIEW dbo.vendtable\n",
+      "[2025-07-29 22:37:26] INFO - 65/61 CREATE OR ALTER VIEW dbo.vendtrans\n",
+      "[2025-07-29 22:37:26] INFO - 65/62 CREATE OR ALTER VIEW dbo.whsinventenabled\n",
+      "[2025-07-29 22:37:26] INFO - 65/63 CREATE OR ALTER VIEW dbo.whsinventreserve\n",
+      "[2025-07-29 22:37:26] INFO - 65/64 CREATE OR ALTER VIEW dbo.whsreservationhierarchyelement\n",
+      "[2025-07-29 22:37:26] INFO - 65/65 CREATE OR ALTER VIEW dbo.dirpartytable\n",
+      "[2025-07-29 22:37:27] INFO - 🔹 Step 6 and 7: Skipped\n",
+      "[2025-07-29 22:37:27] INFO - 🎉 Deployment complete.\n"
+     ]
+    }
+   ],
+   "source": [
+    "# --- Imports ---\n",
+    "import requests\n",
+    "import json\n",
+    "import logging\n",
+    "import time\n",
+    "import struct\n",
+    "import urllib.parse\n",
+    "import pandas as pd\n",
+    "from IPython.display import display, JSON\n",
+    "from sqlalchemy import create_engine, text, event\n",
+    "from requests.exceptions import HTTPError\n",
+    "import sys\n",
+    "import os\n",
+    "import re\n",
+    "\n",
+    "synapse_edl_engine = None\n",
+    "synapse_dw_engine = None\n",
+    "\n",
+    "def download_file_if_not_exists(url, local_path):\n",
+    "    \"\"\"Download a file from GitHub if it doesn't exist locally.\"\"\"\n",
+    "    if not notebookutils.fs.exists(local_path):\n",
+    "        notebookutils.fs.mkdirs(os.path.dirname(local_path))\n",
+    "        logger.info(f\"⬇️ Downloading {local_path} ...\")\n",
+    "        response = requests.get(url)\n",
+    "        response.raise_for_status()  # Fail if not 200 OK\n",
+    "        notebookutils.fs.put(local_path, response.content.decode('utf-8'))  # <-- decode bytes to string\n",
+    "    else:\n",
+    "        logger.info(f\"📄 File already exists locally: {local_path}\")\n",
+    "\n",
+    "def is_not_none_and_empty(s):\n",
+    "    return s is not None and s != ''\n",
+    "\n",
+    "\n",
+    "# --- Database Connections ---\n",
+    "def create_synapse_engine(server, database):\n",
+    "    connection_string = f\"DRIVER={DRIVER};SERVER={server};DATABASE={database};Encrypt=yes;TrustServerCertificate=no;\"\n",
+    "    odbc_conn_str = f\"mssql+pyodbc:///?odbc_connect={urllib.parse.quote_plus(connection_string)}\"\n",
+    "    engine = create_engine(odbc_conn_str)\n",
+    "    return engine\n",
+    "\n",
+    "synapse_edl_engine = create_synapse_engine(SYNAPSE_SERVER, SYNAPSE_EDL_DATABASE)\n",
+    "synapse_dw_engine = create_synapse_engine(SYNAPSE_SERVER, SYNAPSE_DW_DATABASE)\n",
+    "\n",
+    "@event.listens_for(synapse_edl_engine, \"do_connect\")\n",
+    "@event.listens_for(synapse_dw_engine, \"do_connect\")\n",
+    "def inject_access_token(dialect, conn_rec, cargs, cparams):\n",
+    "    token = notebookutils.credentials.getToken(\"https://database.windows.net/\")\n",
+    "    token_bytes = token.encode(\"utf-16-le\")\n",
+    "    token_struct = struct.pack(f\"<I{len(token_bytes)}s\", len(token_bytes), token_bytes)\n",
+    "    cparams[\"attrs_before\"] = {1256: token_struct}\n",
+    "\n",
+    "# --- Utility Functions ---\n",
+    "def load_builtin_file(path):\n",
+    "    \"\"\"Load a built-in SQL or JSON file.\"\"\"\n",
+    "    return notebookutils.fs.head(path)\n",
+    "\n",
+    "def connect_to_fabric_artifact(artifact_name, workspace_id):\n",
+    "    return notebookutils.data.connect_to_artifact(artifact_name, workspace_id)\n",
+    "\n",
+    "# --- Warehouse Management ---\n",
+    "def warehouse_exists(workspace_id, warehouse_name):\n",
+    "    try:\n",
+    "        connect_to_fabric_artifact(warehouse_name, workspace_id)\n",
+    "        return True\n",
+    "    except Exception as e:\n",
+    "        if \"ArtifactNotFoundException\" in str(type(e)):\n",
+    "            logger.info(f\"🔍 Warehouse not found:{warehouse_name}\")\n",
+    "            return False\n",
+    "        raise\n",
+    "\n",
+    "def create_case_insensitive_warehouse(workspace_id, warehouse_name, retries=5, delay=10):\n",
+    "    if warehouse_exists(workspace_id, warehouse_name):\n",
+    "        logger.info(f\"✅ Warehouse '{warehouse_name}' already exists.\")\n",
+    "        return True\n",
+    "\n",
+    "    logger.info(f\"🏗️ Creating new warehouse:{warehouse_name}\")\n",
+    "    token = notebookutils.credentials.getToken(\"https://api.fabric.microsoft.com/\")\n",
+    "    headers = {\"Authorization\": f\"Bearer {token}\", \"Content-Type\": \"application/json\"}\n",
+    "    payload = {\n",
+    "        \"type\": \"Warehouse\",\n",
+    "        \"displayName\": warehouse_name,\n",
+    "        \"description\": \"New warehouse with case-insensitive collation\",\n",
+    "        \"creationPayload\": {\"defaultCollation\": \"Latin1_General_100_CI_AS_KS_WS_SC_UTF8\"}\n",
+    "    }\n",
+    "\n",
+    "    response = requests.post(\n",
+    "        f\"https://api.fabric.microsoft.com/v1/workspaces/{workspace_id}/items\",\n",
+    "        headers=headers, data=json.dumps(payload)\n",
+    "    )\n",
+    "    logger.info(f\"📨 Warehouse creation response: {response.text}\", )\n",
+    "\n",
+    "    for attempt in range(retries):\n",
+    "        time.sleep(delay)\n",
+    "        if warehouse_exists(workspace_id, warehouse_name):\n",
+    "            logger.info(\"✅ Warehouse is ready.\")\n",
+    "            return True\n",
+    "        logger.info(f\"🔄 Retry {attempt + 1}/{retries}: Warehouse not yet available.\")\n",
+    "\n",
+    "    logger.error(\"❌ Failed to create warehouse after retries.\")\n",
+    "    return False\n",
+    "\n",
+    "# --- Schema Retrieval ---\n",
+    "def fetch_tables_and_schema_map(engine):\n",
+    "    \"\"\"Fetch table list and schema mapping.\"\"\"\n",
+    "    table_list_query = f\"\"\"\n",
+    "        SELECT STRING_AGG(CONVERT(NVARCHAR(MAX), TABLE_NAME), ',') AS tablelist\n",
+    "        FROM (\n",
+    "            SELECT DISTINCT LOWER(TABLE_NAME) as TABLE_NAME\n",
+    "            FROM INFORMATION_SCHEMA.COLUMNS\n",
+    "            WHERE TABLE_SCHEMA = '{SYNAPSE_EDL_SCHEMA}' \n",
+    "              AND TABLE_NAME IN (\n",
+    "                  SELECT DISTINCT TABLE_NAME \n",
+    "                  FROM INFORMATION_SCHEMA.COLUMNS\n",
+    "                  WHERE TABLE_SCHEMA = '{SYNAPSE_EDL_SCHEMA}' \n",
+    "                    AND COLUMN_NAME = '{SYNAPSE_EDL_CONST_COLUMN_NAME}'\n",
+    "              )\n",
+    "        ) AS tbl\n",
+    "    \"\"\"\n",
+    "    schema_map_query = f\"\"\"\n",
+    "        SELECT TABLE_NAME as tablename, COLUMN_NAME as columnname,\n",
+    "            DATA_TYPE + \n",
+    "            CASE \n",
+    "                WHEN CHARACTER_MAXIMUM_LENGTH IS NOT NULL THEN '(' + CAST(CHARACTER_MAXIMUM_LENGTH AS VARCHAR(10)) + ')'\n",
+    "                WHEN CHARACTER_MAXIMUM_LENGTH = -1 THEN '(max)'\n",
+    "                WHEN DATA_TYPE = 'decimal' THEN '(' + CAST(NUMERIC_PRECISION AS VARCHAR(10)) + ',' + CAST(NUMERIC_SCALE AS VARCHAR(10)) + ')'\n",
+    "                ELSE '' \n",
+    "            END AS datatype\n",
+    "        FROM INFORMATION_SCHEMA.COLUMNS\n",
+    "        WHERE TABLE_SCHEMA = '{SYNAPSE_EDL_SCHEMA}'\n",
+    "          AND TABLE_NAME IN (\n",
+    "              SELECT DISTINCT TABLE_NAME \n",
+    "              FROM INFORMATION_SCHEMA.COLUMNS\n",
+    "              WHERE TABLE_SCHEMA = '{SYNAPSE_EDL_SCHEMA}' AND COLUMN_NAME = '{SYNAPSE_EDL_CONST_COLUMN_NAME}'\n",
+    "          )\n",
+    "    \"\"\"\n",
+    "    with engine.connect() as conn:\n",
+    "        table_list = conn.execute(text(table_list_query)).scalar()\n",
+    "    schema_map_df = pd.read_sql(schema_map_query, engine)\n",
+    "    schema_map_json = schema_map_df.to_json(orient=\"records\")\n",
+    "\n",
+    "    return table_list, schema_map_json\n",
+    "\n",
+    "# --- DDL and View Creation ---\n",
+    "def generate_lh_view_ddl(lakehouse_name, params):\n",
+    "    \"\"\"Generate view DDL based on Lakehouse tables.\"\"\"\n",
+    "    conn = connect_to_fabric_artifact(lakehouse_name, WORKSPACE_ID)\n",
+    "\n",
+    "    variables_script = f\"\"\"\n",
+    "        DECLARE \n",
+    "            @source_database_name VARCHAR(200) = '{lakehouse_name}',\n",
+    "            @source_table_schema NVARCHAR(10) = '{params[\"source_schema\"]}',\n",
+    "            @target_table_schema NVARCHAR(10) = '{params[\"target_schema\"]}',\n",
+    "            @TablesToInclude_FnOOnly INT = {params[\"only_fno_tables\"]},\n",
+    "            @TablesToIncluce NVARCHAR(MAX) = '{params[\"tables_to_include\"]}',\n",
+    "            @TablesToExcluce NVARCHAR(MAX) = '{params[\"tables_to_exclude\"]}',\n",
+    "            @filter_deleted_rows INT = {params[\"filter_deleted_rows\"]},\n",
+    "            @join_derived_tables INT = {params[\"join_derived_tables\"]},\n",
+    "            @change_column_collation INT = {params[\"change_collation\"]},\n",
+    "            @translate_enums INT = {params[\"translate_enums\"]},\n",
+    "            @schema_map VARCHAR(MAX) = {params[\"schema_map\"]},\n",
+    "            @tableinheritance NVARCHAR(MAX) = LOWER('{params[\"derived_table_map\"]}');\n",
+    "    \"\"\"\n",
+    "    sql_template = load_builtin_file(LH_DDL_TEMPLATE_PATH)\n",
+    "    \n",
+    "    sqlquery = f\"{variables_script} {sql_template}\"\n",
+    "\n",
+    "    logger.debug(f\"Debug: generate_lh_view_ddl query {sqlquery}\")\n",
+    "    df = conn.query(f\"{sqlquery}\")\n",
+    "    ddl_statement = df.iloc[0, 0]\n",
+    "    logger.debug(f\"Debug: generate_lh_view_ddl statement {sqlquery}\")\n",
+    "\n",
+    "    return ddl_statement\n",
+    "\n",
+    "def execute_ddl_on_warehouse(warehouse_name, ddl_query):\n",
+    "    \"\"\"Execute DDL in warehouse.\"\"\"\n",
+    "    conn = connect_to_fabric_artifact(warehouse_name, WORKSPACE_ID)\n",
+    "    conn.query(ddl_query)\n",
+    "\n",
+    "# --- View Dependency Management ---\n",
+    "def fetch_view_dependencies(engine, root_entities, old_db, new_db, old_schema, new_schema):\n",
+    "    sql_template = load_builtin_file(VIEW_DEPENDENCY_TEMPLATE_PATH)\n",
+    "    sql_script = f\"\"\"\n",
+    "        SET NOCOUNT ON;\n",
+    "        DROP TABLE IF EXISTS #myEntitiestree;\n",
+    "        DECLARE @entities NVARCHAR(MAX) = '{root_entities}';\n",
+    "        DECLARE @old_schema VARCHAR(10) = '{old_schema}';\n",
+    "        DECLARE @new_schema VARCHAR(10) = '{new_schema}';\n",
+    "        {sql_template}\n",
+    "    \"\"\"\n",
+    "    with engine.connect() as conn:\n",
+    "        result = conn.execute(text(sql_script))\n",
+    "        df = pd.DataFrame(result.fetchall(), columns=result.keys())\n",
+    "    df[\"definition\"] = df[\"definition\"].str.replace(old_db, new_db, case=False, regex=False)\n",
+    "    return df.to_json(orient=\"records\")\n",
+    "\n",
+    "def deploy_views(warehouse_name, schema, views_json):\n",
+    "    execute_ddl_on_warehouse(warehouse_name, f\"IF NOT EXISTS (SELECT 1 FROM sys.schemas WHERE name = '{schema}') BEGIN EXEC('CREATE SCHEMA {schema}'); END;\")\n",
+    "    views = json.loads(views_json)\n",
+    "    for view in sorted(views, key=lambda x: x[\"depth\"], reverse=True):\n",
+    "        if view.get(\"definition\"):\n",
+    "            try:\n",
+    "                entityname = view[\"entityName\"]\n",
+    "                execute_ddl_on_warehouse(warehouse_name, view[\"definition\"])\n",
+    "                logger.info(f\"✅ Deployed view: {entityname}\")\n",
+    "            except Exception as e:\n",
+    "                logger.error(f\"❌ Failed to deploy view {entityname}: {e}\")\n",
+    "\n",
+    "for handler in logging.root.handlers[:]:  # Clear existing handlers\n",
+    "    logging.root.removeHandler(handler)\n",
+    "\n",
+    "logging.basicConfig(\n",
+    "    level=logging.INFO,\n",
+    "    format=\"[%(asctime)s] %(levelname)s - %(message)s\",\n",
+    "    datefmt=\"%Y-%m-%d %H:%M:%S\",\n",
+    "    handlers=[logging.StreamHandler(sys.stdout)]  # Explicitly use stdout\n",
+    ")\n",
+    "\n",
+    "logger = logging.getLogger()  \n",
+    "\n",
+    "# --- Main Execution ---\n",
+    "def main():\n",
+    "    \n",
+    "    logger.info(\"🏗️ Step 1: Download template files if does not exists.\")\n",
+    "    # --- Download if not exists ---\n",
+    "    for filename, local_path in REQUIRED_FILES:\n",
+    "        file_url = f\"{GITHUB_RAW_BASE_URL}/{filename}\"\n",
+    "        download_file_if_not_exists(file_url, local_path)\n",
+    "        \n",
+    "    logger.info(\"🏗️ Step 2/7: Ensure case-insensitive warehouse exists.\")\n",
+    "    \n",
+    "    if create_case_insensitive_warehouse(WORKSPACE_ID, FABRIC_WH_DATABASE):\n",
+    "\n",
+    "        logger.info(\"🔹 Step 3/7: Fetch tables and schema map from Synapse.\")\n",
+    "        tables_to_include = '*'\n",
+    "        schema_map = '[]'\n",
+    "        if is_not_none_and_empty(SYNAPSE_SERVER) and is_not_none_and_empty(SYNAPSE_EDL_DATABASE) :\n",
+    "          \n",
+    "            tables_to_include, schema_map = fetch_tables_and_schema_map(synapse_edl_engine)\n",
+    "        else:\n",
+    "            logger.info(\"🔹 Step 3/7: Skipped\")\n",
+    "\n",
+    "        logger.info(\"🔁 Step 4/7: Load derived table map.\")\n",
+    "        derived_table_map = load_builtin_file(DERIVED_TABLE_MAP_PATH)\n",
+    "\n",
+    "        logger.info(f\"🛠️ Step 5/7: Generating view DDLfrom {FABRIC_LH_DATABASE}...\")\n",
+    "        \n",
+    "        # Create a copy of the parameters and update the dynamic fields\n",
+    "        params = WAREHOUSE_VIEW_DDL_PARAMETERS.copy()\n",
+    "        params[\"schema_map\"] = f\"'{schema_map}'\"\n",
+    "        params[\"derived_table_map\"] = derived_table_map\n",
+    "        params[\"tables_to_include\"] = tables_to_include\n",
+    "\n",
+    "\n",
+    "        views_ddl = generate_lh_view_ddl(FABRIC_LH_DATABASE, params)\n",
+    "      \n",
+    "      \n",
+    "        logger.info(f\"🛠️ Step 5/7: Executing views DDL on Fabric DW {FABRIC_WH_DATABASE}.\")\n",
+    "\n",
+    "        view_ddls = views_ddl.split(';')\n",
+    "        view_count = len(view_ddls)\n",
+    "        logger.info(f\"{view_count} views to deploy.\")\n",
+    "        # Loop through each item\n",
+    "        counter= 1\n",
+    "        for view_ddl in view_ddls:\n",
+    "            match = re.search(r\"CREATE\\s+OR\\s+ALTER\\s+VIEW\\s+([\\w\\.]+)\", view_ddl, re.IGNORECASE)\n",
+    "            if match:\n",
+    "                view_name = match.group(1)\n",
+    "                logger.info(f\"{view_count}/{counter} CREATE OR ALTER VIEW {view_name}\")\n",
+    "                execute_ddl_on_warehouse(FABRIC_WH_DATABASE, view_ddl)\n",
+    "                counter += 1\n",
+    "\n",
+    "        if is_not_none_and_empty(SYNAPSE_SERVER) and is_not_none_and_empty(SYNAPSE_DW_DATABASE) :\n",
+    " \n",
+    "            logger.info(\"🚀 Step 6/7: Fetch view dependencies.\")\n",
+    "            views_json = fetch_view_dependencies(\n",
+    "                synapse_dw_engine, SYNAPSE_DW_VIEWS,\n",
+    "                SYNAPSE_EDL_DATABASE, FABRIC_WH_DATABASE,\n",
+    "                SYNAPSE_DW_SCHEMA, FABRIC_WH_SCHEMA\n",
+    "            )\n",
+    "\n",
+    "            logger.info(\"🚀 Step 7/7: Deploy views to warehouse.\")\n",
+    "            deploy_views(FABRIC_WH_DATABASE, FABRIC_WH_SCHEMA, views_json)\n",
+    "        else:\n",
+    "            logger.info(\"🔹 Step 6 and 7: Skipped\")\n",
+    "\n",
+    "        logger.info(\"🎉 Deployment complete.\")\n",
+    "    else:\n",
+    "        logger.error(\"❌ Warehouse setup failed.\")\n",
+    "    \n",
+    "\n",
+    "if __name__ == \"__main__\":\n",
+    "    main()\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "dependencies": {
+   "lakehouse": {
+    "default_lakehouse": "70459bf8-2d31-42b6-947f-9eea8f2e6c8c",
+    "default_lakehouse_name": "dataverse_jjunoenvfre",
+    "default_lakehouse_workspace_id": "cca1475b-c4fe-417f-844a-f3e8a061e55d",
+    "known_lakehouses": [
+     {
+      "id": "70459bf8-2d31-42b6-947f-9eea8f2e6c8c"
+     }
+    ]
+   },
+   "warehouse": {
+    "known_warehouses": []
+   }
+  },
+  "kernel_info": {
+   "jupyter_kernel_name": "python3.11",
+   "name": "jupyter"
+  },
+  "kernelspec": {
+   "display_name": "Jupyter",
+   "name": "jupyter"
+  },
+  "language_info": {
+   "name": "python"
+  },
+  "microsoft": {
+   "language": "python",
+   "language_group": "jupyter_python",
+   "ms_spell_check": {
+    "ms_spell_check_language": "en"
+   }
+  },
+  "nteract": {
+   "version": "nteract-front-end@1.0.0"
+  },
+  "spark_compute": {
+   "compute_id": "/trident/default",
+   "session_options": {
+    "conf": {
+     "spark.synapse.nbs.session.timeout": "1200000"
+    }
+   }
+  },
+  "synapse_widget": {
+   "state": {},
+   "version": "0.1"
+  },
+  "widgets": {}
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

From 275736e9208081641c7132946bb6544c1381a374 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Wed, 1 Jul 2026 07:12:24 +0000
Subject: [PATCH 2/2] Normalize notebook JSON while keeping URL fix

---
 .../DVFabricLinkUtil.ipynb                    | 575 +-----------------
 1 file changed, 1 insertion(+), 574 deletions(-)

diff --git a/Administration/Analytics/DataverseLink/FabricLink_SQLAnalyticsEndpoint/DVFabricLinkUtil.ipynb b/Administration/Analytics/DataverseLink/FabricLink_SQLAnalyticsEndpoint/DVFabricLinkUtil.ipynb
index 684cf9a8..6fbf292e 100644
--- a/Administration/Analytics/DataverseLink/FabricLink_SQLAnalyticsEndpoint/DVFabricLinkUtil.ipynb
+++ b/Administration/Analytics/DataverseLink/FabricLink_SQLAnalyticsEndpoint/DVFabricLinkUtil.ipynb
@@ -1,574 +1 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "id": "97ea5c26-d6c2-4e71-a58c-217d10d63b4f",
-   "metadata": {
-    "microsoft": {
-     "language": "python",
-     "language_group": "jupyter_python"
-    },
-    "nteract": {
-     "transient": {
-      "deleting": false
-     }
-    }
-   },
-   "source": [
-    "# Notebook does following \n",
-    "\n",
-    "- 🏗️ Step 1: Create case-insensitive Data Warehouse.\n",
-    "- 🔹 Step 2: Connect to Synapse Serverless Database and get table list and schema information.\n",
-    "- 🔁 Step 3: Get parent and child tables information from local file.\n",
-    "- 🛠️ Step 4: Connect to Fabric Link lakehouse get table metadata and generate view ddl statements.\n",
-    "- 🏃‍♂️ Step 5: Connect to Fabric case in-senstive data warehouse and create views.\n",
-    "- 🚀 Step 6: Connect to Synaspe serverless virtual datawarehouse and collect views and dependencies.\n",
-    "- 🚀 Step 7: Connect to Fabric datawarehouse and deploy create views\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "50733e90-a4fc-4424-896b-16399342f769",
-   "metadata": {
-    "microsoft": {
-     "language": "python",
-     "language_group": "jupyter_python"
-    }
-   },
-   "outputs": [
-    {
-     "data": {
-      "application/vnd.jupyter.statement-meta+json": {
-       "execution_finish_time": "2025-07-29T22:36:44.4572043Z",
-       "execution_start_time": "2025-07-29T22:36:44.0752036Z",
-       "normalized_state": "finished",
-       "parent_msg_id": "7deffeec-b256-4647-bd8f-893cbaccb677",
-       "queued_time": "2025-07-29T22:36:44.0741704Z",
-       "session_id": "801eb0aa-9a3a-4c30-b877-75c0402afbf2",
-       "session_start_time": null
-      }
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    }
-   ],
-   "source": [
-    "# --- Configuration Constants ---\n",
-    "WORKSPACE_ID        = \"cca1475b-c4fe-417f-844a-f3e8a061e55d\"\n",
-    "FABRIC_LH_DATABASE  = \"dataverse_jjunoenvfre\"\n",
-    "FABRIC_WH_DATABASE  = \"Sales_DW\"\n",
-    "\n",
-    "#Optional parameters to connect to your Synapse Serveress to get table and schema info\n",
-    "DRIVER              = \"{ODBC Driver 18 for SQL Server}\"\n",
-    "SYNAPSE_SERVER      = None # update to example value \"d365analyticsfabricsynapse-ondemand.sql.azuresynapse.net\"  \n",
-    "SYNAPSE_EDL_DATABASE= None #update to example value \"analytics.sandbox.operations.dynamics.com\" \n",
-    "SYNAPSE_EDL_SCHEMA  = None  #\"dbo\" \n",
-    "SYNAPSE_EDL_CONST_COLUMN_NAME = None #=\"_SysRowId\" \n",
-    "\n",
-    "\n",
-    "#Optional parameters to connect to your Synapse Serveress Datawarehouse to get views and dependencies\n",
-    "SYNAPSE_DW_DATABASE = None # example value \"Dynamics365_DW\"\n",
-    "SYNAPSE_DW_SCHEMA   = None # example value \"dbo\"\n",
-    "SYNAPSE_DW_VIEWS    = None # example value \"CustomerDim,SalesFact,SupplierDim\"\n",
-    "FABRIC_WH_SCHEMA    = None # example value \"edw\"\n",
-    "\n",
-    "#Fixed parameter - No need to change\n",
-    "GITHUB_RAW_BASE_URL = \"https://raw.githubusercontent.com/microsoft/Dynamics-365-FastTrack-Implementation-Assets/refs/heads/master/Administration/Analytics/DataverseLink/FabricLink_SQLAnalyticsEndpoint/DVFabricLinkUtil/\"\n",
-    "\n",
-    "DERIVED_TABLE_MAP_PATH = \"./builtin/resources/derived_table_map.json\"\n",
-    "LH_DDL_TEMPLATE_PATH = \"./builtin/resources/get_lh_ddl_as_view.sql\"\n",
-    "VIEW_DEPENDENCY_TEMPLATE_PATH = \"./builtin/resources/get_view_dependency.sql\"\n",
-    "REQUIRED_FILES = [\n",
-    "    (\"derived_table_map.json\", DERIVED_TABLE_MAP_PATH),\n",
-    "    (\"get_lh_ddl_as_view.sql\", LH_DDL_TEMPLATE_PATH),\n",
-    "    (\"get_view_dependency.sql\", VIEW_DEPENDENCY_TEMPLATE_PATH)]\n",
-    "\n",
-    "WAREHOUSE_VIEW_DDL_PARAMETERS = {\n",
-    "            \"source_schema\": \"dbo\",\n",
-    "            \"target_schema\": \"dbo\",\n",
-    "            \"only_fno_tables\": 1,\n",
-    "            \"tables_to_include\":\"*\",\n",
-    "            \"tables_to_exclude\": \"*\",\n",
-    "            \"filter_deleted_rows\": 1,\n",
-    "            \"join_derived_tables\": 1,\n",
-    "            \"change_collation\": 1,\n",
-    "            \"translate_enums\": 0,\n",
-    "            \"schema_map\": '[]',\n",
-    "            \"derived_table_map\": '[]'\n",
-    "        }\n",
-    "   \n",
-    "\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
-   "id": "8c2d3a9a-f5c8-4078-a36f-b540e44e6bfd",
-   "metadata": {
-    "jupyter": {
-     "outputs_hidden": false,
-     "source_hidden": false
-    },
-    "microsoft": {
-     "language": "python",
-     "language_group": "jupyter_python"
-    },
-    "nteract": {
-     "transient": {
-      "deleting": false
-     }
-    }
-   },
-   "outputs": [
-    {
-     "data": {
-      "application/vnd.jupyter.statement-meta+json": {
-       "execution_finish_time": "2025-07-29T22:37:29.0235544Z",
-       "execution_start_time": "2025-07-29T22:36:44.4586243Z",
-       "normalized_state": "finished",
-       "parent_msg_id": "3785be1d-73c9-43c0-901e-5ad3b7920f19",
-       "queued_time": "2025-07-29T22:36:44.133805Z",
-       "session_id": "801eb0aa-9a3a-4c30-b877-75c0402afbf2",
-       "session_start_time": null
-      }
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "[2025-07-29 22:36:44] INFO - 🏗️ Step 1: Download template files if does not exists.\n",
-      "[2025-07-29 22:36:44] INFO - 📄 File already exists locally: ./builtin/resources/derived_table_map.json\n",
-      "[2025-07-29 22:36:44] INFO - 📄 File already exists locally: ./builtin/resources/get_lh_ddl_as_view.sql\n",
-      "[2025-07-29 22:36:44] INFO - 📄 File already exists locally: ./builtin/resources/get_view_dependency.sql\n",
-      "[2025-07-29 22:36:44] INFO - 🏗️ Step 2/7: Ensure case-insensitive warehouse exists.\n",
-      "[2025-07-29 22:36:45] INFO - ✅ Warehouse 'Sales_DW' already exists.\n",
-      "[2025-07-29 22:36:45] INFO - 🔹 Step 3/7: Fetch tables and schema map from Synapse.\n",
-      "[2025-07-29 22:36:45] INFO - 🔹 Step 3/7: Skipped\n",
-      "[2025-07-29 22:36:45] INFO - 🔁 Step 4/7: Load derived table map.\n",
-      "[2025-07-29 22:36:45] INFO - 🛠️ Step 5/7: Generating view DDLfrom {FABRIC_LH_DATABASE}...\n",
-      "[2025-07-29 22:37:06] INFO - 🛠️ Step 5/7: Executing views DDL on Fabric DW Sales_DW.\n",
-      "[2025-07-29 22:37:06] INFO - 65 views to deploy.\n",
-      "[2025-07-29 22:37:06] INFO - 65/1 CREATE OR ALTER VIEW dbo.commissionsalesgroup\n",
-      "[2025-07-29 22:37:06] INFO - 65/2 CREATE OR ALTER VIEW dbo.companyinfo\n",
-      "[2025-07-29 22:37:06] INFO - 65/3 CREATE OR ALTER VIEW dbo.custgroup\n",
-      "[2025-07-29 22:37:06] INFO - 65/4 CREATE OR ALTER VIEW dbo.custinvoicejour\n",
-      "[2025-07-29 22:37:07] INFO - 65/5 CREATE OR ALTER VIEW dbo.custinvoicetrans\n",
-      "[2025-07-29 22:37:08] INFO - 65/6 CREATE OR ALTER VIEW dbo.custpackingslipjour\n",
-      "[2025-07-29 22:37:08] INFO - 65/7 CREATE OR ALTER VIEW dbo.custpackingsliptrans\n",
-      "[2025-07-29 22:37:08] INFO - 65/8 CREATE OR ALTER VIEW dbo.custtable\n",
-      "[2025-07-29 22:37:10] INFO - 65/9 CREATE OR ALTER VIEW dbo.custtrans\n",
-      "[2025-07-29 22:37:10] INFO - 65/10 CREATE OR ALTER VIEW dbo.custtransopen\n",
-      "[2025-07-29 22:37:10] INFO - 65/11 CREATE OR ALTER VIEW dbo.dimensionattribute\n",
-      "[2025-07-29 22:37:11] INFO - 65/12 CREATE OR ALTER VIEW dbo.dimensionattributevalue\n",
-      "[2025-07-29 22:37:11] INFO - 65/13 CREATE OR ALTER VIEW dbo.dimensionattributevaluesetitem\n",
-      "[2025-07-29 22:37:11] INFO - 65/14 CREATE OR ALTER VIEW dbo.dirdunsnumber\n",
-      "[2025-07-29 22:37:11] INFO - 65/15 CREATE OR ALTER VIEW dbo.dirnameaffix\n",
-      "[2025-07-29 22:37:11] INFO - 65/16 CREATE OR ALTER VIEW dbo.dirpartylocation\n",
-      "[2025-07-29 22:37:11] INFO - 65/17 CREATE OR ALTER VIEW dbo.dirpartytable\n",
-      "[2025-07-29 22:37:11] INFO - 65/18 CREATE OR ALTER VIEW dbo.dirpersonname\n",
-      "[2025-07-29 22:37:11] INFO - 65/19 CREATE OR ALTER VIEW dbo.ecoresproduct\n",
-      "[2025-07-29 22:37:11] INFO - 65/20 CREATE OR ALTER VIEW dbo.ecoresproducttranslation\n",
-      "[2025-07-29 22:37:11] INFO - 65/21 CREATE OR ALTER VIEW dbo.generaljournalaccountentry\n",
-      "[2025-07-29 22:37:11] INFO - 65/22 CREATE OR ALTER VIEW dbo.generaljournalentry\n",
-      "[2025-07-29 22:37:11] INFO - 65/23 CREATE OR ALTER VIEW dbo.hcmworker\n",
-      "[2025-07-29 22:37:11] INFO - 65/24 CREATE OR ALTER VIEW dbo.inventdim\n",
-      "[2025-07-29 22:37:11] INFO - 65/25 CREATE OR ALTER VIEW dbo.inventitemgroup\n",
-      "[2025-07-29 22:37:11] INFO - 65/26 CREATE OR ALTER VIEW dbo.inventitemgroupitem\n",
-      "[2025-07-29 22:37:11] INFO - 65/27 CREATE OR ALTER VIEW dbo.inventiteminventsetup\n",
-      "[2025-07-29 22:37:11] INFO - 65/28 CREATE OR ALTER VIEW dbo.inventitemlocation\n",
-      "[2025-07-29 22:37:12] INFO - 65/29 CREATE OR ALTER VIEW dbo.inventitemprice\n",
-      "[2025-07-29 22:37:12] INFO - 65/30 CREATE OR ALTER VIEW dbo.inventitempurchsetup\n",
-      "[2025-07-29 22:37:12] INFO - 65/31 CREATE OR ALTER VIEW dbo.inventitemsalessetup\n",
-      "[2025-07-29 22:37:12] INFO - 65/32 CREATE OR ALTER VIEW dbo.inventitemsampling\n",
-      "[2025-07-29 22:37:12] INFO - 65/33 CREATE OR ALTER VIEW dbo.inventjournaltable\n",
-      "[2025-07-29 22:37:12] INFO - 65/34 CREATE OR ALTER VIEW dbo.inventjournaltrans\n",
-      "[2025-07-29 22:37:12] INFO - 65/35 CREATE OR ALTER VIEW dbo.inventsettlement\n",
-      "[2025-07-29 22:37:12] INFO - 65/36 CREATE OR ALTER VIEW dbo.inventsum\n",
-      "[2025-07-29 22:37:12] INFO - 65/37 CREATE OR ALTER VIEW dbo.inventtable\n",
-      "[2025-07-29 22:37:13] INFO - 65/38 CREATE OR ALTER VIEW dbo.inventtrans\n",
-      "[2025-07-29 22:37:14] INFO - 65/39 CREATE OR ALTER VIEW dbo.inventtransorigin\n",
-      "[2025-07-29 22:37:14] INFO - 65/40 CREATE OR ALTER VIEW dbo.inventtransposting\n",
-      "[2025-07-29 22:37:14] INFO - 65/41 CREATE OR ALTER VIEW dbo.ledgerjournaltable\n",
-      "[2025-07-29 22:37:14] INFO - 65/42 CREATE OR ALTER VIEW dbo.ledgerjournaltrans\n",
-      "[2025-07-29 22:37:17] INFO - 65/43 CREATE OR ALTER VIEW dbo.logisticsaddresscountryregion\n",
-      "[2025-07-29 22:37:17] INFO - 65/44 CREATE OR ALTER VIEW dbo.logisticsaddresscountryregiontranslation\n",
-      "[2025-07-29 22:37:17] INFO - 65/45 CREATE OR ALTER VIEW dbo.logisticselectronicaddress\n",
-      "[2025-07-29 22:37:17] INFO - 65/46 CREATE OR ALTER VIEW dbo.logisticslocation\n",
-      "[2025-07-29 22:37:17] INFO - 65/47 CREATE OR ALTER VIEW dbo.logisticslocationext\n",
-      "[2025-07-29 22:37:17] INFO - 65/48 CREATE OR ALTER VIEW dbo.logisticspostaladdress\n",
-      "[2025-07-29 22:37:17] INFO - 65/49 CREATE OR ALTER VIEW dbo.omteammembershipcriterion\n",
-      "[2025-07-29 22:37:17] INFO - 65/50 CREATE OR ALTER VIEW dbo.pdscatchweightitem\n",
-      "[2025-07-29 22:37:17] INFO - 65/51 CREATE OR ALTER VIEW dbo.purchline\n",
-      "[2025-07-29 22:37:19] INFO - 65/52 CREATE OR ALTER VIEW dbo.purchtable\n",
-      "[2025-07-29 22:37:20] INFO - 65/53 CREATE OR ALTER VIEW dbo.reqtrans\n",
-      "[2025-07-29 22:37:20] INFO - 65/54 CREATE OR ALTER VIEW dbo.salesline\n",
-      "[2025-07-29 22:37:22] INFO - 65/55 CREATE OR ALTER VIEW dbo.salesline_w\n",
-      "[2025-07-29 22:37:22] INFO - 65/56 CREATE OR ALTER VIEW dbo.salestable\n",
-      "[2025-07-29 22:37:24] INFO - 65/57 CREATE OR ALTER VIEW dbo.systemparameters\n",
-      "[2025-07-29 22:37:24] INFO - 65/58 CREATE OR ALTER VIEW dbo.taxtrans\n",
-      "[2025-07-29 22:37:24] INFO - 65/59 CREATE OR ALTER VIEW dbo.vendgroup\n",
-      "[2025-07-29 22:37:24] INFO - 65/60 CREATE OR ALTER VIEW dbo.vendtable\n",
-      "[2025-07-29 22:37:26] INFO - 65/61 CREATE OR ALTER VIEW dbo.vendtrans\n",
-      "[2025-07-29 22:37:26] INFO - 65/62 CREATE OR ALTER VIEW dbo.whsinventenabled\n",
-      "[2025-07-29 22:37:26] INFO - 65/63 CREATE OR ALTER VIEW dbo.whsinventreserve\n",
-      "[2025-07-29 22:37:26] INFO - 65/64 CREATE OR ALTER VIEW dbo.whsreservationhierarchyelement\n",
-      "[2025-07-29 22:37:26] INFO - 65/65 CREATE OR ALTER VIEW dbo.dirpartytable\n",
-      "[2025-07-29 22:37:27] INFO - 🔹 Step 6 and 7: Skipped\n",
-      "[2025-07-29 22:37:27] INFO - 🎉 Deployment complete.\n"
-     ]
-    }
-   ],
-   "source": [
-    "# --- Imports ---\n",
-    "import requests\n",
-    "import json\n",
-    "import logging\n",
-    "import time\n",
-    "import struct\n",
-    "import urllib.parse\n",
-    "import pandas as pd\n",
-    "from IPython.display import display, JSON\n",
-    "from sqlalchemy import create_engine, text, event\n",
-    "from requests.exceptions import HTTPError\n",
-    "import sys\n",
-    "import os\n",
-    "import re\n",
-    "\n",
-    "synapse_edl_engine = None\n",
-    "synapse_dw_engine = None\n",
-    "\n",
-    "def download_file_if_not_exists(url, local_path):\n",
-    "    \"\"\"Download a file from GitHub if it doesn't exist locally.\"\"\"\n",
-    "    if not notebookutils.fs.exists(local_path):\n",
-    "        notebookutils.fs.mkdirs(os.path.dirname(local_path))\n",
-    "        logger.info(f\"⬇️ Downloading {local_path} ...\")\n",
-    "        response = requests.get(url)\n",
-    "        response.raise_for_status()  # Fail if not 200 OK\n",
-    "        notebookutils.fs.put(local_path, response.content.decode('utf-8'))  # <-- decode bytes to string\n",
-    "    else:\n",
-    "        logger.info(f\"📄 File already exists locally: {local_path}\")\n",
-    "\n",
-    "def is_not_none_and_empty(s):\n",
-    "    return s is not None and s != ''\n",
-    "\n",
-    "\n",
-    "# --- Database Connections ---\n",
-    "def create_synapse_engine(server, database):\n",
-    "    connection_string = f\"DRIVER={DRIVER};SERVER={server};DATABASE={database};Encrypt=yes;TrustServerCertificate=no;\"\n",
-    "    odbc_conn_str = f\"mssql+pyodbc:///?odbc_connect={urllib.parse.quote_plus(connection_string)}\"\n",
-    "    engine = create_engine(odbc_conn_str)\n",
-    "    return engine\n",
-    "\n",
-    "synapse_edl_engine = create_synapse_engine(SYNAPSE_SERVER, SYNAPSE_EDL_DATABASE)\n",
-    "synapse_dw_engine = create_synapse_engine(SYNAPSE_SERVER, SYNAPSE_DW_DATABASE)\n",
-    "\n",
-    "@event.listens_for(synapse_edl_engine, \"do_connect\")\n",
-    "@event.listens_for(synapse_dw_engine, \"do_connect\")\n",
-    "def inject_access_token(dialect, conn_rec, cargs, cparams):\n",
-    "    token = notebookutils.credentials.getToken(\"https://database.windows.net/\")\n",
-    "    token_bytes = token.encode(\"utf-16-le\")\n",
-    "    token_struct = struct.pack(f\"<I{len(token_bytes)}s\", len(token_bytes), token_bytes)\n",
-    "    cparams[\"attrs_before\"] = {1256: token_struct}\n",
-    "\n",
-    "# --- Utility Functions ---\n",
-    "def load_builtin_file(path):\n",
-    "    \"\"\"Load a built-in SQL or JSON file.\"\"\"\n",
-    "    return notebookutils.fs.head(path)\n",
-    "\n",
-    "def connect_to_fabric_artifact(artifact_name, workspace_id):\n",
-    "    return notebookutils.data.connect_to_artifact(artifact_name, workspace_id)\n",
-    "\n",
-    "# --- Warehouse Management ---\n",
-    "def warehouse_exists(workspace_id, warehouse_name):\n",
-    "    try:\n",
-    "        connect_to_fabric_artifact(warehouse_name, workspace_id)\n",
-    "        return True\n",
-    "    except Exception as e:\n",
-    "        if \"ArtifactNotFoundException\" in str(type(e)):\n",
-    "            logger.info(f\"🔍 Warehouse not found:{warehouse_name}\")\n",
-    "            return False\n",
-    "        raise\n",
-    "\n",
-    "def create_case_insensitive_warehouse(workspace_id, warehouse_name, retries=5, delay=10):\n",
-    "    if warehouse_exists(workspace_id, warehouse_name):\n",
-    "        logger.info(f\"✅ Warehouse '{warehouse_name}' already exists.\")\n",
-    "        return True\n",
-    "\n",
-    "    logger.info(f\"🏗️ Creating new warehouse:{warehouse_name}\")\n",
-    "    token = notebookutils.credentials.getToken(\"https://api.fabric.microsoft.com/\")\n",
-    "    headers = {\"Authorization\": f\"Bearer {token}\", \"Content-Type\": \"application/json\"}\n",
-    "    payload = {\n",
-    "        \"type\": \"Warehouse\",\n",
-    "        \"displayName\": warehouse_name,\n",
-    "        \"description\": \"New warehouse with case-insensitive collation\",\n",
-    "        \"creationPayload\": {\"defaultCollation\": \"Latin1_General_100_CI_AS_KS_WS_SC_UTF8\"}\n",
-    "    }\n",
-    "\n",
-    "    response = requests.post(\n",
-    "        f\"https://api.fabric.microsoft.com/v1/workspaces/{workspace_id}/items\",\n",
-    "        headers=headers, data=json.dumps(payload)\n",
-    "    )\n",
-    "    logger.info(f\"📨 Warehouse creation response: {response.text}\", )\n",
-    "\n",
-    "    for attempt in range(retries):\n",
-    "        time.sleep(delay)\n",
-    "        if warehouse_exists(workspace_id, warehouse_name):\n",
-    "            logger.info(\"✅ Warehouse is ready.\")\n",
-    "            return True\n",
-    "        logger.info(f\"🔄 Retry {attempt + 1}/{retries}: Warehouse not yet available.\")\n",
-    "\n",
-    "    logger.error(\"❌ Failed to create warehouse after retries.\")\n",
-    "    return False\n",
-    "\n",
-    "# --- Schema Retrieval ---\n",
-    "def fetch_tables_and_schema_map(engine):\n",
-    "    \"\"\"Fetch table list and schema mapping.\"\"\"\n",
-    "    table_list_query = f\"\"\"\n",
-    "        SELECT STRING_AGG(CONVERT(NVARCHAR(MAX), TABLE_NAME), ',') AS tablelist\n",
-    "        FROM (\n",
-    "            SELECT DISTINCT LOWER(TABLE_NAME) as TABLE_NAME\n",
-    "            FROM INFORMATION_SCHEMA.COLUMNS\n",
-    "            WHERE TABLE_SCHEMA = '{SYNAPSE_EDL_SCHEMA}' \n",
-    "              AND TABLE_NAME IN (\n",
-    "                  SELECT DISTINCT TABLE_NAME \n",
-    "                  FROM INFORMATION_SCHEMA.COLUMNS\n",
-    "                  WHERE TABLE_SCHEMA = '{SYNAPSE_EDL_SCHEMA}' \n",
-    "                    AND COLUMN_NAME = '{SYNAPSE_EDL_CONST_COLUMN_NAME}'\n",
-    "              )\n",
-    "        ) AS tbl\n",
-    "    \"\"\"\n",
-    "    schema_map_query = f\"\"\"\n",
-    "        SELECT TABLE_NAME as tablename, COLUMN_NAME as columnname,\n",
-    "            DATA_TYPE + \n",
-    "            CASE \n",
-    "                WHEN CHARACTER_MAXIMUM_LENGTH IS NOT NULL THEN '(' + CAST(CHARACTER_MAXIMUM_LENGTH AS VARCHAR(10)) + ')'\n",
-    "                WHEN CHARACTER_MAXIMUM_LENGTH = -1 THEN '(max)'\n",
-    "                WHEN DATA_TYPE = 'decimal' THEN '(' + CAST(NUMERIC_PRECISION AS VARCHAR(10)) + ',' + CAST(NUMERIC_SCALE AS VARCHAR(10)) + ')'\n",
-    "                ELSE '' \n",
-    "            END AS datatype\n",
-    "        FROM INFORMATION_SCHEMA.COLUMNS\n",
-    "        WHERE TABLE_SCHEMA = '{SYNAPSE_EDL_SCHEMA}'\n",
-    "          AND TABLE_NAME IN (\n",
-    "              SELECT DISTINCT TABLE_NAME \n",
-    "              FROM INFORMATION_SCHEMA.COLUMNS\n",
-    "              WHERE TABLE_SCHEMA = '{SYNAPSE_EDL_SCHEMA}' AND COLUMN_NAME = '{SYNAPSE_EDL_CONST_COLUMN_NAME}'\n",
-    "          )\n",
-    "    \"\"\"\n",
-    "    with engine.connect() as conn:\n",
-    "        table_list = conn.execute(text(table_list_query)).scalar()\n",
-    "    schema_map_df = pd.read_sql(schema_map_query, engine)\n",
-    "    schema_map_json = schema_map_df.to_json(orient=\"records\")\n",
-    "\n",
-    "    return table_list, schema_map_json\n",
-    "\n",
-    "# --- DDL and View Creation ---\n",
-    "def generate_lh_view_ddl(lakehouse_name, params):\n",
-    "    \"\"\"Generate view DDL based on Lakehouse tables.\"\"\"\n",
-    "    conn = connect_to_fabric_artifact(lakehouse_name, WORKSPACE_ID)\n",
-    "\n",
-    "    variables_script = f\"\"\"\n",
-    "        DECLARE \n",
-    "            @source_database_name VARCHAR(200) = '{lakehouse_name}',\n",
-    "            @source_table_schema NVARCHAR(10) = '{params[\"source_schema\"]}',\n",
-    "            @target_table_schema NVARCHAR(10) = '{params[\"target_schema\"]}',\n",
-    "            @TablesToInclude_FnOOnly INT = {params[\"only_fno_tables\"]},\n",
-    "            @TablesToIncluce NVARCHAR(MAX) = '{params[\"tables_to_include\"]}',\n",
-    "            @TablesToExcluce NVARCHAR(MAX) = '{params[\"tables_to_exclude\"]}',\n",
-    "            @filter_deleted_rows INT = {params[\"filter_deleted_rows\"]},\n",
-    "            @join_derived_tables INT = {params[\"join_derived_tables\"]},\n",
-    "            @change_column_collation INT = {params[\"change_collation\"]},\n",
-    "            @translate_enums INT = {params[\"translate_enums\"]},\n",
-    "            @schema_map VARCHAR(MAX) = {params[\"schema_map\"]},\n",
-    "            @tableinheritance NVARCHAR(MAX) = LOWER('{params[\"derived_table_map\"]}');\n",
-    "    \"\"\"\n",
-    "    sql_template = load_builtin_file(LH_DDL_TEMPLATE_PATH)\n",
-    "    \n",
-    "    sqlquery = f\"{variables_script} {sql_template}\"\n",
-    "\n",
-    "    logger.debug(f\"Debug: generate_lh_view_ddl query {sqlquery}\")\n",
-    "    df = conn.query(f\"{sqlquery}\")\n",
-    "    ddl_statement = df.iloc[0, 0]\n",
-    "    logger.debug(f\"Debug: generate_lh_view_ddl statement {sqlquery}\")\n",
-    "\n",
-    "    return ddl_statement\n",
-    "\n",
-    "def execute_ddl_on_warehouse(warehouse_name, ddl_query):\n",
-    "    \"\"\"Execute DDL in warehouse.\"\"\"\n",
-    "    conn = connect_to_fabric_artifact(warehouse_name, WORKSPACE_ID)\n",
-    "    conn.query(ddl_query)\n",
-    "\n",
-    "# --- View Dependency Management ---\n",
-    "def fetch_view_dependencies(engine, root_entities, old_db, new_db, old_schema, new_schema):\n",
-    "    sql_template = load_builtin_file(VIEW_DEPENDENCY_TEMPLATE_PATH)\n",
-    "    sql_script = f\"\"\"\n",
-    "        SET NOCOUNT ON;\n",
-    "        DROP TABLE IF EXISTS #myEntitiestree;\n",
-    "        DECLARE @entities NVARCHAR(MAX) = '{root_entities}';\n",
-    "        DECLARE @old_schema VARCHAR(10) = '{old_schema}';\n",
-    "        DECLARE @new_schema VARCHAR(10) = '{new_schema}';\n",
-    "        {sql_template}\n",
-    "    \"\"\"\n",
-    "    with engine.connect() as conn:\n",
-    "        result = conn.execute(text(sql_script))\n",
-    "        df = pd.DataFrame(result.fetchall(), columns=result.keys())\n",
-    "    df[\"definition\"] = df[\"definition\"].str.replace(old_db, new_db, case=False, regex=False)\n",
-    "    return df.to_json(orient=\"records\")\n",
-    "\n",
-    "def deploy_views(warehouse_name, schema, views_json):\n",
-    "    execute_ddl_on_warehouse(warehouse_name, f\"IF NOT EXISTS (SELECT 1 FROM sys.schemas WHERE name = '{schema}') BEGIN EXEC('CREATE SCHEMA {schema}'); END;\")\n",
-    "    views = json.loads(views_json)\n",
-    "    for view in sorted(views, key=lambda x: x[\"depth\"], reverse=True):\n",
-    "        if view.get(\"definition\"):\n",
-    "            try:\n",
-    "                entityname = view[\"entityName\"]\n",
-    "                execute_ddl_on_warehouse(warehouse_name, view[\"definition\"])\n",
-    "                logger.info(f\"✅ Deployed view: {entityname}\")\n",
-    "            except Exception as e:\n",
-    "                logger.error(f\"❌ Failed to deploy view {entityname}: {e}\")\n",
-    "\n",
-    "for handler in logging.root.handlers[:]:  # Clear existing handlers\n",
-    "    logging.root.removeHandler(handler)\n",
-    "\n",
-    "logging.basicConfig(\n",
-    "    level=logging.INFO,\n",
-    "    format=\"[%(asctime)s] %(levelname)s - %(message)s\",\n",
-    "    datefmt=\"%Y-%m-%d %H:%M:%S\",\n",
-    "    handlers=[logging.StreamHandler(sys.stdout)]  # Explicitly use stdout\n",
-    ")\n",
-    "\n",
-    "logger = logging.getLogger()  \n",
-    "\n",
-    "# --- Main Execution ---\n",
-    "def main():\n",
-    "    \n",
-    "    logger.info(\"🏗️ Step 1: Download template files if does not exists.\")\n",
-    "    # --- Download if not exists ---\n",
-    "    for filename, local_path in REQUIRED_FILES:\n",
-    "        file_url = f\"{GITHUB_RAW_BASE_URL}/{filename}\"\n",
-    "        download_file_if_not_exists(file_url, local_path)\n",
-    "        \n",
-    "    logger.info(\"🏗️ Step 2/7: Ensure case-insensitive warehouse exists.\")\n",
-    "    \n",
-    "    if create_case_insensitive_warehouse(WORKSPACE_ID, FABRIC_WH_DATABASE):\n",
-    "\n",
-    "        logger.info(\"🔹 Step 3/7: Fetch tables and schema map from Synapse.\")\n",
-    "        tables_to_include = '*'\n",
-    "        schema_map = '[]'\n",
-    "        if is_not_none_and_empty(SYNAPSE_SERVER) and is_not_none_and_empty(SYNAPSE_EDL_DATABASE) :\n",
-    "          \n",
-    "            tables_to_include, schema_map = fetch_tables_and_schema_map(synapse_edl_engine)\n",
-    "        else:\n",
-    "            logger.info(\"🔹 Step 3/7: Skipped\")\n",
-    "\n",
-    "        logger.info(\"🔁 Step 4/7: Load derived table map.\")\n",
-    "        derived_table_map = load_builtin_file(DERIVED_TABLE_MAP_PATH)\n",
-    "\n",
-    "        logger.info(f\"🛠️ Step 5/7: Generating view DDLfrom {FABRIC_LH_DATABASE}...\")\n",
-    "        \n",
-    "        # Create a copy of the parameters and update the dynamic fields\n",
-    "        params = WAREHOUSE_VIEW_DDL_PARAMETERS.copy()\n",
-    "        params[\"schema_map\"] = f\"'{schema_map}'\"\n",
-    "        params[\"derived_table_map\"] = derived_table_map\n",
-    "        params[\"tables_to_include\"] = tables_to_include\n",
-    "\n",
-    "\n",
-    "        views_ddl = generate_lh_view_ddl(FABRIC_LH_DATABASE, params)\n",
-    "      \n",
-    "      \n",
-    "        logger.info(f\"🛠️ Step 5/7: Executing views DDL on Fabric DW {FABRIC_WH_DATABASE}.\")\n",
-    "\n",
-    "        view_ddls = views_ddl.split(';')\n",
-    "        view_count = len(view_ddls)\n",
-    "        logger.info(f\"{view_count} views to deploy.\")\n",
-    "        # Loop through each item\n",
-    "        counter= 1\n",
-    "        for view_ddl in view_ddls:\n",
-    "            match = re.search(r\"CREATE\\s+OR\\s+ALTER\\s+VIEW\\s+([\\w\\.]+)\", view_ddl, re.IGNORECASE)\n",
-    "            if match:\n",
-    "                view_name = match.group(1)\n",
-    "                logger.info(f\"{view_count}/{counter} CREATE OR ALTER VIEW {view_name}\")\n",
-    "                execute_ddl_on_warehouse(FABRIC_WH_DATABASE, view_ddl)\n",
-    "                counter += 1\n",
-    "\n",
-    "        if is_not_none_and_empty(SYNAPSE_SERVER) and is_not_none_and_empty(SYNAPSE_DW_DATABASE) :\n",
-    " \n",
-    "            logger.info(\"🚀 Step 6/7: Fetch view dependencies.\")\n",
-    "            views_json = fetch_view_dependencies(\n",
-    "                synapse_dw_engine, SYNAPSE_DW_VIEWS,\n",
-    "                SYNAPSE_EDL_DATABASE, FABRIC_WH_DATABASE,\n",
-    "                SYNAPSE_DW_SCHEMA, FABRIC_WH_SCHEMA\n",
-    "            )\n",
-    "\n",
-    "            logger.info(\"🚀 Step 7/7: Deploy views to warehouse.\")\n",
-    "            deploy_views(FABRIC_WH_DATABASE, FABRIC_WH_SCHEMA, views_json)\n",
-    "        else:\n",
-    "            logger.info(\"🔹 Step 6 and 7: Skipped\")\n",
-    "\n",
-    "        logger.info(\"🎉 Deployment complete.\")\n",
-    "    else:\n",
-    "        logger.error(\"❌ Warehouse setup failed.\")\n",
-    "    \n",
-    "\n",
-    "if __name__ == \"__main__\":\n",
-    "    main()\n"
-   ]
-  }
- ],
- "metadata": {
-  "dependencies": {
-   "lakehouse": {
-    "default_lakehouse": "70459bf8-2d31-42b6-947f-9eea8f2e6c8c",
-    "default_lakehouse_name": "dataverse_jjunoenvfre",
-    "default_lakehouse_workspace_id": "cca1475b-c4fe-417f-844a-f3e8a061e55d",
-    "known_lakehouses": [
-     {
-      "id": "70459bf8-2d31-42b6-947f-9eea8f2e6c8c"
-     }
-    ]
-   },
-   "warehouse": {
-    "known_warehouses": []
-   }
-  },
-  "kernel_info": {
-   "jupyter_kernel_name": "python3.11",
-   "name": "jupyter"
-  },
-  "kernelspec": {
-   "display_name": "Jupyter",
-   "name": "jupyter"
-  },
-  "language_info": {
-   "name": "python"
-  },
-  "microsoft": {
-   "language": "python",
-   "language_group": "jupyter_python",
-   "ms_spell_check": {
-    "ms_spell_check_language": "en"
-   }
-  },
-  "nteract": {
-   "version": "nteract-front-end@1.0.0"
-  },
-  "spark_compute": {
-   "compute_id": "/trident/default",
-   "session_options": {
-    "conf": {
-     "spark.synapse.nbs.session.timeout": "1200000"
-    }
-   }
-  },
-  "synapse_widget": {
-   "state": {},
-   "version": "0.1"
-  },
-  "widgets": {}
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}
+{"cells":[{"cell_type":"markdown","source":["# Notebook does following \n","\n","- 🏗️ Step 1: Create case-insensitive Data Warehouse.\n","- 🔹 Step 2: Connect to Synapse Serverless Database and get table list and schema information.\n","- 🔁 Step 3: Get parent and child tables information from local file.\n","- 🛠️ Step 4: Connect to Fabric Link lakehouse get table metadata and generate view ddl statements.\n","- 🏃‍♂️ Step 5: Connect to Fabric case in-senstive data warehouse and create views.\n","- 🚀 Step 6: Connect to Synaspe serverless virtual datawarehouse and collect views and dependencies.\n","- 🚀 Step 7: Connect to Fabric datawarehouse and deploy create views\n"],"metadata":{"microsoft":{"language":"python","language_group":"jupyter_python"},"nteract":{"transient":{"deleting":false}}},"id":"97ea5c26-d6c2-4e71-a58c-217d10d63b4f"},{"cell_type":"code","source":["# --- Configuration Constants ---\n","WORKSPACE_ID        = \"cca1475b-c4fe-417f-844a-f3e8a061e55d\"\n","FABRIC_LH_DATABASE  = \"dataverse_jjunoenvfre\"\n","FABRIC_WH_DATABASE  = \"Sales_DW\"\n","\n","#Optional parameters to connect to your Synapse Serveress to get table and schema info\n","DRIVER              = \"{ODBC Driver 18 for SQL Server}\"\n","SYNAPSE_SERVER      = None # update to example value \"d365analyticsfabricsynapse-ondemand.sql.azuresynapse.net\"  \n","SYNAPSE_EDL_DATABASE= None #update to example value \"analytics.sandbox.operations.dynamics.com\" \n","SYNAPSE_EDL_SCHEMA  = None  #\"dbo\" \n","SYNAPSE_EDL_CONST_COLUMN_NAME = None #=\"_SysRowId\" \n","\n","\n","#Optional parameters to connect to your Synapse Serveress Datawarehouse to get views and dependencies\n","SYNAPSE_DW_DATABASE = None # example value \"Dynamics365_DW\"\n","SYNAPSE_DW_SCHEMA   = None # example value \"dbo\"\n","SYNAPSE_DW_VIEWS    = None # example value \"CustomerDim,SalesFact,SupplierDim\"\n","FABRIC_WH_SCHEMA    = None # example value \"edw\"\n","\n","#Fixed parameter - No need to change\n","GITHUB_RAW_BASE_URL = \"https://raw.githubusercontent.com/microsoft/Dynamics-365-FastTrack-Implementation-Assets/refs/heads/master/Administration/Analytics/DataverseLink/FabricLink_SQLAnalyticsEndpoint/DVFabricLinkUtil/\"\n","\n","DERIVED_TABLE_MAP_PATH = \"./builtin/resources/derived_table_map.json\"\n","LH_DDL_TEMPLATE_PATH = \"./builtin/resources/get_lh_ddl_as_view.sql\"\n","VIEW_DEPENDENCY_TEMPLATE_PATH = \"./builtin/resources/get_view_dependency.sql\"\n","REQUIRED_FILES = [\n","    (\"derived_table_map.json\", DERIVED_TABLE_MAP_PATH),\n","    (\"get_lh_ddl_as_view.sql\", LH_DDL_TEMPLATE_PATH),\n","    (\"get_view_dependency.sql\", VIEW_DEPENDENCY_TEMPLATE_PATH)]\n","\n","WAREHOUSE_VIEW_DDL_PARAMETERS = {\n","            \"source_schema\": \"dbo\",\n","            \"target_schema\": \"dbo\",\n","            \"only_fno_tables\": 1,\n","            \"tables_to_include\":\"*\",\n","            \"tables_to_exclude\": \"*\",\n","            \"filter_deleted_rows\": 1,\n","            \"join_derived_tables\": 1,\n","            \"change_collation\": 1,\n","            \"translate_enums\": 0,\n","            \"schema_map\": '[]',\n","            \"derived_table_map\": '[]'\n","        }\n","   \n","\n"],"outputs":[{"output_type":"display_data","data":{"application/vnd.jupyter.statement-meta+json":{"session_id":"801eb0aa-9a3a-4c30-b877-75c0402afbf2","normalized_state":"finished","queued_time":"2025-07-29T22:36:44.0741704Z","session_start_time":null,"execution_start_time":"2025-07-29T22:36:44.0752036Z","execution_finish_time":"2025-07-29T22:36:44.4572043Z","parent_msg_id":"7deffeec-b256-4647-bd8f-893cbaccb677"}},"metadata":{}}],"execution_count":3,"metadata":{"microsoft":{"language":"python","language_group":"jupyter_python"}},"id":"50733e90-a4fc-4424-896b-16399342f769"},{"cell_type":"code","source":["# --- Imports ---\n","import requests\n","import json\n","import logging\n","import time\n","import struct\n","import urllib.parse\n","import pandas as pd\n","from IPython.display import display, JSON\n","from sqlalchemy import create_engine, text, event\n","from requests.exceptions import HTTPError\n","import sys\n","import os\n","import re\n","\n","synapse_edl_engine = None\n","synapse_dw_engine = None\n","\n","def download_file_if_not_exists(url, local_path):\n","    \"\"\"Download a file from GitHub if it doesn't exist locally.\"\"\"\n","    if not notebookutils.fs.exists(local_path):\n","        notebookutils.fs.mkdirs(os.path.dirname(local_path))\n","        logger.info(f\"⬇️ Downloading {local_path} ...\")\n","        response = requests.get(url)\n","        response.raise_for_status()  # Fail if not 200 OK\n","        notebookutils.fs.put(local_path, response.content.decode('utf-8'))  # <-- decode bytes to string\n","    else:\n","        logger.info(f\"📄 File already exists locally: {local_path}\")\n","\n","def is_not_none_and_empty(s):\n","    return s is not None and s != ''\n","\n","\n","# --- Database Connections ---\n","def create_synapse_engine(server, database):\n","    connection_string = f\"DRIVER={DRIVER};SERVER={server};DATABASE={database};Encrypt=yes;TrustServerCertificate=no;\"\n","    odbc_conn_str = f\"mssql+pyodbc:///?odbc_connect={urllib.parse.quote_plus(connection_string)}\"\n","    engine = create_engine(odbc_conn_str)\n","    return engine\n","\n","synapse_edl_engine = create_synapse_engine(SYNAPSE_SERVER, SYNAPSE_EDL_DATABASE)\n","synapse_dw_engine = create_synapse_engine(SYNAPSE_SERVER, SYNAPSE_DW_DATABASE)\n","\n","@event.listens_for(synapse_edl_engine, \"do_connect\")\n","@event.listens_for(synapse_dw_engine, \"do_connect\")\n","def inject_access_token(dialect, conn_rec, cargs, cparams):\n","    token = notebookutils.credentials.getToken(\"https://database.windows.net/\")\n","    token_bytes = token.encode(\"utf-16-le\")\n","    token_struct = struct.pack(f\"<I{len(token_bytes)}s\", len(token_bytes), token_bytes)\n","    cparams[\"attrs_before\"] = {1256: token_struct}\n","\n","# --- Utility Functions ---\n","def load_builtin_file(path):\n","    \"\"\"Load a built-in SQL or JSON file.\"\"\"\n","    return notebookutils.fs.head(path)\n","\n","def connect_to_fabric_artifact(artifact_name, workspace_id):\n","    return notebookutils.data.connect_to_artifact(artifact_name, workspace_id)\n","\n","# --- Warehouse Management ---\n","def warehouse_exists(workspace_id, warehouse_name):\n","    try:\n","        connect_to_fabric_artifact(warehouse_name, workspace_id)\n","        return True\n","    except Exception as e:\n","        if \"ArtifactNotFoundException\" in str(type(e)):\n","            logger.info(f\"🔍 Warehouse not found:{warehouse_name}\")\n","            return False\n","        raise\n","\n","def create_case_insensitive_warehouse(workspace_id, warehouse_name, retries=5, delay=10):\n","    if warehouse_exists(workspace_id, warehouse_name):\n","        logger.info(f\"✅ Warehouse '{warehouse_name}' already exists.\")\n","        return True\n","\n","    logger.info(f\"🏗️ Creating new warehouse:{warehouse_name}\")\n","    token = notebookutils.credentials.getToken(\"https://api.fabric.microsoft.com/\")\n","    headers = {\"Authorization\": f\"Bearer {token}\", \"Content-Type\": \"application/json\"}\n","    payload = {\n","        \"type\": \"Warehouse\",\n","        \"displayName\": warehouse_name,\n","        \"description\": \"New warehouse with case-insensitive collation\",\n","        \"creationPayload\": {\"defaultCollation\": \"Latin1_General_100_CI_AS_KS_WS_SC_UTF8\"}\n","    }\n","\n","    response = requests.post(\n","        f\"https://api.fabric.microsoft.com/v1/workspaces/{workspace_id}/items\",\n","        headers=headers, data=json.dumps(payload)\n","    )\n","    logger.info(f\"📨 Warehouse creation response: {response.text}\", )\n","\n","    for attempt in range(retries):\n","        time.sleep(delay)\n","        if warehouse_exists(workspace_id, warehouse_name):\n","            logger.info(\"✅ Warehouse is ready.\")\n","            return True\n","        logger.info(f\"🔄 Retry {attempt + 1}/{retries}: Warehouse not yet available.\")\n","\n","    logger.error(\"❌ Failed to create warehouse after retries.\")\n","    return False\n","\n","# --- Schema Retrieval ---\n","def fetch_tables_and_schema_map(engine):\n","    \"\"\"Fetch table list and schema mapping.\"\"\"\n","    table_list_query = f\"\"\"\n","        SELECT STRING_AGG(CONVERT(NVARCHAR(MAX), TABLE_NAME), ',') AS tablelist\n","        FROM (\n","            SELECT DISTINCT LOWER(TABLE_NAME) as TABLE_NAME\n","            FROM INFORMATION_SCHEMA.COLUMNS\n","            WHERE TABLE_SCHEMA = '{SYNAPSE_EDL_SCHEMA}' \n","              AND TABLE_NAME IN (\n","                  SELECT DISTINCT TABLE_NAME \n","                  FROM INFORMATION_SCHEMA.COLUMNS\n","                  WHERE TABLE_SCHEMA = '{SYNAPSE_EDL_SCHEMA}' \n","                    AND COLUMN_NAME = '{SYNAPSE_EDL_CONST_COLUMN_NAME}'\n","              )\n","        ) AS tbl\n","    \"\"\"\n","    schema_map_query = f\"\"\"\n","        SELECT TABLE_NAME as tablename, COLUMN_NAME as columnname,\n","            DATA_TYPE + \n","            CASE \n","                WHEN CHARACTER_MAXIMUM_LENGTH IS NOT NULL THEN '(' + CAST(CHARACTER_MAXIMUM_LENGTH AS VARCHAR(10)) + ')'\n","                WHEN CHARACTER_MAXIMUM_LENGTH = -1 THEN '(max)'\n","                WHEN DATA_TYPE = 'decimal' THEN '(' + CAST(NUMERIC_PRECISION AS VARCHAR(10)) + ',' + CAST(NUMERIC_SCALE AS VARCHAR(10)) + ')'\n","                ELSE '' \n","            END AS datatype\n","        FROM INFORMATION_SCHEMA.COLUMNS\n","        WHERE TABLE_SCHEMA = '{SYNAPSE_EDL_SCHEMA}'\n","          AND TABLE_NAME IN (\n","              SELECT DISTINCT TABLE_NAME \n","              FROM INFORMATION_SCHEMA.COLUMNS\n","              WHERE TABLE_SCHEMA = '{SYNAPSE_EDL_SCHEMA}' AND COLUMN_NAME = '{SYNAPSE_EDL_CONST_COLUMN_NAME}'\n","          )\n","    \"\"\"\n","    with engine.connect() as conn:\n","        table_list = conn.execute(text(table_list_query)).scalar()\n","    schema_map_df = pd.read_sql(schema_map_query, engine)\n","    schema_map_json = schema_map_df.to_json(orient=\"records\")\n","\n","    return table_list, schema_map_json\n","\n","# --- DDL and View Creation ---\n","def generate_lh_view_ddl(lakehouse_name, params):\n","    \"\"\"Generate view DDL based on Lakehouse tables.\"\"\"\n","    conn = connect_to_fabric_artifact(lakehouse_name, WORKSPACE_ID)\n","\n","    variables_script = f\"\"\"\n","        DECLARE \n","            @source_database_name VARCHAR(200) = '{lakehouse_name}',\n","            @source_table_schema NVARCHAR(10) = '{params[\"source_schema\"]}',\n","            @target_table_schema NVARCHAR(10) = '{params[\"target_schema\"]}',\n","            @TablesToInclude_FnOOnly INT = {params[\"only_fno_tables\"]},\n","            @TablesToIncluce NVARCHAR(MAX) = '{params[\"tables_to_include\"]}',\n","            @TablesToExcluce NVARCHAR(MAX) = '{params[\"tables_to_exclude\"]}',\n","            @filter_deleted_rows INT = {params[\"filter_deleted_rows\"]},\n","            @join_derived_tables INT = {params[\"join_derived_tables\"]},\n","            @change_column_collation INT = {params[\"change_collation\"]},\n","            @translate_enums INT = {params[\"translate_enums\"]},\n","            @schema_map VARCHAR(MAX) = {params[\"schema_map\"]},\n","            @tableinheritance NVARCHAR(MAX) = LOWER('{params[\"derived_table_map\"]}');\n","    \"\"\"\n","    sql_template = load_builtin_file(LH_DDL_TEMPLATE_PATH)\n","    \n","    sqlquery = f\"{variables_script} {sql_template}\"\n","\n","    logger.debug(f\"Debug: generate_lh_view_ddl query {sqlquery}\")\n","    df = conn.query(f\"{sqlquery}\")\n","    ddl_statement = df.iloc[0, 0]\n","    logger.debug(f\"Debug: generate_lh_view_ddl statement {sqlquery}\")\n","\n","    return ddl_statement\n","\n","def execute_ddl_on_warehouse(warehouse_name, ddl_query):\n","    \"\"\"Execute DDL in warehouse.\"\"\"\n","    conn = connect_to_fabric_artifact(warehouse_name, WORKSPACE_ID)\n","    conn.query(ddl_query)\n","\n","# --- View Dependency Management ---\n","def fetch_view_dependencies(engine, root_entities, old_db, new_db, old_schema, new_schema):\n","    sql_template = load_builtin_file(VIEW_DEPENDENCY_TEMPLATE_PATH)\n","    sql_script = f\"\"\"\n","        SET NOCOUNT ON;\n","        DROP TABLE IF EXISTS #myEntitiestree;\n","        DECLARE @entities NVARCHAR(MAX) = '{root_entities}';\n","        DECLARE @old_schema VARCHAR(10) = '{old_schema}';\n","        DECLARE @new_schema VARCHAR(10) = '{new_schema}';\n","        {sql_template}\n","    \"\"\"\n","    with engine.connect() as conn:\n","        result = conn.execute(text(sql_script))\n","        df = pd.DataFrame(result.fetchall(), columns=result.keys())\n","    df[\"definition\"] = df[\"definition\"].str.replace(old_db, new_db, case=False, regex=False)\n","    return df.to_json(orient=\"records\")\n","\n","def deploy_views(warehouse_name, schema, views_json):\n","    execute_ddl_on_warehouse(warehouse_name, f\"IF NOT EXISTS (SELECT 1 FROM sys.schemas WHERE name = '{schema}') BEGIN EXEC('CREATE SCHEMA {schema}'); END;\")\n","    views = json.loads(views_json)\n","    for view in sorted(views, key=lambda x: x[\"depth\"], reverse=True):\n","        if view.get(\"definition\"):\n","            try:\n","                entityname = view[\"entityName\"]\n","                execute_ddl_on_warehouse(warehouse_name, view[\"definition\"])\n","                logger.info(f\"✅ Deployed view: {entityname}\")\n","            except Exception as e:\n","                logger.error(f\"❌ Failed to deploy view {entityname}: {e}\")\n","\n","for handler in logging.root.handlers[:]:  # Clear existing handlers\n","    logging.root.removeHandler(handler)\n","\n","logging.basicConfig(\n","    level=logging.INFO,\n","    format=\"[%(asctime)s] %(levelname)s - %(message)s\",\n","    datefmt=\"%Y-%m-%d %H:%M:%S\",\n","    handlers=[logging.StreamHandler(sys.stdout)]  # Explicitly use stdout\n",")\n","\n","logger = logging.getLogger()  \n","\n","# --- Main Execution ---\n","def main():\n","    \n","    logger.info(\"🏗️ Step 1: Download template files if does not exists.\")\n","    # --- Download if not exists ---\n","    for filename, local_path in REQUIRED_FILES:\n","        file_url = f\"{GITHUB_RAW_BASE_URL}/{filename}\"\n","        download_file_if_not_exists(file_url, local_path)\n","        \n","    logger.info(\"🏗️ Step 2/7: Ensure case-insensitive warehouse exists.\")\n","    \n","    if create_case_insensitive_warehouse(WORKSPACE_ID, FABRIC_WH_DATABASE):\n","\n","        logger.info(\"🔹 Step 3/7: Fetch tables and schema map from Synapse.\")\n","        tables_to_include = '*'\n","        schema_map = '[]'\n","        if is_not_none_and_empty(SYNAPSE_SERVER) and is_not_none_and_empty(SYNAPSE_EDL_DATABASE) :\n","          \n","            tables_to_include, schema_map = fetch_tables_and_schema_map(synapse_edl_engine)\n","        else:\n","            logger.info(\"🔹 Step 3/7: Skipped\")\n","\n","        logger.info(\"🔁 Step 4/7: Load derived table map.\")\n","        derived_table_map = load_builtin_file(DERIVED_TABLE_MAP_PATH)\n","\n","        logger.info(f\"🛠️ Step 5/7: Generating view DDLfrom {FABRIC_LH_DATABASE}...\")\n","        \n","        # Create a copy of the parameters and update the dynamic fields\n","        params = WAREHOUSE_VIEW_DDL_PARAMETERS.copy()\n","        params[\"schema_map\"] = f\"'{schema_map}'\"\n","        params[\"derived_table_map\"] = derived_table_map\n","        params[\"tables_to_include\"] = tables_to_include\n","\n","\n","        views_ddl = generate_lh_view_ddl(FABRIC_LH_DATABASE, params)\n","      \n","      \n","        logger.info(f\"🛠️ Step 5/7: Executing views DDL on Fabric DW {FABRIC_WH_DATABASE}.\")\n","\n","        view_ddls = views_ddl.split(';')\n","        view_count = len(view_ddls)\n","        logger.info(f\"{view_count} views to deploy.\")\n","        # Loop through each item\n","        counter= 1\n","        for view_ddl in view_ddls:\n","            match = re.search(r\"CREATE\\s+OR\\s+ALTER\\s+VIEW\\s+([\\w\\.]+)\", view_ddl, re.IGNORECASE)\n","            if match:\n","                view_name = match.group(1)\n","                logger.info(f\"{view_count}/{counter} CREATE OR ALTER VIEW {view_name}\")\n","                execute_ddl_on_warehouse(FABRIC_WH_DATABASE, view_ddl)\n","                counter += 1\n","\n","        if is_not_none_and_empty(SYNAPSE_SERVER) and is_not_none_and_empty(SYNAPSE_DW_DATABASE) :\n"," \n","            logger.info(\"🚀 Step 6/7: Fetch view dependencies.\")\n","            views_json = fetch_view_dependencies(\n","                synapse_dw_engine, SYNAPSE_DW_VIEWS,\n","                SYNAPSE_EDL_DATABASE, FABRIC_WH_DATABASE,\n","                SYNAPSE_DW_SCHEMA, FABRIC_WH_SCHEMA\n","            )\n","\n","            logger.info(\"🚀 Step 7/7: Deploy views to warehouse.\")\n","            deploy_views(FABRIC_WH_DATABASE, FABRIC_WH_SCHEMA, views_json)\n","        else:\n","            logger.info(\"🔹 Step 6 and 7: Skipped\")\n","\n","        logger.info(\"🎉 Deployment complete.\")\n","    else:\n","        logger.error(\"❌ Warehouse setup failed.\")\n","    \n","\n","if __name__ == \"__main__\":\n","    main()\n"],"outputs":[{"output_type":"display_data","data":{"application/vnd.jupyter.statement-meta+json":{"session_id":"801eb0aa-9a3a-4c30-b877-75c0402afbf2","normalized_state":"finished","queued_time":"2025-07-29T22:36:44.133805Z","session_start_time":null,"execution_start_time":"2025-07-29T22:36:44.4586243Z","execution_finish_time":"2025-07-29T22:37:29.0235544Z","parent_msg_id":"3785be1d-73c9-43c0-901e-5ad3b7920f19"}},"metadata":{}},{"output_type":"stream","name":"stdout","text":["[2025-07-29 22:36:44] INFO - 🏗️ Step 1: Download template files if does not exists.\n[2025-07-29 22:36:44] INFO - 📄 File already exists locally: ./builtin/resources/derived_table_map.json\n[2025-07-29 22:36:44] INFO - 📄 File already exists locally: ./builtin/resources/get_lh_ddl_as_view.sql\n[2025-07-29 22:36:44] INFO - 📄 File already exists locally: ./builtin/resources/get_view_dependency.sql\n[2025-07-29 22:36:44] INFO - 🏗️ Step 2/7: Ensure case-insensitive warehouse exists.\n[2025-07-29 22:36:45] INFO - ✅ Warehouse 'Sales_DW' already exists.\n[2025-07-29 22:36:45] INFO - 🔹 Step 3/7: Fetch tables and schema map from Synapse.\n[2025-07-29 22:36:45] INFO - 🔹 Step 3/7: Skipped\n[2025-07-29 22:36:45] INFO - 🔁 Step 4/7: Load derived table map.\n[2025-07-29 22:36:45] INFO - 🛠️ Step 5/7: Generating view DDLfrom {FABRIC_LH_DATABASE}...\n[2025-07-29 22:37:06] INFO - 🛠️ Step 5/7: Executing views DDL on Fabric DW Sales_DW.\n[2025-07-29 22:37:06] INFO - 65 views to deploy.\n[2025-07-29 22:37:06] INFO - 65/1 CREATE OR ALTER VIEW dbo.commissionsalesgroup\n[2025-07-29 22:37:06] INFO - 65/2 CREATE OR ALTER VIEW dbo.companyinfo\n[2025-07-29 22:37:06] INFO - 65/3 CREATE OR ALTER VIEW dbo.custgroup\n[2025-07-29 22:37:06] INFO - 65/4 CREATE OR ALTER VIEW dbo.custinvoicejour\n[2025-07-29 22:37:07] INFO - 65/5 CREATE OR ALTER VIEW dbo.custinvoicetrans\n[2025-07-29 22:37:08] INFO - 65/6 CREATE OR ALTER VIEW dbo.custpackingslipjour\n[2025-07-29 22:37:08] INFO - 65/7 CREATE OR ALTER VIEW dbo.custpackingsliptrans\n[2025-07-29 22:37:08] INFO - 65/8 CREATE OR ALTER VIEW dbo.custtable\n[2025-07-29 22:37:10] INFO - 65/9 CREATE OR ALTER VIEW dbo.custtrans\n[2025-07-29 22:37:10] INFO - 65/10 CREATE OR ALTER VIEW dbo.custtransopen\n[2025-07-29 22:37:10] INFO - 65/11 CREATE OR ALTER VIEW dbo.dimensionattribute\n[2025-07-29 22:37:11] INFO - 65/12 CREATE OR ALTER VIEW dbo.dimensionattributevalue\n[2025-07-29 22:37:11] INFO - 65/13 CREATE OR ALTER VIEW dbo.dimensionattributevaluesetitem\n[2025-07-29 22:37:11] INFO - 65/14 CREATE OR ALTER VIEW dbo.dirdunsnumber\n[2025-07-29 22:37:11] INFO - 65/15 CREATE OR ALTER VIEW dbo.dirnameaffix\n[2025-07-29 22:37:11] INFO - 65/16 CREATE OR ALTER VIEW dbo.dirpartylocation\n[2025-07-29 22:37:11] INFO - 65/17 CREATE OR ALTER VIEW dbo.dirpartytable\n[2025-07-29 22:37:11] INFO - 65/18 CREATE OR ALTER VIEW dbo.dirpersonname\n[2025-07-29 22:37:11] INFO - 65/19 CREATE OR ALTER VIEW dbo.ecoresproduct\n[2025-07-29 22:37:11] INFO - 65/20 CREATE OR ALTER VIEW dbo.ecoresproducttranslation\n[2025-07-29 22:37:11] INFO - 65/21 CREATE OR ALTER VIEW dbo.generaljournalaccountentry\n[2025-07-29 22:37:11] INFO - 65/22 CREATE OR ALTER VIEW dbo.generaljournalentry\n[2025-07-29 22:37:11] INFO - 65/23 CREATE OR ALTER VIEW dbo.hcmworker\n[2025-07-29 22:37:11] INFO - 65/24 CREATE OR ALTER VIEW dbo.inventdim\n[2025-07-29 22:37:11] INFO - 65/25 CREATE OR ALTER VIEW dbo.inventitemgroup\n[2025-07-29 22:37:11] INFO - 65/26 CREATE OR ALTER VIEW dbo.inventitemgroupitem\n[2025-07-29 22:37:11] INFO - 65/27 CREATE OR ALTER VIEW dbo.inventiteminventsetup\n[2025-07-29 22:37:11] INFO - 65/28 CREATE OR ALTER VIEW dbo.inventitemlocation\n[2025-07-29 22:37:12] INFO - 65/29 CREATE OR ALTER VIEW dbo.inventitemprice\n[2025-07-29 22:37:12] INFO - 65/30 CREATE OR ALTER VIEW dbo.inventitempurchsetup\n[2025-07-29 22:37:12] INFO - 65/31 CREATE OR ALTER VIEW dbo.inventitemsalessetup\n[2025-07-29 22:37:12] INFO - 65/32 CREATE OR ALTER VIEW dbo.inventitemsampling\n[2025-07-29 22:37:12] INFO - 65/33 CREATE OR ALTER VIEW dbo.inventjournaltable\n[2025-07-29 22:37:12] INFO - 65/34 CREATE OR ALTER VIEW dbo.inventjournaltrans\n[2025-07-29 22:37:12] INFO - 65/35 CREATE OR ALTER VIEW dbo.inventsettlement\n[2025-07-29 22:37:12] INFO - 65/36 CREATE OR ALTER VIEW dbo.inventsum\n[2025-07-29 22:37:12] INFO - 65/37 CREATE OR ALTER VIEW dbo.inventtable\n[2025-07-29 22:37:13] INFO - 65/38 CREATE OR ALTER VIEW dbo.inventtrans\n[2025-07-29 22:37:14] INFO - 65/39 CREATE OR ALTER VIEW dbo.inventtransorigin\n[2025-07-29 22:37:14] INFO - 65/40 CREATE OR ALTER VIEW dbo.inventtransposting\n[2025-07-29 22:37:14] INFO - 65/41 CREATE OR ALTER VIEW dbo.ledgerjournaltable\n[2025-07-29 22:37:14] INFO - 65/42 CREATE OR ALTER VIEW dbo.ledgerjournaltrans\n[2025-07-29 22:37:17] INFO - 65/43 CREATE OR ALTER VIEW dbo.logisticsaddresscountryregion\n[2025-07-29 22:37:17] INFO - 65/44 CREATE OR ALTER VIEW dbo.logisticsaddresscountryregiontranslation\n[2025-07-29 22:37:17] INFO - 65/45 CREATE OR ALTER VIEW dbo.logisticselectronicaddress\n[2025-07-29 22:37:17] INFO - 65/46 CREATE OR ALTER VIEW dbo.logisticslocation\n[2025-07-29 22:37:17] INFO - 65/47 CREATE OR ALTER VIEW dbo.logisticslocationext\n[2025-07-29 22:37:17] INFO - 65/48 CREATE OR ALTER VIEW dbo.logisticspostaladdress\n[2025-07-29 22:37:17] INFO - 65/49 CREATE OR ALTER VIEW dbo.omteammembershipcriterion\n[2025-07-29 22:37:17] INFO - 65/50 CREATE OR ALTER VIEW dbo.pdscatchweightitem\n[2025-07-29 22:37:17] INFO - 65/51 CREATE OR ALTER VIEW dbo.purchline\n[2025-07-29 22:37:19] INFO - 65/52 CREATE OR ALTER VIEW dbo.purchtable\n[2025-07-29 22:37:20] INFO - 65/53 CREATE OR ALTER VIEW dbo.reqtrans\n[2025-07-29 22:37:20] INFO - 65/54 CREATE OR ALTER VIEW dbo.salesline\n[2025-07-29 22:37:22] INFO - 65/55 CREATE OR ALTER VIEW dbo.salesline_w\n[2025-07-29 22:37:22] INFO - 65/56 CREATE OR ALTER VIEW dbo.salestable\n[2025-07-29 22:37:24] INFO - 65/57 CREATE OR ALTER VIEW dbo.systemparameters\n[2025-07-29 22:37:24] INFO - 65/58 CREATE OR ALTER VIEW dbo.taxtrans\n[2025-07-29 22:37:24] INFO - 65/59 CREATE OR ALTER VIEW dbo.vendgroup\n[2025-07-29 22:37:24] INFO - 65/60 CREATE OR ALTER VIEW dbo.vendtable\n[2025-07-29 22:37:26] INFO - 65/61 CREATE OR ALTER VIEW dbo.vendtrans\n[2025-07-29 22:37:26] INFO - 65/62 CREATE OR ALTER VIEW dbo.whsinventenabled\n[2025-07-29 22:37:26] INFO - 65/63 CREATE OR ALTER VIEW dbo.whsinventreserve\n[2025-07-29 22:37:26] INFO - 65/64 CREATE OR ALTER VIEW dbo.whsreservationhierarchyelement\n[2025-07-29 22:37:26] INFO - 65/65 CREATE OR ALTER VIEW dbo.dirpartytable\n[2025-07-29 22:37:27] INFO - 🔹 Step 6 and 7: Skipped\n[2025-07-29 22:37:27] INFO - 🎉 Deployment complete.\n"]}],"execution_count":4,"metadata":{"jupyter":{"outputs_hidden":false,"source_hidden":false},"microsoft":{"language":"python","language_group":"jupyter_python"},"nteract":{"transient":{"deleting":false}}},"id":"8c2d3a9a-f5c8-4078-a36f-b540e44e6bfd"}],"metadata":{"kernel_info":{"jupyter_kernel_name":"python3.11","name":"jupyter"},"kernelspec":{"name":"jupyter","display_name":"Jupyter"},"language_info":{"name":"python"},"microsoft":{"language":"python","language_group":"jupyter_python","ms_spell_check":{"ms_spell_check_language":"en"}},"nteract":{"version":"nteract-front-end@1.0.0"},"widgets":{},"synapse_widget":{"state":{},"version":"0.1"},"spark_compute":{"compute_id":"/trident/default","session_options":{"conf":{"spark.synapse.nbs.session.timeout":"1200000"}}},"dependencies":{"lakehouse":{"known_lakehouses":[{"id":"70459bf8-2d31-42b6-947f-9eea8f2e6c8c"}],"default_lakehouse":"70459bf8-2d31-42b6-947f-9eea8f2e6c8c","default_lakehouse_name":"dataverse_jjunoenvfre","default_lakehouse_workspace_id":"cca1475b-c4fe-417f-844a-f3e8a061e55d"},"warehouse":{"known_warehouses":[]}}},"nbformat":4,"nbformat_minor":5}
\ No newline at end of file