Add workspace maintenance and cleanup blog

2026-04-workspace-maintenance-cleanup/README.md

@@ -0,0 +1,49 @@
+# Databricks Workspace Cleanup
+
+Automated workspace maintenance using Databricks Asset Bundles — clean up unused jobs, dashboards, vector search indexes, and clusters with full audit logging and Lakeview dashboards.
+
+## Two Approaches
+
+1. **API-Driven Cleanup** — REST APIs scan and remove unused resources by metadata (last run date, status, source table existence)
+2. **System Table-Driven Cleanup** — Query `system.billing.usage`, `system.compute.clusters`, `system.lakeflow.job_run_timeline`, and `system.query.history` to find resources costing money but delivering no value
+
+Both approaches log every action (deleted, skipped, flagged) to a Delta table for auditability.
+
+## Structure
+
+```
+├── databricks.yml              # DAB bundle definition
+├── config/
+│   ├── config.yaml             # Cleanup toggles per environment
+│   └── thresholds.yaml         # Retention thresholds
+├── notebooks/
+│   ├── 00_cleanup_logger.py    # Structured logging module
+│   ├── 01_api_job_cleanup.py   # API: delete inactive jobs
+│   ├── 02_api_dashboard_cleanup.py  # API: remove stale dashboards
+│   ├── 03_api_vector_cleanup.py     # API: purge orphaned indexes
+│   ├── 04_system_table_analysis.py  # System tables: discover waste
+│   └── 05_system_table_cleanup.py   # System tables: act on flagged items
+└── dashboards/
+    └── cleanup_dashboard.sql   # Lakeview dashboard queries
+```
+
+## Quick Start
+
+```bash
+# Deploy to dev (dry run)
+databricks bundle deploy --target dev
+databricks bundle run cleanup_workflow --target dev
+
+# Review flagged items in the Lakeview dashboard, then:
+databricks bundle deploy --target prod
+```
+
+## Configuration
+
+Edit `config/config.yaml` to toggle cleanups per environment. Edit `config/thresholds.yaml` to set retention periods.
+
+Dev always runs in dry-run mode. Production executes deletions on a weekly Sunday 2 AM schedule.
+
+## Blog Post
+
+[Databricks Maintenance and Cleanup: Visualise, Clean, and Log with Asset Bundles](https://community.databricks.com/t5/technical-blog/databricks-maintenance-and-cleanup-visualise-clean-and-log-with/ba-p/135657)

2026-04-workspace-maintenance-cleanup/config/config.yaml

@@ -0,0 +1,20 @@
+dev:
+  job_cleanup: false
+  dashboard_cleanup: false
+  vector_cleanup: true
+  system_table_cleanup: true
+  dry_run: true
+
+stage:
+  job_cleanup: true
+  dashboard_cleanup: true
+  vector_cleanup: true
+  system_table_cleanup: true
+  dry_run: false
+
+prod:
+  job_cleanup: true
+  dashboard_cleanup: true
+  vector_cleanup: true
+  system_table_cleanup: true
+  dry_run: false

2026-04-workspace-maintenance-cleanup/config/thresholds.yaml

@@ -0,0 +1,5 @@
+job_inactive_days: 90
+dashboard_inactive_days: 60
+cluster_idle_hours: 48
+cost_anomaly_multiplier: 3.0
+min_dbu_threshold: 0.01

2026-04-workspace-maintenance-cleanup/dashboards/cleanup_dashboard.sql

@@ -0,0 +1,44 @@
+-- ============================================================
+-- Lakeview Dashboard Queries for Workspace Cleanup
+-- ============================================================
+
+-- 1. Cleanup Summary by Type and Action
+SELECT resource_type, action, COUNT(*) AS count, environment
+FROM maintenance.cleanup.cleanup_log
+WHERE timestamp >= DATEADD(DAY, -30, CURRENT_DATE())
+GROUP BY resource_type, action, environment
+ORDER BY count DESC;
+
+-- 2. Daily Cleanup Trend
+SELECT DATE(timestamp) AS cleanup_date, action, COUNT(*) AS count
+FROM maintenance.cleanup.cleanup_log
+WHERE timestamp >= DATEADD(DAY, -30, CURRENT_DATE())
+GROUP BY DATE(timestamp), action
+ORDER BY cleanup_date;
+
+-- 3. Top Wasted Resources by Cost
+SELECT resource_type, resource_name, reason,
+    get_json_object(details, '$.cost') AS estimated_cost,
+    owner, timestamp
+FROM maintenance.cleanup.cleanup_log
+WHERE action = 'FLAGGED'
+    AND timestamp >= DATEADD(DAY, -7, CURRENT_DATE())
+ORDER BY CAST(get_json_object(details, '$.cost') AS DOUBLE) DESC
+LIMIT 20;
+
+-- 4. Environment-Wise Retention
+SELECT environment, resource_type,
+    SUM(CASE WHEN action = 'DELETED' THEN 1 ELSE 0 END) AS deleted,
+    SUM(CASE WHEN action = 'SKIPPED' THEN 1 ELSE 0 END) AS retained,
+    SUM(CASE WHEN action = 'FLAGGED' THEN 1 ELSE 0 END) AS flagged,
+    SUM(CASE WHEN action = 'DRY_RUN' THEN 1 ELSE 0 END) AS dry_run
+FROM maintenance.cleanup.cleanup_log
+WHERE timestamp >= DATEADD(DAY, -30, CURRENT_DATE())
+GROUP BY environment, resource_type;
+
+-- 5. Full Audit Trail
+SELECT timestamp, environment, resource_type, resource_id,
+    resource_name, owner, action, reason, dry_run, details
+FROM maintenance.cleanup.cleanup_log
+ORDER BY timestamp DESC
+LIMIT 100;

2026-04-workspace-maintenance-cleanup/databricks.yml

@@ -0,0 +1,72 @@
+bundle:
+  name: workspace-cleanup
+
+workspace:
+  host: https://<your-workspace>.azuredatabricks.net
+
+variables:
+  environment:
+    default: dev
+
+resources:
+  jobs:
+    cleanup_workflow:
+      name: "[Maintenance] Workspace Cleanup"
+      schedule:
+        quartz_cron_expression: "0 0 2 ? * SUN"
+        timezone_id: "UTC"
+      parameters:
+        - name: environment
+          default: ${var.environment}
+      tasks:
+        - task_key: system_table_analysis
+          notebook_task:
+            notebook_path: ./notebooks/04_system_table_analysis.py
+            base_parameters:
+              environment: ${var.environment}
+
+        - task_key: job_cleanup
+          depends_on:
+            - task_key: system_table_analysis
+          notebook_task:
+            notebook_path: ./notebooks/01_api_job_cleanup.py
+            base_parameters:
+              environment: ${var.environment}
+
+        - task_key: dashboard_cleanup
+          depends_on:
+            - task_key: system_table_analysis
+          notebook_task:
+            notebook_path: ./notebooks/02_api_dashboard_cleanup.py
+            base_parameters:
+              environment: ${var.environment}
+
+        - task_key: vector_cleanup
+          depends_on:
+            - task_key: system_table_analysis
+          notebook_task:
+            notebook_path: ./notebooks/03_api_vector_cleanup.py
+            base_parameters:
+              environment: ${var.environment}
+
+        - task_key: system_table_cleanup
+          depends_on:
+            - task_key: job_cleanup
+            - task_key: dashboard_cleanup
+            - task_key: vector_cleanup
+          notebook_task:
+            notebook_path: ./notebooks/05_system_table_cleanup.py
+            base_parameters:
+              environment: ${var.environment}
+
+targets:
+  dev:
+    default: true
+    variables:
+      environment: dev
+  stage:
+    variables:
+      environment: stage
+  prod:
+    variables:
+      environment: prod

2026-04-workspace-maintenance-cleanup/notebooks/00_cleanup_logger.py

@@ -0,0 +1,56 @@
+# Databricks notebook source
+# Structured logging for all cleanup operations — writes to Delta table
+
+import json
+from datetime import datetime
+
+
+class CleanupLogger:
+    """Log every cleanup action to a Delta table for auditability."""
+
+    def __init__(self, spark, catalog="finops", schema="cleanup"):
+        self.spark = spark
+        self.table = f"{catalog}.{schema}.cleanup_log"
+        self.entries = []
+        self._ensure_table(catalog, schema)
+
+    def _ensure_table(self, catalog, schema):
+        self.spark.sql(f"CREATE SCHEMA IF NOT EXISTS {catalog}.{schema}")
+        self.spark.sql(f"""
+            CREATE TABLE IF NOT EXISTS {self.table} (
+                timestamp TIMESTAMP,
+                environment STRING,
+                resource_type STRING,
+                resource_id STRING,
+                resource_name STRING,
+                owner STRING,
+                action STRING,
+                reason STRING,
+                dry_run BOOLEAN,
+                details STRING
+            )
+        """)
+
+    def log(self, environment, resource_type, resource_id, resource_name,
+            owner, action, reason, dry_run=False, details=None):
+        self.entries.append({
+            "timestamp": datetime.utcnow(),
+            "environment": environment,
+            "resource_type": resource_type,
+            "resource_id": str(resource_id),
+            "resource_name": resource_name,
+            "owner": owner or "unknown",
+            "action": action,
+            "reason": reason,
+            "dry_run": dry_run,
+            "details": json.dumps(details) if details else None
+        })
+
+    def flush(self):
+        if not self.entries:
+            return 0
+        df = self.spark.createDataFrame(self.entries)
+        df.write.mode("append").saveAsTable(self.table)
+        count = len(self.entries)
+        self.entries = []
+        return count

2026-04-workspace-maintenance-cleanup/notebooks/01_api_job_cleanup.py

@@ -0,0 +1,110 @@
+# Databricks notebook source
+# API-Driven Job Cleanup — deletes jobs inactive beyond threshold
+
+import requests
+import yaml
+from datetime import datetime, timedelta
+
+# COMMAND ----------
+
+dbutils.widgets.text("environment", "dev")
+env = dbutils.widgets.get("environment")
+
+with open("/Workspace/config/config.yaml") as f:
+    config = yaml.safe_load(f)[env]
+with open("/Workspace/config/thresholds.yaml") as f:
+    thresholds = yaml.safe_load(f)
+
+if not config.get("job_cleanup", False):
+    dbutils.notebook.exit(f"Job cleanup disabled for {env}")
+
+dry_run = config.get("dry_run", True)
+inactive_days = thresholds.get("job_inactive_days", 90)
+cutoff = datetime.utcnow() - timedelta(days=inactive_days)
+
+# COMMAND ----------
+
+# Setup
+%run ./00_cleanup_logger
+
+host = spark.conf.get("spark.databricks.workspaceUrl")
+token = (dbutils.notebook.entry_point.getDbutils()
+         .notebook().getContext().apiToken().get())
+headers = {"Authorization": f"Bearer {token}"}
+logger = CleanupLogger(spark)
+
+# COMMAND ----------
+
+# List all jobs
+has_more = True
+offset = 0
+jobs = []
+
+while has_more:
+    resp = requests.get(
+        f"https://{host}/api/2.1/jobs/list",
+        headers=headers,
+        params={"limit": 25, "offset": offset, "expand_tasks": False}
+    )
+    data = resp.json()
+    jobs.extend(data.get("jobs", []))
+    has_more = data.get("has_more", False)
+    offset += 25
+
+print(f"Found {len(jobs)} jobs in workspace")
+
+# COMMAND ----------
+
+deleted, skipped = 0, 0
+
+for job in jobs:
+    job_id = job["job_id"]
+    job_name = job.get("settings", {}).get("name", "unnamed")
+    creator = job.get("creator_user_name", "unknown")
+
+    # Get last run
+    runs_resp = requests.get(
+        f"https://{host}/api/2.1/jobs/runs/list",
+        headers=headers,
+        params={"job_id": job_id, "limit": 1}
+    )
+    runs = runs_resp.json().get("runs", [])
+
+    if runs:
+        last_run = datetime.utcfromtimestamp(
+            runs[0].get("start_time", 0) / 1000
+        )
+    else:
+        last_run = datetime(2020, 1, 1)
+
+    days_idle = (datetime.utcnow() - last_run).days
+
+    if last_run < cutoff:
+        if not dry_run:
+            requests.post(
+                f"https://{host}/api/2.1/jobs/delete",
+                headers=headers, json={"job_id": job_id}
+            )
+
+        logger.log(
+            environment=env, resource_type="job",
+            resource_id=job_id, resource_name=job_name, owner=creator,
+            action="DELETED" if not dry_run else "FLAGGED",
+            reason=f"Inactive {days_idle} days (threshold: {inactive_days})",
+            dry_run=dry_run,
+            details={"last_run": str(last_run), "total_runs": len(runs)}
+        )
+        deleted += 1
+    else:
+        logger.log(
+            environment=env, resource_type="job",
+            resource_id=job_id, resource_name=job_name, owner=creator,
+            action="SKIPPED",
+            reason=f"Active — last run {days_idle} days ago",
+            dry_run=dry_run
+        )
+        skipped += 1
+
+flushed = logger.flush()
+print(f"Jobs — {'[DRY RUN] ' if dry_run else ''}Deleted: {deleted}, "
+      f"Skipped: {skipped}, Logged: {flushed}")