From 727dd86cd8645e85007acbde3e37666424c1aa76 Mon Sep 17 00:00:00 2001 From: Mridul Bhatt <46847918+embiway@users.noreply.github.com> Date: Tue, 23 Jun 2026 15:34:08 +0530 Subject: [PATCH 1/3] Added a multiplier to adjust IPA ETAs (#82) --- estimators/estimator.py | 3 ++- ui/exchange_online_ui.py | 1 + util/constants.py | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/estimators/estimator.py b/estimators/estimator.py index db16ca10..096a254f 100644 --- a/estimators/estimator.py +++ b/estimators/estimator.py @@ -26,6 +26,7 @@ def calculate_migration_eta(self, data: Dict[str, Any]) -> float: global_limit = data.get("global_limit", 100) batch_time = data.get("batch_time", 1) user_limit = data.get("user_limit", 1) + multiplier = data.get("multiplier", 1) active_counts = [c for c in item_counts if c > 0] if not active_counts: @@ -50,7 +51,7 @@ def calculate_migration_eta(self, data: Dict[str, Any]) -> float: total_seconds += seconds_for_layer previous_level = current_level - return total_seconds / 3600.0 + return multiplier * (total_seconds / 3600.0) def get_resource_type(self) -> str: raise NotImplementedError("Subclasses must implement the get_resource_type method") diff --git a/ui/exchange_online_ui.py b/ui/exchange_online_ui.py index 59f22716..1b8c7caa 100644 --- a/ui/exchange_online_ui.py +++ b/ui/exchange_online_ui.py @@ -2648,6 +2648,7 @@ def get_batch_eta(subset_df): "user_limit": ETA_EMAIL_USER_LIMIT, "batch_size": ETA_EMAIL_BATCH_SIZE, "batch_time": ETA_EMAIL_BATCH_TIME, + "multiplier": IPA_ETA_MULTIPLIER } ) eta_shared_mail_box = 0.0 diff --git a/util/constants.py b/util/constants.py index 6281f95e..ae968d39 100644 --- a/util/constants.py +++ b/util/constants.py @@ -73,6 +73,8 @@ ETA_EMAIL_BATCH_SIZE = 1 ETA_EMAIL_BATCH_TIME = 6 +IPA_ETA_MULTIPLIER = 1.5 + FILES_GLOBAL_COUNT_LIMIT = 4 # 4 files/folders per second FILES_GLOBAL_CORPUS_SIZE_LIMIT = (400 * 1024 * 1024 * 1024) // 3600 # 400 GB per hour in bytes per second From 3c71f18fe2950c671965f63833124f7e382d757f Mon Sep 17 00:00:00 2001 From: Mridul Bhatt <46847918+embiway@users.noreply.github.com> Date: Thu, 25 Jun 2026 12:43:30 +0530 Subject: [PATCH 2/3] Added support for marking DL and subsites as large resrources (#83) --- estimators/file_estimator.py | 65 ++++++++++++++++++++++++++----- tests/files/data_state_creator.py | 52 ++++++++++++++++++++++++- tests/files/load_tests.py | 24 ++++++++++-- ui/files_ui.py | 37 ++++++++++-------- util/enums.py | 7 ++-- 5 files changed, 151 insertions(+), 34 deletions(-) diff --git a/estimators/file_estimator.py b/estimators/file_estimator.py index 99da825e..30ebeb76 100644 --- a/estimators/file_estimator.py +++ b/estimators/file_estimator.py @@ -159,6 +159,8 @@ def calculate_resource_metrics( url_to_site_id = self._get_sites_from_urls(data["siteUrls"], site_discovery_progress_metrics, failures) for url, site_id in url_to_site_id.items(): top_level_sites.append(site_id) + self.site_to_metadata[site_id] = {"isPersonalSite": False} + metrics["teamSiteCount"] += 1 site_id_to_url = {site_id: url for url, site_id in url_to_site_id.items()} @@ -166,7 +168,7 @@ def calculate_resource_metrics( metrics["siteCount"] = len(top_level_sites) all_sites = [{"siteId": site_id, "siteLevel": 0} for site_id in top_level_sites] - self._get_subsites_in_site(top_level_sites, all_sites, subsite_to_top_level_site, site_discovery_progress_metrics, failures, 1) + self._get_subsites_in_site(top_level_sites, all_sites, subsite_to_top_level_site, site_discovery_progress_metrics, failures, metrics, 1) if not has_emails and not has_urls: metrics["personalSiteCount"] = site_discovery_progress_metrics.get("personalSiteCount", 0) @@ -358,8 +360,7 @@ def _update_tenant_metrics_from_drive_metrics( metrics["maxSubsiteDepth"] = max(metrics["maxSubsiteDepth"], metrics["siteMetrics"][subsite_id]["siteLevel"]) top_level_site = subsite_to_top_level_site.get(subsite_id, subsite_id) - if self.id_to_display.get(subsite_id, "") == "https://smh3v.sharepoint.com/subsiteofrootsite": - print(f"FOUND the URL: {subsite_to_top_level_site.get(subsite_id, "")}") + subsite_item_count = 0 # Used to track if this subsite is a Large Resource if top_level_site != subsite_id: metrics["siteMetrics"][top_level_site]["subsiteCount"] = metrics["siteMetrics"][top_level_site].get("subsiteCount", 0) + 1 @@ -368,7 +369,20 @@ def _update_tenant_metrics_from_drive_metrics( if drive_id in metrics["driveMetrics"]: drive_metric = metrics["driveMetrics"][drive_id] + subsite_item_count += drive_metric.get("fileCount", 0) + drive_metric.get("folderCount", 0) metrics["siteMetrics"][top_level_site]["largeResourceCount"] = metrics["siteMetrics"].get(top_level_site, {}).get("largeResourceCount", 0) + len(drive_metric.get("largeResources", [])) + if drive_metric.get("fileCount", 0) + drive_metric.get("folderCount", 0) > self.config.large_resource_count_limit: + metrics["siteMetrics"][top_level_site]["largeResourceCount"] += 1 + metrics["tenantLevelLargeResources"].append( + { + "type": ResourceType.DL.value, + "id": drive_id, + "subTreeCount": drive_metric.get("fileCount", 0) + drive_metric.get("folderCount", 0), + "parent": subsite_id, # Explicitly showing subsite id here as users can use it to determine site collection easily (webUrl will be displayed in final report). + "Limit": self.config.large_resource_count_limit + } + ) + metrics["siteMetrics"][top_level_site]["folderCount"] = metrics["siteMetrics"].get(top_level_site, {}).get("folderCount", 0) + drive_metric.get("folderCount", 0) metrics["siteMetrics"][top_level_site]["fileCount"] = metrics["siteMetrics"].get(top_level_site, {}).get("fileCount", 0) + drive_metric.get("fileCount", 0) metrics["siteMetrics"][top_level_site]["shortcutCount"] = metrics["siteMetrics"].get(top_level_site, {}).get("shortcutCount", 0) + drive_metric.get("shortcutCount", 0) @@ -376,7 +390,19 @@ def _update_tenant_metrics_from_drive_metrics( metrics["siteMetrics"][top_level_site]["folderCountExceedingDepthLimit"] = metrics["siteMetrics"].get(top_level_site, {}).get("folderCountExceedingDepthLimit", 0) + drive_metric.get("folderCountExceedingDepthLimit", 0) metrics["siteMetrics"][top_level_site]["fileCountExceedingDepthLimit"] = metrics["siteMetrics"].get(top_level_site, {}).get("fileCountExceedingDepthLimit", 0) + drive_metric.get("fileCountExceedingDepthLimit", 0) - + # Check if this subsite is a Large Resource + if subsite_item_count > self.config.large_resource_count_limit and subsite_id != top_level_site: + metrics["siteMetrics"][top_level_site]["largeResourceCount"] = metrics["siteMetrics"][top_level_site].get("largeResourceCount", 0) + 1 + metrics["tenantLevelLargeResources"].append( + { + "type": ResourceType.SUBSITE.value, + "id": subsite_id, + "subTreeCount": subsite_item_count, + "parent": top_level_site, + "Limit": self.config.large_resource_count_limit + } + ) + metrics["siteMetrics"][top_level_site]["dlCount"] = metrics["siteMetrics"].get(top_level_site, {}).get("dlCount", 0) + len(drive_ids) if self._is_subsite_personal(subsite_id): @@ -386,6 +412,18 @@ def _update_tenant_metrics_from_drive_metrics( if top_level_site != subsite_id: metrics["subsiteCount"] += 1 + + for site_id, metric in metrics["siteMetrics"].items(): + if metric.get("folderCount", 0) + metric.get("fileCount", 0) > self.config.large_resource_count_limit: + metrics["tenantLevelLargeResources"].append( + { + "type": ResourceType.SITE.value, + "id": site_id, + "subTreeCount": metric.get("fileCount", 0) + metric.get("folderCount", 0), + "parent": "N/A (Top level site)", + "Limit": self.config.large_resource_count_limit + } + ) for siteId in subsite_to_drives.keys(): top_level_site = subsite_to_top_level_site.get(siteId, siteId) @@ -419,7 +457,7 @@ def _update_tenant_metrics_from_drive_metrics( for drive_id, metric in metrics["driveMetrics"].items(): for large_resource in metric["largeResources"]: curr_dict = large_resource - curr_dict["drive"] = drive_id + curr_dict["parent"] = drive_id metrics["tenantLevelLargeResources"].append(curr_dict) metrics["tenantLevelLargeResourceCount"] = len(metrics["tenantLevelLargeResources"]) @@ -438,10 +476,11 @@ def _get_subsites_in_site( subsite_to_top_level_site: Dict[str, str], site_discovery_progress_metrics: Dict[str, Any], failures: List[Dict[str, str]], + tenant_metrics: Dict[str, Any], level: int = 1 ): try: - site_url = "/sites/{siteId}/sites?$select=id,weburl,isPersonalSite&$top=999" + site_url = "/sites/{siteId}/sites?$select=id,webUrl,isPersonalSite&$top=999" batches = create_batches(site_url, [{"siteId": site_id} for site_id in site_ids], self.config.parallel_batches, True) futures_map: Dict[int, Future[List[Dict[str, Any]]]] = {} @@ -542,9 +581,15 @@ def local_progress_callback(responses: List, has_next=False): self.site_to_metadata[site["id"]] = { "isPersonalSite": site.get("isPersonalSite", False) } + self.id_to_display[site["id"]] = site.get("webUrl", site["id"]) + + if site.get("isPersonalSite", False): + tenant_metrics["personalSiteCount"] += 1 + else: + tenant_metrics["teamSiteCount"] += 1 if new_sub_site_ids: - self._get_subsites_in_site(new_sub_site_ids, all_sites, subsite_to_top_level_site, site_discovery_progress_metrics, failures, level + 1) + self._get_subsites_in_site(new_sub_site_ids, all_sites, subsite_to_top_level_site, site_discovery_progress_metrics, failures, tenant_metrics, level + 1) except Exception as e: self._log_and_fail("Error in _get_subsites_in_site", e, failures) @@ -1030,7 +1075,7 @@ def _create_in_memory_tree( parent_references[drive_id] = {} try: # use delta api to fetch the folders - delta_api = "/drives/{driveId}/root/delta?$select=id,parentReference,name,folder,file,remoteItem,size" + delta_api = "/drives/{driveId}/root/delta?$select=id,parentReference,name,webUrl,folder,file,remoteItem,size" batches = create_batches(delta_api, [{"driveId": drive_id} for drive_id in drive_ids], self.config.parallel_batches, True) futures_map: Dict[int, Future[List[Dict[str, Any]]]] = {} @@ -1146,7 +1191,7 @@ def local_progress_callback(responses: List, has_next=False): if "body" in resp and "value" in resp["body"]: for file in resp["body"]["value"]: resource_id_to_details[file["id"]] = file - + self.id_to_display[file["id"]] = file.get("webUrl", file["name"]) if "parentReference" in file and "id" in file["parentReference"]: parent_references[drive_id][file["id"]] = file["parentReference"]["id"] @@ -1430,7 +1475,7 @@ def _update_drive_metrics_from_resource( if resource_metric["subTreeCount"] >= self.config.large_resource_count_limit: drive_metric["largeResources"].append({ "type": ResourceType.FOLDER.value if "folder" in resource else ResourceType.FILE.value, - "id": resource["name"], + "id": resource["id"], "subTreeCount": resource_metric["subTreeCount"], "Limit": self.config.large_resource_count_limit }) diff --git a/tests/files/data_state_creator.py b/tests/files/data_state_creator.py index 874bd111..0b2a8241 100644 --- a/tests/files/data_state_creator.py +++ b/tests/files/data_state_creator.py @@ -422,10 +422,10 @@ def calculate_expected(ignore_failures=False): if f_metrics and f_metrics["subTreeCount"] >= 50: # Using limit 50 drive_metrics["largeResources"].append({ "type": "FOLDER", - "id": sub_item["name"], + "id": sub_item["id"], "subTreeCount": f_metrics["subTreeCount"], "Limit": 50, - "drive": drive_id + "parent": drive_id }) expected["tenantLevelLargeResources"].append(drive_metrics["largeResources"][-1]) @@ -467,9 +467,57 @@ def calculate_expected(ignore_failures=False): expected["siteMetrics"][root_site_id]["fileCountExceedingDepthLimit"] += drive_metric["fileCountExceedingDepthLimit"] expected["siteMetrics"][root_site_id]["totalSize"] += drive_metric["totalSize"] + + # Second pass: compute DL, Subsite, and Site Collection large resources + for site_id, site in data["sites"].items(): + curr_site = site + while "parentReference" in curr_site and "siteId" in curr_site["parentReference"]: + parent_id = curr_site["parentReference"]["siteId"] + curr_site = data["sites"][parent_id] + root_site_id = curr_site["id"] + + subsite_item_count = 0 + for drive_id in site["drives"]: + if drive_id in expected["driveMetrics"]: + drive_metric = expected["driveMetrics"][drive_id] + drive_item_count = drive_metric["folderCount"] + drive_metric["fileCount"] + subsite_item_count += drive_item_count + + # Check if DL is a Large Resource + if drive_item_count > 50: + expected["siteMetrics"][root_site_id]["largeResourceCount"] += 1 + expected["tenantLevelLargeResources"].append({ + "type": "DOCUMENT LIBRARY", + "id": drive_id, + "subTreeCount": drive_item_count, + "parent": site_id, + "Limit": 50 + }) + + # Check if Subsite is a Large Resource + if site["siteLevel"] > 0 and subsite_item_count > 50: + expected["siteMetrics"][root_site_id]["largeResourceCount"] += 1 + expected["tenantLevelLargeResources"].append({ + "type": "SUBSITE", + "id": site_id, + "subTreeCount": subsite_item_count, + "parent": root_site_id, + "Limit": 50 + }) for root_site_id, s_metrics in expected["siteMetrics"].items(): s_metrics["resourceCount"] = s_metrics["folderCount"] + s_metrics["fileCount"] + s_metrics["shortcutCount"] + + # Check if Site Collection is a Large Resource + total_site_count = s_metrics["folderCount"] + s_metrics["fileCount"] + if total_site_count > 50: + expected["tenantLevelLargeResources"].append({ + "type": "SITE COLLECTION", + "id": root_site_id, + "subTreeCount": total_site_count, + "parent": "N/A (Top level site)", + "Limit": 50 + }) for site_id, site in data["sites"].items(): is_personal = site.get("isPersonalSite", False) diff --git a/tests/files/load_tests.py b/tests/files/load_tests.py index 901ab0ea..a011abc5 100644 --- a/tests/files/load_tests.py +++ b/tests/files/load_tests.py @@ -139,7 +139,11 @@ def test_load_simulation_all_sites(self): self.assertEqual(r_site.get("shortcutCount", 0), e_site.get("shortcutCount", 0)) self.assertEqual(r_site.get("folderCountExceedingDepthLimit", 0), e_site.get("folderCountExceedingDepthLimit", 0)) self.assertEqual(r_site.get("fileCountExceedingDepthLimit", 0), e_site.get("fileCountExceedingDepthLimit", 0)) - self.assertEqual(r_site.get("largeResourceCount", 0), e_site.get("largeResourceCount", 0)) + try: + self.assertEqual(r_site.get("largeResourceCount", 0), e_site.get("largeResourceCount", 0)) + except AssertionError as e: + print(f"\nDISCREPANCY for site {site_id}: result={r_site} | expected={e_site}\n") + raise e self.assertEqual(r_site.get("totalSize", 0), e_site.get("totalSize", 0)) self.assertEqual(r_site.get("resourceCount", 0), e_site.get("resourceCount", 0)) @@ -151,7 +155,8 @@ def test_load_simulation_all_sites(self): self.assertEqual(sum(s.get("shortcutCount", 0) for s in site_metrics_values), result.get("shortcutCount", 0)) self.assertEqual(sum(s.get("folderCountExceedingDepthLimit", 0) for s in site_metrics_values), result.get("folderCountExceedingDepthLimit", 0)) self.assertEqual(sum(s.get("fileCountExceedingDepthLimit", 0) for s in site_metrics_values), result.get("fileCountExceedingDepthLimit", 0)) - self.assertEqual(sum(s.get("largeResourceCount", 0) for s in site_metrics_values), result.get("tenantLevelLargeResourceCount", 0)) + site_collection_large_res_count = sum(1 for res in result.get("tenantLevelLargeResources", []) if res.get("type") == "SITE COLLECTION") + self.assertEqual(sum(s.get("largeResourceCount", 0) for s in site_metrics_values) + site_collection_large_res_count, result.get("tenantLevelLargeResourceCount", 0)) self.assertEqual(sum(s.get("dlCount", 0) for s in site_metrics_values), sum(result.get("driveCounts", {}).values())) def _get_expected_for_subset(self, email_ids: List[str]) -> Tuple[Dict[str, Any], List[str]]: @@ -205,7 +210,7 @@ def collect_subsites(site_id, level): "siteCount": len(root_site_ids), "subsiteCount": len(all_site_ids) - len(root_site_ids), "personalSiteCount": len(email_ids), - "teamSiteCount": 0, + "teamSiteCount": len(all_site_ids) - len(root_site_ids), "personalSiteDLCount": personal_site_dl_count, "teamSiteDLCount": team_site_dl_count, "listCount": sum(len(self.test_data.get("sites", {}).get(sid, {}).get("lists", [])) for sid in all_site_ids), @@ -245,6 +250,19 @@ def collect_subsites(site_id, level): r_bucket = next(b for b in expected["tenantLevelFileSizeDistribution"]["buckets"] if b["sizeRange"] == tuple(bucket["sizeRange"])) r_bucket["count"] += bucket["count"] + # Count other large resources (DLs, Subsites, Site Collections) + all_large_resources = self.test_data.get(expected_key, {}).get("tenantLevelLargeResources", []) + for lr in all_large_resources: + lr_type = lr.get("type") + lr_id = lr.get("id") + + if lr_type == "DOCUMENT LIBRARY" and lr_id in scanned_drives: + expected["tenantLevelLargeResourceCount"] += 1 + elif lr_type == "SUBSITE" and lr_id in all_site_ids and lr_id not in root_site_ids: + expected["tenantLevelLargeResourceCount"] += 1 + elif lr_type == "SITE COLLECTION" and lr_id in root_site_ids: + expected["tenantLevelLargeResourceCount"] += 1 + return expected, subset_drive_ids def test_load_simulation_from_csv(self): diff --git a/ui/files_ui.py b/ui/files_ui.py index d631ccc5..586a3020 100644 --- a/ui/files_ui.py +++ b/ui/files_ui.py @@ -362,7 +362,7 @@ def _try_get_metrics_from_csv_report(self, config): "Shortcut Count", "Folder Count > Depth Limit 100", "File Count > Depth Limit 100", - "Folder with > 500k item count", + "Entities with > 500k item count", "Corpus Size", } id_col = "Site URL/Name" if "Site URL/Name" in df.columns else ("Site Id" if "Site Id" in df.columns else ("Entity" if "Entity" in df.columns else None)) @@ -408,7 +408,7 @@ def _parse_size_str(val): "shortcutCount": shortcut_cnt, "folderCountExceedingDepthLimit": int(pd.to_numeric(row.get("Folder Count > Depth Limit 100", 0), errors="coerce") or 0), "fileCountExceedingDepthLimit": int(pd.to_numeric(row.get("File Count > Depth Limit 100", 0), errors="coerce") or 0), - "largeResourceCount": int(pd.to_numeric(row.get("Folder with > 500k item count", 0), errors="coerce") or 0), + "largeResourceCount": int(pd.to_numeric(row.get("Entities with > 500k item count", 0), errors="coerce") or 0), "totalSize": _parse_size_str(row.get("Corpus Size", 0)), "resourceCount": res_cnt, } @@ -430,7 +430,7 @@ def _parse_size_str(val): "listCount": int(pd.to_numeric(df.get("List Count", pd.Series([0])), errors="coerce").fillna(0).sum()), "folderCountExceedingDepthLimit": int(pd.to_numeric(df.get("Folder Count > Depth Limit 100", pd.Series([0])), errors="coerce").fillna(0).sum()), "fileCountExceedingDepthLimit": int(pd.to_numeric(df.get("File Count > Depth Limit 100", pd.Series([0])), errors="coerce").fillna(0).sum()), - "tenantLevelLargeResourceCount": int(pd.to_numeric(df.get("Folder with > 500k item count", pd.Series([0])), errors="coerce").fillna(0).sum()), + "tenantLevelLargeResourceCount": int(pd.to_numeric(df.get("Entities with > 500k item count", pd.Series([0])), errors="coerce").fillna(0).sum()), "siteClassification": {site_id: "personal" for site_id in site_metrics.keys()}, "licenseMetrics": {}, "tenantLevelFileSizeDistribution": {}, @@ -553,7 +553,7 @@ def execute_migration_scan(self, config): "Shortcut Count": s_data.get("shortcutCount", 0), "Folder Count > Depth Limit 100": s_data.get("folderCountExceedingDepthLimit", 0), "File Count > Depth Limit 100": s_data.get("fileCountExceedingDepthLimit", 0), - "Folder with > 500k item count": s_data.get("largeResourceCount", 0), + "Entities with > 500k item count": s_data.get("largeResourceCount", 0), "Corpus Size": s_data.get("totalSize", 0), "Resource Count": s_data.get("resourceCount", 0) }) @@ -959,7 +959,7 @@ def show_results_content(self, data): self.create_stat_card(card_frame, "List Count", f"{data.get('listCount', 0):,}", "🗃️") self.create_stat_card(card_frame, "Folder count beyond depth limit 100", f"{data.get('folderCountExceedingDepthLimit', 0):,}", "📁") self.create_stat_card(card_frame, "File count beyond depth limit 100", f"{data.get('fileCountExceedingDepthLimit', 0):,}", "📄") - self.create_stat_card(card_frame, "Large Resource Count (Folders with >500k items)", f"{data.get('tenantLevelLargeResourceCount', 0):,}", "📄") + self.create_stat_card(card_frame, "Large Resource Count (Entities with >500k items)", f"{data.get('tenantLevelLargeResourceCount', 0):,}", "📄") if self.show_eta: # Timeline @@ -1177,7 +1177,7 @@ def _validate_csv(self): "Shortcut Count", "Folder Count > Depth Limit 100", "File Count > Depth Limit 100", - "Folder with > 500k item count", + "Entities with > 500k item count", "Corpus Size", } is_report_csv = "Entity" in df.columns and report_cols.issubset(df.columns) @@ -1263,7 +1263,7 @@ def export_current_report(self): ("Shortcut Count", data.get("shortcutCount", 0)), ("Folder count beyond depth limit 100", data.get("folderCountExceedingDepthLimit", 0)), ("File count beyond depth limit 100", data.get("fileCountExceedingDepthLimit", 0)), - ("Large Resource Count (Folders with >500k items)", data.get("tenantLevelLargeResourceCount", 0)) + ("Large Resource Count (Entities with >500k items)", data.get("tenantLevelLargeResourceCount", 0)) ] for label, val in summary_rows: @@ -1292,15 +1292,20 @@ def export_current_report(self): # Section 4: Large Resources if len(data.get("tenantLevelLargeResources", [])) > 0: - writer.writerow(["Large Resources", ""]) - writer.writerow(["Type", "ID", "SubTreeCount", "Drive"]) - large_resources = data.get("tenantLevelLargeResources", []) - for res in large_resources: + weights = { + "SITE COLLECTION": 1, + "SUBSITE": 2, + "DOCUMENT LIBRARY": 3, + "FOLDER": 4 + } + sorted_large_resources = sorted(data.get("tenantLevelLargeResources", []), key=lambda x: weights.get(x.get("Type", x.get("type", "")), 0), reverse=False) + writer.writerow(["Large Resources (Entities with >500k items)", ""]) + writer.writerow(["Type", "URL", "Item Count"]) + for res in sorted_large_resources: writer.writerow([ res.get("Type", res.get("type", "")), - res.get("Id", res.get("id", "")), - res.get("subTreeCount", 0), - self._get_display_name(res.get("drive", "")) + self._get_display_name(res.get("Id", res.get("id", ""))), + res.get("subTreeCount", 0) ]) writer.writerow([]) # Blank line separator @@ -1309,9 +1314,9 @@ def export_current_report(self): # Section 5: Site Details writer.writerow(["Site Details", ""]) if "siteIdToMail" not in data: - row = ["Site Collection", "Subsite Count", "DL Count", "List Count", "Folder Count", "File Count", "Shortcut Count", "Folder Count > Depth Limit 100", "File Count > Depth Limit 100", "Folder with > 500k item count", "Corpus Size"] + row = ["Site Collection", "Subsite Count", "DL Count", "List Count", "Folder Count", "File Count", "Shortcut Count", "Folder Count > Depth Limit 100", "File Count > Depth Limit 100", "Entities with > 500k item count", "Corpus Size"] else: - row = ["Site Collection", "Email Id", "Subsite Count", "DL Count", "List Count", "Folder Count", "File Count", "Shortcut Count", "Folder Count > Depth Limit 100", "File Count > Depth Limit 100", "Folder with > 500k item count", "Corpus Size"] + row = ["Site Collection", "Email Id", "Subsite Count", "DL Count", "List Count", "Folder Count", "File Count", "Shortcut Count", "Folder Count > Depth Limit 100", "File Count > Depth Limit 100", "Entities with > 500k item count", "Corpus Size"] if self.show_eta: row.append("Suggested Batch") diff --git a/util/enums.py b/util/enums.py index 00eae1ab..c1665407 100644 --- a/util/enums.py +++ b/util/enums.py @@ -10,7 +10,8 @@ class FailureType(Enum): UNKNOWN_ERROR = 7 class ResourceType(Enum): - SITE = "SITE" + SITE = "SITE COLLECTION" FOLDER = "FOLDER" - DL = "DL" - FILE = "FILE" \ No newline at end of file + DL = "DOCUMENT LIBRARY" + FILE = "FILE" + SUBSITE = "SUBSITE" \ No newline at end of file From da3ff2f84a2e05b0df3e18acf9be735e04dabff2 Mon Sep 17 00:00:00 2001 From: Mayuresh Mulmule Date: Wed, 1 Jul 2026 10:53:15 +0530 Subject: [PATCH 3/3] feat: add Deal Assistant telemetry suite and fixes --- .gitignore | 5 +- README.md | 455 +++- core/cert_auth.py | 149 ++ core/graph/__init__.py | 15 + core/graph/client.py | 84 + core/graph/db.py | 253 ++ core/graph/delegated_auth.py | 112 + core/graph/directory.py | 572 +++++ core/graph/directory/__init__.py | 260 ++ core/graph/directory/domains.py | 93 + core/graph/directory/organization.py | 68 + core/graph/directory/provisioning_logs.py | 159 ++ core/graph/directory/service_principals.py | 68 + core/graph/directory/subscribed_skus.py | 43 + core/graph/directory/user_logs.py | 136 + core/graph/directory/users_groups.py | 153 ++ core/graph/ediscovery.py | 61 + core/graph/entra/__init__.py | 203 ++ core/graph/entra/app_registrations.py | 51 + core/graph/entra/app_signins.py | 51 + core/graph/entra/auth_methods.py | 53 + core/graph/entra/user_signins.py | 51 + core/graph/exchange/__init__.py | 24 + core/graph/exchange/calendar.py | 122 + core/graph/exchange/connectors.py | 58 + core/graph/exchange/email_clients.py | 101 + core/graph/exchange/integrated_apps.py | 21 + core/graph/exchange/mail_security.py | 90 + core/graph/exchange/mailbox.py | 164 ++ core/graph/exchange/pst_files.py | 36 + core/graph/exchange/transport_rules.py | 41 + core/graph/files/__init__.py | 18 + core/graph/files/msteams_overview.py | 40 + core/graph/files/onedrive.py | 171 ++ core/graph/files/sharepoint.py | 93 + core/graph/files/sharepoint_data_types.py | 122 + core/graph/intune/__init__.py | 431 ++++ core/graph/intune/byod_configs.py | 110 + core/graph/intune/detected_apps.py | 83 + core/graph/intune/device_compliance.py | 87 + core/graph/intune/device_configs.py | 157 ++ core/graph/intune/managed_devices.py | 92 + core/graph/intune/mdm_policies.py | 110 + core/graph/intune/mobile_apps.py | 114 + core/graph/m365_apps/__init__.py | 19 + core/graph/m365_apps/active_users.py | 107 + core/graph/m365_apps/active_users_trend.py | 99 + core/graph/m365_apps/app_usage.py | 84 + core/graph/network_security/__init__.py | 59 + .../network_security/conditional_access.py | 96 + core/graph/network_security/filtering.py | 86 + core/graph/network_security/firewall.py | 104 + core/graph/reports.py | 574 +++++ core/graph/security/__init__.py | 153 ++ core/graph/security/authentication.py | 99 + core/graph/security/dlp_policies.py | 55 + core/graph/security/retention_policies.py | 55 + core/graph/security/sensitive_info_types.py | 55 + core/graph/security/sensitivity_labels.py | 101 + core/graph/security/service_principals_sso.py | 80 + core/powershell/__init__.py | 1 + core/powershell/calendar.py | 87 + core/powershell/client.py | 46 + core/powershell/dlp.py | 120 + core/powershell/encryption.py | 45 + core/powershell/exchange_connectors.py | 46 + core/powershell/mailbox.py | 39 + core/powershell/retention.py | 39 + .../scripts/exchange_calendar_metadata.ps1 | 99 + .../scripts/exchange_room_mailboxes.ps1 | 55 + .../scripts/export_encryption_policies.ps1 | 67 + core/powershell/scripts/get_connectors.ps1 | 76 + core/powershell/scripts/get_dlp_policies.ps1 | 107 + .../scripts/get_mailbox_and_folder_stats.ps1 | 131 + .../scripts/get_retention_policies.ps1 | 102 + .../scripts/get_sensitive_info_types.ps1 | 40 + .../scripts/get_transport_rules.ps1 | 58 + core/powershell/transport_rules.py | 43 + deal_assistant.py | 1493 +++++++++++ docs/m365_telemetry_scaling_skill.md | 326 +++ estimators/estimator.py | 3 +- estimators/file_estimator.py | 65 +- flet_app/README.md | 91 + flet_app/auth_view.py | 86 + flet_app/cert_instructions_view.py | 75 + flet_app/custom_chart.py | 59 + flet_app/dashboard.py | 1184 +++++++++ flet_app/main.py | 113 + flet_app/sidebar.py | 61 + flet_app/styles.py | 27 + migration_planner.py | 5 + requirements.txt | 13 + scripts/app_creation_script.ps1 | 301 +++ telemetry/README.md | 77 + telemetry/active_users_usage.py | 25 + telemetry/calendar_telemetry.py | 21 + telemetry/data_security_governance.py | 97 + telemetry/devices_apps_telemetry.py | 27 + telemetry/directory/__init__.py | 193 ++ telemetry/directory/domains.py | 402 +++ telemetry/directory/organization.py | 352 +++ telemetry/directory/provisioning_logs.py | 370 +++ telemetry/directory/user_logs.py | 352 +++ telemetry/directory/users_groups.py | 321 +++ telemetry/ediscovery_ui.py | 292 +++ telemetry/email_client_support.py | 22 + telemetry/entra/__init__.py | 149 ++ telemetry/entra/app_registrations.py | 382 +++ telemetry/entra/app_signins.py | 370 +++ telemetry/entra/auth_methods.py | 342 +++ telemetry/entra/user_signins.py | 342 +++ telemetry/exchange/__init__.py | 238 ++ telemetry/exchange/calendar.py | 245 ++ telemetry/exchange/connectors.py | 385 +++ telemetry/exchange/email_clients.py | 253 ++ telemetry/exchange/integrated_apps.py | 247 ++ telemetry/exchange/mail_security.py | 295 +++ telemetry/exchange/mailbox.py | 227 ++ telemetry/exchange/pst_files.py | 266 ++ telemetry/exchange/transport_rules.py | 330 +++ telemetry/exchange_apps.py | 21 + telemetry/exchange_connectors_ui.py | 21 + telemetry/files/__init__.py | 110 + telemetry/files/msteams_overview.py | 229 ++ telemetry/files/onedrive.py | 193 ++ telemetry/files/sharepoint.py | 203 ++ telemetry/files_telemetry.py | 18 + telemetry/intune/__init__.py | 291 +++ telemetry/intune/byod_configs.py | 303 +++ telemetry/intune/detected_apps.py | 261 ++ telemetry/intune/device_compliance.py | 299 +++ telemetry/intune/device_configs.py | 343 +++ telemetry/intune/managed_devices.py | 299 +++ telemetry/intune/mdm_policies.py | 314 +++ telemetry/intune/mobile_apps.py | 182 ++ telemetry/intune/vc_devices.py | 365 +++ telemetry/intune_policies.py | 23 + telemetry/m365_apps/__init__.py | 134 + telemetry/m365_apps/active_users.py | 191 ++ telemetry/m365_apps/active_users_trend.py | 235 ++ telemetry/m365_apps/app_usage.py | 260 ++ telemetry/m365_telemetry.py | 1126 +++++++++ telemetry/mail_security.py | 21 + telemetry/mailbox_usage.py | 21 + telemetry/network_security.py | 18 + telemetry/network_security/__init__.py | 135 + .../network_security/conditional_access.py | 246 ++ telemetry/network_security/filtering.py | 246 ++ telemetry/network_security/firewall.py | 246 ++ telemetry/pdf_report.py | 2234 +++++++++++++++++ telemetry/power_automate.py | 768 ++++++ telemetry/security/__init__.py | 225 ++ telemetry/security/authentication.py | 284 +++ telemetry/security/dlp_policies.py | 306 +++ telemetry/security/retention_policies.py | 348 +++ telemetry/security/sensitive_info_types.py | 303 +++ telemetry/security/sensitivity_labels.py | 319 +++ telemetry/security/service_principals_sso.py | 249 ++ telemetry/sharepoint_onedrive_usage.py | 23 + telemetry/styles.py | 39 + telemetry/subscribed_skus.py | 426 ++++ telemetry/transport_rules_ui.py | 21 + tests/files/data_state_creator.py | 52 +- tests/files/load_tests.py | 24 +- ui/chats_ui.py | 4 + ui/exchange_online_ui.py | 6 +- ui/files_ui.py | 41 +- util/constants.py | 2 - util/enums.py | 7 +- 169 files changed, 30690 insertions(+), 276 deletions(-) create mode 100644 core/cert_auth.py create mode 100644 core/graph/__init__.py create mode 100644 core/graph/client.py create mode 100644 core/graph/db.py create mode 100644 core/graph/delegated_auth.py create mode 100644 core/graph/directory.py create mode 100644 core/graph/directory/__init__.py create mode 100644 core/graph/directory/domains.py create mode 100644 core/graph/directory/organization.py create mode 100644 core/graph/directory/provisioning_logs.py create mode 100644 core/graph/directory/service_principals.py create mode 100644 core/graph/directory/subscribed_skus.py create mode 100644 core/graph/directory/user_logs.py create mode 100644 core/graph/directory/users_groups.py create mode 100644 core/graph/ediscovery.py create mode 100644 core/graph/entra/__init__.py create mode 100644 core/graph/entra/app_registrations.py create mode 100644 core/graph/entra/app_signins.py create mode 100644 core/graph/entra/auth_methods.py create mode 100644 core/graph/entra/user_signins.py create mode 100644 core/graph/exchange/__init__.py create mode 100644 core/graph/exchange/calendar.py create mode 100644 core/graph/exchange/connectors.py create mode 100644 core/graph/exchange/email_clients.py create mode 100644 core/graph/exchange/integrated_apps.py create mode 100644 core/graph/exchange/mail_security.py create mode 100644 core/graph/exchange/mailbox.py create mode 100644 core/graph/exchange/pst_files.py create mode 100644 core/graph/exchange/transport_rules.py create mode 100644 core/graph/files/__init__.py create mode 100644 core/graph/files/msteams_overview.py create mode 100644 core/graph/files/onedrive.py create mode 100644 core/graph/files/sharepoint.py create mode 100644 core/graph/files/sharepoint_data_types.py create mode 100644 core/graph/intune/__init__.py create mode 100644 core/graph/intune/byod_configs.py create mode 100644 core/graph/intune/detected_apps.py create mode 100644 core/graph/intune/device_compliance.py create mode 100644 core/graph/intune/device_configs.py create mode 100644 core/graph/intune/managed_devices.py create mode 100644 core/graph/intune/mdm_policies.py create mode 100644 core/graph/intune/mobile_apps.py create mode 100644 core/graph/m365_apps/__init__.py create mode 100644 core/graph/m365_apps/active_users.py create mode 100644 core/graph/m365_apps/active_users_trend.py create mode 100644 core/graph/m365_apps/app_usage.py create mode 100644 core/graph/network_security/__init__.py create mode 100644 core/graph/network_security/conditional_access.py create mode 100644 core/graph/network_security/filtering.py create mode 100644 core/graph/network_security/firewall.py create mode 100644 core/graph/reports.py create mode 100644 core/graph/security/__init__.py create mode 100644 core/graph/security/authentication.py create mode 100644 core/graph/security/dlp_policies.py create mode 100644 core/graph/security/retention_policies.py create mode 100644 core/graph/security/sensitive_info_types.py create mode 100644 core/graph/security/sensitivity_labels.py create mode 100644 core/graph/security/service_principals_sso.py create mode 100644 core/powershell/__init__.py create mode 100644 core/powershell/calendar.py create mode 100644 core/powershell/client.py create mode 100644 core/powershell/dlp.py create mode 100644 core/powershell/encryption.py create mode 100644 core/powershell/exchange_connectors.py create mode 100644 core/powershell/mailbox.py create mode 100644 core/powershell/retention.py create mode 100644 core/powershell/scripts/exchange_calendar_metadata.ps1 create mode 100644 core/powershell/scripts/exchange_room_mailboxes.ps1 create mode 100644 core/powershell/scripts/export_encryption_policies.ps1 create mode 100644 core/powershell/scripts/get_connectors.ps1 create mode 100644 core/powershell/scripts/get_dlp_policies.ps1 create mode 100644 core/powershell/scripts/get_mailbox_and_folder_stats.ps1 create mode 100644 core/powershell/scripts/get_retention_policies.ps1 create mode 100644 core/powershell/scripts/get_sensitive_info_types.ps1 create mode 100644 core/powershell/scripts/get_transport_rules.ps1 create mode 100644 core/powershell/transport_rules.py create mode 100644 deal_assistant.py create mode 100644 docs/m365_telemetry_scaling_skill.md create mode 100644 flet_app/README.md create mode 100644 flet_app/auth_view.py create mode 100644 flet_app/cert_instructions_view.py create mode 100644 flet_app/custom_chart.py create mode 100644 flet_app/dashboard.py create mode 100644 flet_app/main.py create mode 100644 flet_app/sidebar.py create mode 100644 flet_app/styles.py create mode 100644 requirements.txt create mode 100644 scripts/app_creation_script.ps1 create mode 100644 telemetry/README.md create mode 100644 telemetry/active_users_usage.py create mode 100644 telemetry/calendar_telemetry.py create mode 100644 telemetry/data_security_governance.py create mode 100644 telemetry/devices_apps_telemetry.py create mode 100644 telemetry/directory/__init__.py create mode 100644 telemetry/directory/domains.py create mode 100644 telemetry/directory/organization.py create mode 100644 telemetry/directory/provisioning_logs.py create mode 100644 telemetry/directory/user_logs.py create mode 100644 telemetry/directory/users_groups.py create mode 100644 telemetry/ediscovery_ui.py create mode 100644 telemetry/email_client_support.py create mode 100644 telemetry/entra/__init__.py create mode 100644 telemetry/entra/app_registrations.py create mode 100644 telemetry/entra/app_signins.py create mode 100644 telemetry/entra/auth_methods.py create mode 100644 telemetry/entra/user_signins.py create mode 100644 telemetry/exchange/__init__.py create mode 100644 telemetry/exchange/calendar.py create mode 100644 telemetry/exchange/connectors.py create mode 100644 telemetry/exchange/email_clients.py create mode 100644 telemetry/exchange/integrated_apps.py create mode 100644 telemetry/exchange/mail_security.py create mode 100644 telemetry/exchange/mailbox.py create mode 100644 telemetry/exchange/pst_files.py create mode 100644 telemetry/exchange/transport_rules.py create mode 100644 telemetry/exchange_apps.py create mode 100644 telemetry/exchange_connectors_ui.py create mode 100644 telemetry/files/__init__.py create mode 100644 telemetry/files/msteams_overview.py create mode 100644 telemetry/files/onedrive.py create mode 100644 telemetry/files/sharepoint.py create mode 100644 telemetry/files_telemetry.py create mode 100644 telemetry/intune/__init__.py create mode 100644 telemetry/intune/byod_configs.py create mode 100644 telemetry/intune/detected_apps.py create mode 100644 telemetry/intune/device_compliance.py create mode 100644 telemetry/intune/device_configs.py create mode 100644 telemetry/intune/managed_devices.py create mode 100644 telemetry/intune/mdm_policies.py create mode 100644 telemetry/intune/mobile_apps.py create mode 100644 telemetry/intune/vc_devices.py create mode 100644 telemetry/intune_policies.py create mode 100644 telemetry/m365_apps/__init__.py create mode 100644 telemetry/m365_apps/active_users.py create mode 100644 telemetry/m365_apps/active_users_trend.py create mode 100644 telemetry/m365_apps/app_usage.py create mode 100644 telemetry/m365_telemetry.py create mode 100644 telemetry/mail_security.py create mode 100644 telemetry/mailbox_usage.py create mode 100644 telemetry/network_security.py create mode 100644 telemetry/network_security/__init__.py create mode 100644 telemetry/network_security/conditional_access.py create mode 100644 telemetry/network_security/filtering.py create mode 100644 telemetry/network_security/firewall.py create mode 100644 telemetry/pdf_report.py create mode 100644 telemetry/power_automate.py create mode 100644 telemetry/security/__init__.py create mode 100644 telemetry/security/authentication.py create mode 100644 telemetry/security/dlp_policies.py create mode 100644 telemetry/security/retention_policies.py create mode 100644 telemetry/security/sensitive_info_types.py create mode 100644 telemetry/security/sensitivity_labels.py create mode 100644 telemetry/security/service_principals_sso.py create mode 100644 telemetry/sharepoint_onedrive_usage.py create mode 100644 telemetry/styles.py create mode 100644 telemetry/subscribed_skus.py create mode 100644 telemetry/transport_rules_ui.py diff --git a/.gitignore b/.gitignore index 7cd8bb2f..828d2f1b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,12 +1,15 @@ +.DS_Store __pycache__/ *.pyc *.pyo */__pycache__/ -__pycache__/ */*/__pycache__/ */*/test_* .env env/ outputs/ +output/ +reports/ +certificate/ data/ test_*.py \ No newline at end of file diff --git a/README.md b/README.md index f1b3f4c8..ab74e6de 100644 --- a/README.md +++ b/README.md @@ -1,28 +1,15 @@ -# Migration Planner Tool +# Deal Assistant & Migration Planner **Doc version: v2.1.0** ## What's new -- **Microsoft Teams & Chat Migration Planning**: Full support for scanning and projecting migration timelines for Microsoft Teams, Channels, and Private Chats alongside Exchange Online. -- **Files in OneDrive / SharePoint Planning**: Support added for metrics related to OneDrive / SharePoint sites. -- **Process-Level Decoupling Architecture**: Architectural refactoring that decouples the launcher (`migration_planner.py`) and individual workload planners into isolated OS subprocesses to prevent runtime contention, memory corruption, or GIL clashes during highly concurrent MS Graph API scanning. -- **Bidirectional Navigation**: Addition of a top navigation bar featuring a `← Back to Selector` button for seamless transitions between workload planners. - -#### UX change to support new features -- The startup screen would now show a selector that enables user to select if they want to run Exchange online estimations, Files estimations or Chat Estimations. -- The progress screen would have three progress bars for Files estimation: - - **Site Discovery**: This progress bar will report the progress while scanning all the sites/subsites in the tenant under the root site, along with some other metadata like List count, Drives/DLs count, License units count, etc.. - - **Drive Discovery**: This progress bar will report the progress while scanning all the folders in the drives found in the site scan. - - **Metrics Calculation**: This will show the progress of the metrics (like max depth, folder count, files count, etc.) for the drives. -- The report screen would display: - - **Summary Metrics**: This will display the summary metrics for the entire tenant. - - **File Size Distribution**: This will display the distribution of files based on their sizes as per the bucket ranges provided in the input screen. - -#### System behaviour changes -- **Site Discovery** and **Drive Discovery** phases in the progress screen would show indeterminate progress as the total number of sites/folders/files are not known during those phases. -- However the **Metrics Calculation** phase would be determinate and show proper progress. -- The logs and CSV report would only be available through the export option and not under the outputs/ directory to minimize report creation latency for huge reports. +- **Unified Interface**: "Deal Assistant" merges the traditional "Migration Planner" with a powerful "Usage and Adoption" telemetry dashboard into a single, unified application (`deal_assistant.py`). +- **Unified Login Screen**: A centralized connection interface for entering Tenant ID, Client ID, and Client Secret. +- **Optional Delegated Authentication**: Support for delegated auth flows required by eDiscovery modules. +- **Certificate-Based Authentication**: Automated generation and handling of Certificates to access Data Security & Governance metrics via Exchange Online PowerShell. +- **Usage and Adoption Telemetry**: Comprehensive tenant insights including Subscribed SKUs, Directory Summaries, M365 App Usage, Power Automate, and Intune Policies, exportable as a high-fidelity PDF report. +- **Process-Level Decoupling Architecture**: Architectural refactoring that decouples the launcher and individual workload planners into isolated OS subprocesses to prevent runtime contention during highly concurrent MS Graph API scanning. ## DISCLAIMER @@ -50,34 +37,43 @@ - [Prerequisites & Installation](#prerequisites--installation) - [1. Python Version](#1-python-version) - [2. Installation Steps](#2-installation-steps) - - [Windows](#windows) - - [macOS](#macos) - - [Linux (Ubuntu/Debian)](#linux-ubuntudebian) - [3. Setting up a Virtual Environment (Optional / Corp Policy)](#3-setting-up-a-virtual-environment-optional--corp-policy) + - [4. PowerShell Core (pwsh) Installation (Optional but Recommended)](#4-powershell-core-pwsh-installation-optional-but-recommended) - [Setting up Microsoft Azure](#setting-up-microsoft-azure) - [1. Register the App](#1-register-the-app) - - [2. Grant Permissions](#2-grant-permissions) - - [3. Get Credentials](#3-get-credentials) -- [Running the Tool & Process Decoupling](#running-the-tool--process-decoupling) -- [Tool Configuration & Scanning](#tool-configuration--scanning) - - [Workflow A: Exchange Online Planner](#workflow-a-exchange-online-planner) - - [Workflow B: Microsoft Files (OneDrive + SharePoint) Planner](#workflow-b-microsoft-onedrive--sharepoint) - - [Workflow C: Microsoft Teams & Chat Planner](#workflow-c-microsoft-teams--chat-planner) -- [Understanding the Results](#understanding-the-results) - - [Workflow A: Exchange Online Planner](#workflow-a-results-exchange-online-planner) - - [Workflow B: Microsoft Files (OneDrive + SharePoint) Planner](#workflow-b-results-microsoft-onedrive--sharepoint) - - [Workflow C: Microsoft Teams & Chat Planner](#workflow-c-results-microsoft-teams--chat-planner) -- [Outputs & Artifacts](#outputs--artifacts) - - [Workflow A: Exchange Online Planner](#workflow-a-outputs-exchange-online-planner) - - [Workflow B: Microsoft Files (OneDrive + SharePoint) Planner](#workflow-b-outputs-microsoft-onedrive--sharepoint) - - [Workflow C: Microsoft Teams & Chat Planner](#workflow-c-outputs-microsoft-teams--chat-planner) + - [2. Graph API Permissions](#2-graph-api-permissions) + - [3. Power Platform & Dataverse Permissions](#3-power-platform--dataverse-permissions) + - [4. Get Credentials](#4-get-credentials) +- [Advanced Authentication & Setup (Deal Assistant)](#advanced-authentication--setup-deal-assistant) + - [1. Delegated Authentication Flow](#1-delegated-authentication-flow) + - [2. PowerShell & Certificate-Based Authentication](#2-powershell--certificate-based-authentication) + - [3. Entra ID Directory Roles](#3-entra-id-directory-roles) +- [Running the Tool](#running-the-tool) +- [Unified Interface Navigation](#unified-interface-navigation) +- [Tab 1: Usage and Adoption](#tab-1-usage-and-adoption) + - [Tab 1 Outputs: Usage and Adoption](#tab-1-outputs-usage-and-adoption) +- [Tab 2: Migration Planner](#tab-2-migration-planner) + - [Tool Configuration & Scanning](#tool-configuration--scanning) + - [Workflow A: Exchange Online Planner](#workflow-a-exchange-online-planner) + - [Workflow B: Microsoft Files (OneDrive + SharePoint)](#workflow-b-microsoft-files-onedrive--sharepoint) + - [Workflow C: Microsoft Teams & Chat Planner](#workflow-c-microsoft-teams--chat-planner) + - [Understanding the Results](#understanding-the-results) + - [Workflow A: Exchange Online Planner](#workflow-a-exchange-online-planner-1) + - [Workflow B: Microsoft Files (OneDrive + SharePoint)](#workflow-b-microsoft-files-onedrive--sharepoint-1) + - [Workflow C: Microsoft Teams & Chat Planner](#workflow-c-microsoft-teams--chat-planner-1) + - [Outputs & Artifacts](#outputs--artifacts) + - [Workflow A: Exchange Online Planner](#workflow-a-exchange-online-planner-2) + - [Workflow B: Microsoft Files (OneDrive + SharePoint)](#workflow-b-microsoft-files-onedrive--sharepoint-2) + - [Workflow C: Microsoft Teams & Chat Planner](#workflow-c-microsoft-teams--chat-planner-2) - [Terms & Disclaimer](#terms--disclaimer) --- ## Introduction -The Migration Planner is a desktop application designed to help deployment partners and IT administrators assess a Microsoft 365 tenant before migration. Through its process-decoupled architecture, administrators can independently assess Exchange Online (Emails, Contacts, Calendars, In-Place Archives, Group Mails), Files in OneDrive / SharePoint or Microsoft Teams (Channels, Private Chats) to provide volume metrics and generate optimized Migration Batch Plans with estimated completion times (ETAs) (not applicable for files estimation). +The **Deal Assistant** is a comprehensive desktop application designed to help deployment partners and IT administrators assess a Microsoft 365 tenant before migration. It is split into two primary modules: +1. **Usage and Adoption**: A deep telemetry and discovery module providing insights into a tenant's directory, license usage, security governance, and endpoint management. +2. **Migration Planner**: Independently assess Exchange Online (Emails, Contacts, Calendars), Files in OneDrive / SharePoint, or Microsoft Teams to provide volume metrics and generate optimized Migration Batch Plans with estimated completion times. --- @@ -91,32 +87,32 @@ Please ensure you have **Python 3.10** or newer installed on your system. #### Windows 1. **Download Python**: Visit [python.org/downloads](https://www.python.org/downloads/) and download the latest installer. - * **Important**: Ensure the checkbox **"tcl/tk and IDLE"** is selected during installation (it is usually selected by default). This installs the necessary GUI components. - * **Important**: Check the box **"Add Python to PATH"** during installation. -2. **Verify Installation**: Open Command Prompt (cmd) or PowerShell and make sure the following commands run successfully and return you the version ids of python and pip: + * **Important**: Ensure the checkbox **"tcl/tk and IDLE"** is selected during installation. + * **Important**: Check the box **"Add Python to PATH"**. +2. **Verify Installation**: Open Command Prompt (cmd) or PowerShell: ```cmd python --version pip --version ``` 3. **Install Dependencies**: Run the following command: ```cmd - pip install customtkinter requests pandas psutil Pillow urllib3 sortedcontainers aiohttp certifi + pip install -r requirements.txt ``` #### macOS -1. **Download Python**: Visit [python.org/downloads](https://www.python.org/downloads/) and download the macOS installer. Alternatively, use Homebrew from the terminal (`brew install python`). -2. **Install Tkinter**: If you are using Homebrew or encounter GUI errors, you may need to explicitly install the Tkinter library: +1. **Download Python**: Visit [python.org/downloads](https://www.python.org/downloads/) or use Homebrew (`brew install python`). +2. **Install Tkinter**: If you are using Homebrew or encounter GUI errors: ```bash brew install python-tk ``` -3. **Verify Installation**: Open Command Prompt (cmd) or PowerShell and make sure the following commands run successfully and return you the version ids of python and pip: +3. **Verify Installation**: Open terminal and make sure the following commands run successfully and return you the version IDs of python and pip: ```bash python3 --version pip3 --version ``` 4. **Install Dependencies**: ```bash - pip3 install customtkinter requests pandas psutil Pillow urllib3 sortedcontainers aiohttp certifi + pip3 install -r requirements.txt ``` #### Linux (Ubuntu/Debian) @@ -131,7 +127,7 @@ Please ensure you have **Python 3.10** or newer installed on your system. ``` 3. **Install Dependencies**: ```bash - pip3 install customtkinter requests pandas psutil Pillow urllib3 sortedcontainers aiohttp certifi + pip3 install -r requirements.txt ``` ### 3. Setting up a Virtual Environment (Optional / Corp Policy) @@ -151,85 +147,305 @@ If your organization restricts installing packages globally, use a virtual envir * Mac/Linux: `source venv/bin/activate` 3. **Install packages** as shown above inside this environment. +### 4. PowerShell Core (pwsh) Installation (Optional but Recommended) + +[PowerShell](https://learn.microsoft.com/en-us/powershell/scripting/install/install-powershell?view=powershell-7.6) is a cross-platform command-line shell and scripting language designed for task automation. It is required by specific features in the [Usage and Adoption](#tab-1-usage-and-adoption) tab, such as extracting: + +- Sensitivity Labels +- Retention Policies +- Sensitive Information Types +- Conditional Access Policies +- Data Loss Prevention policies +- Exchange Transport Rules +- Shared/Public mailbox statistics +- Detailed Calendar settings +- Exchange Inbound and Outbound Connectors + +> **Note**: If you do not install PowerShell, the tool will still run perfectly fine for all other sections (like the Migration Planner modules and standard Graph API telemetry), but these specific PowerShell-dependent reports will be skipped and marked as unavailable. + +Please follow the official Microsoft guides to install PowerShell Core on your platform: + +* **Windows**: Refer to the official [Windows Installation Guide](https://learn.microsoft.com/en-us/powershell/scripting/install/install-powershell-on-windows?view=powershell-7.6). +* **macOS**: Refer to the official [macOS Installation Guide](https://learn.microsoft.com/en-us/powershell/scripting/install/install-powershell-on-macos?view=powershell-7.6) or learn how to [Install on macOS using Homebrew](https://learn.microsoft.com/en-us/powershell/scripting/install/alternate-install-methods?view=powershell-7.6#install-on-macos-using-homebrew). +* **Linux**: Refer to the official [Linux Overview & Installation Guide](https://learn.microsoft.com/en-us/powershell/scripting/install/linux-overview?view=powershell-7.6). + +#### Install the Exchange Online Module +Please follow the [official Microsoft guide](https://learn.microsoft.com/en-us/powershell/exchange/exchange-online-powershell-v2?view=exchange-ps#windows-support-for-the-module) to install the required Exchange module: +* **Windows**: Refer to the official [Windows support for the module](https://learn.microsoft.com/en-us/powershell/exchange/exchange-online-powershell-v2?view=exchange-ps#windows-support-for-the-module). +* **macOS**: Refer to the official [macOS support for the module](https://learn.microsoft.com/en-us/powershell/exchange/exchange-online-powershell-v2?view=exchange-ps#macos-support-for-the-module). +* **Linux**: Refer to the official [Linux support for the module](https://learn.microsoft.com/en-us/powershell/exchange/exchange-online-powershell-v2?view=exchange-ps#linux-support-for-the-module). + +#### Common Errors and Troubleshooting Steps with Powershell +* **{script} cannot be loaded because running scripts is disabled on this system**: This error arises if the current execution policy is set to `Restricted`. This can be checked by running the command `Get-ExecutionPolicy`. Before running deal assistant, this must be changed to `RemoteSigned` or a more lenient permission level. To change the permission level of the local machine on which deal assistant is running, run the command `Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser`. Verify that the permission level has changed by again running `Get-ExecutionPolicy`. +* **{script} is not digitally signed. You cannot run this script on the current system.**: For each script present in **core/powershell/scripts**, right click on it and go to Properties > General and check the **Unblock** checkbox. Click on Apply/Ok. + --- ## Setting up Microsoft Azure To scan your tenant, you need to register an app in the Microsoft Entra ID (formerly Azure AD) portal. +> **💡 AUTOMATED SETUP AVAILABLE:** You can completely automate Steps 1-3 below (App Creation, API Permissions, Admin Consent, and Secret Generation) by running the provided PowerShell script. Simply execute `.\scripts\app_creation_script.ps1` from your PowerShell terminal and follow the prompts. Note that this requires the Microsoft Graph PowerShell module. If you prefer manual setup, follow the steps below. +In order to execute the script right click and select "Run with Powershell" + ### 1. Register the App 1. Go to [portal.azure.com](https://portal.azure.com/). 2. Navigate to **Microsoft Entra ID > App registrations > New registration**. -3. Name the app (e.g., "Migration Planner Tool"). +3. Name the app (e.g., "Deal Assistant Tool"). 4. Select **"Accounts in this organizational directory only"** (Single Tenant). 5. Click **Register**. +6. **(Required for Delegated Auth & eDiscovery)**: Go to **Authentication**. Under **Redirect URI configuration**, click **Add Redirect URI** -> **Web**. Add `http://localhost` as the redirect URI. + +### 2. API Permissions +In your new app, go to **API permissions > Add a permission**, and assign permissions based on the workloads you plan to scan. *Don't forget to click **"Grant admin consent"** after adding these permissions.* + +#### 2.1. Deal Assistant Telemetry Permissions (Usage and Adoption) +The Usage and Adoption tab performs extensive tenant auditing. While the following permissions are recommended for a complete report, you may choose to grant only a subset. +**NOTE: Be aware that any missing permissions will simply cause the tool to gracefully skip those specific telemetry sections.** + +**2.1.1. Microsoft Graph API Application permissions:** +* `Reports.Read.All`: Used to retrieve active user trends, mailbox/SharePoint usage reports, M365 Apps, and Email Client usage. +* `Directory.Read.All`: Used to read tenant organization configuration data, Domain, User, and Group summaries. +* `Policy.Read.All`: Required for Conditional Access & Authentication mechanics. +* `NetworkAccess.Read.All`: Required for Entra Global Secure Access Filtering Policies (Network Security). +* `DeviceManagementConfiguration.Read.All`: Required for Intune Policies. +* `DeviceManagementServiceConfig.Read.All`: Required for Mobile BYOD Configurations. +* `DeviceManagementApps.Read.All`: Required to read all Intune apps +* `DeviceManagementManagedDevices.Read.All`: Required to read all Intune managed devices +* `Organization.Read.All`: Required to resolve tenant domains and Subscribed SKUs. +* `Place.Read.All`: Used to list meeting rooms and resource device counts. +* `Calendars.ReadBasic.All`: Used to audit organizational calendar permissions. +* `Sites.Read.All`: Used to search for pst files across OneDrive and SharePoint, and to query aggregate counts of Document Libraries, Web Pages, and Lists via the Search Query API. +* `AuditLog.Read.All`: to read audit log data +* `SensitivityLabels.Read.All`: to read all sensitivity labels +* `Application.Read.All`: Required to retrieve App Registrations directory details and Service Principal SSO configurations. + +**2.1.2. Microsoft Graph API Delegated permissions:** +* `eDiscovery.Read.All`: Required to retrieve active/closed Microsoft Purview eDiscovery cases on behalf of the user. +* `Policy.Read.All`: Required to retrieve Mobile Device Management (MDM) Policies on behalf of the user. +* `offline_access`: required to maintain access to data you have given the app access to + +> **Note** : The user who logs in must be an eDiscovery administrator. + +**2.1.3. Office 365 Exchange Online Application Permissions (under `APIs my organization uses`):** +* `Exchange.ManageAsApp`: required to read data governance ans security policies (sensitive, information types, exchange connectors etc) +* `Exchange.ManageAsAppV2`: required to read data governance ans security policies (sensitive, information types, exchange connectors etc) + +#### 2.2. Migration Planner Permissions (Microsoft Graph Application Permissions) + +**2.2.1. Shared Core Permissions** +* `User.Read.All` (To list users) +* `Group.Read.All` (To get M365 group and team structures) -### 2. Grant Permissions -In your new app, go to **API permissions > Add a permission > Microsoft Graph > Application permissions** (NOT Delegated), and assign permissions based on the workloads you plan to scan: +**2.2.2. Exchange Planner Specific Permissions** +* `Mail.Read` +* `Contacts.Read` +* `Calendars.Read` +* `MailboxFolder.Read.All` +* `MailboxSettings.Read` -#### Shared Permissions (Required for Both Workloads) -* `User.Read.All` (To list users and enumerate rosters) -* `Group.Read.All` (To get M365 group and team structures) +**2.2.3. Chat & Teams Planner Specific Permissions** +* `Reports.Read.All` +* `Chat.Read.All` +* `ChannelMessage.Read.All` +* `ChannelSettings.Read.All` +* `TeamsActivity.Read.All` +* `TeamMember.Read.All` +* `Group.Read.All` + +**2.2.4. Files Planner Specific Permissions** +* `Sites.Read.All` +* `Files.Read.All` +* `LicenseAssignment.Read.All` + +### 3. Power Platform & Dataverse Permissions (Optional) +The tool can scan Power Automate flows (Tenant-Wide Cloud Flows and Desktop Flows). These configurations are optional; however, if they are not completed, the Power Platform & Automate Flows Analytics section will fail to load and will be skipped. + +> **Note**: If you choose not to register the Management App or configure Dataverse permissions, the Power Platform & Automate Flows Analytics report will be skipped and marked as unavailable. All other sections of the tool will continue to function normally. + +To enable this scan, the App Registration must have the following configuration: + +1. **Register as a Power Platform Management App (Via PowerShell)**: + The App Registration must be registered as an administrative management application with the Power Platform backend. Log in using a Global Administrator or Power Platform Administrator account and run the following PowerShell commands: + ```powershell + # 1. Install the Power Apps Admin module + Install-Module -Name Microsoft.PowerApps.Administration.PowerShell -AllowClobber -Force + + # 2. Log in to your tenant + Add-PowerAppsAccount -Endpoint prod -TenantID "{tenant_id}" -#### Exchange Planner Specific Permissions -* `Mail.Read` (To count emails) -* `Contacts.Read` (To count contacts) -* `Calendars.Read` (To count calendar events) -* `MailboxFolder.Read.All` (To count emails in in-place archives) -* `MailboxSettings.Read` (To distinguish user and shared mailboxes) - -#### Chat & Teams Planner Specific Permissions -* `Reports.Read.All` (To fetch automated M365 activity reports for heuristics estimation) -* `Chat.Read.All` (To scan private chats and chat memberships) -* `ChannelMessage.Read.All` (To estimate channel messages across teams) -* `ChannelSettings.Read.All` (To list channels) -* `TeamsActivity.Read.All` (To read Teams activity analytics) -* `TeamMember.Read.All` (To read team memberships) -* `Group.Read.All` (To list teams) - -### Files Planner Specific Permissions -* `Sites.Read.All` (To list sites) -* `Files.Read.All` (To count files) -* `LicenseAssignment.Read.All` (To check license information) - -5. Click **Add permissions**. -6. **Crucial Step**: Click **"Grant admin consent for [Your Organization]"** and confirm "Yes". All status icons should turn green. - -### 3. Get Credentials + # 3. Register your App Registration as a Management App + New-PowerAppManagementApp -ApplicationId "{client_id}" + ``` + +2. **Dataverse Environment Permissions (Desktop Flows)**: + The App Registration must be added as an application user and assigned the **System Administrator** role in every Dataverse environment where you want to scan desktop flows: + * Go to the [**Power Platform Admin Center**](https://admin.powerplatform.microsoft.com/manage/environments) > **Environments** > [Select Environment] > **Settings** > **Users + permissions** > **Application users**. + * Click **+ New app user**, select your App Registration, choose the default business unit, and assign the **System Administrator** security role. + +### 4. Data Governance and Security Permissions (Optional) +To allow the App Registration's Service Principal to read Compliance, Retention data, and Exchange settings via PowerShell, it must be assigned the following directory roles in the **[Entra portal](https://entra.microsoft.com/)** > **Roles & admins** > [Select Role and Click] > **Active Assignments** > **Add assignments**. These roles are optional; however, if they are not assigned, those respective security reports will fail to load and will be skipped. + +* **Compliance Administrator** +* **Compliance Data Administrator** + +> **Note**: If these directory roles are not assigned to the App Registration's Service Principal, the Compliance, Retention Policies, and Data Security & Governance telemetry sections will be skipped and marked as unavailable. Other parts of the tool will continue to function normally. + +### 5. Get Credentials You will need three values for the tool: -1. **Tenant ID**: Found on the app's Overview page ("Directory (tenant) ID"). -2. **Client ID**: Found on the app's Overview page ("Application (client) ID"). +1. **Tenant ID**: Found on the app's Overview page. +2. **Client ID**: Found on the app's Overview page. 3. **Client Secret**: * Go to **Certificates & secrets > New client secret**. * Add a description and click **Add**. - * Copy the **"Value"** immediately (you won't see it again). + * Copy the **"Value"** immediately. --- -## Running the Tool & Process Decoupling +## Advanced Authentication & Setup (Deal Assistant) -1. Open your terminal or command prompt. -2. Navigate to the folder containing the script: +### 1. Delegated Authentication Flow +Certain features like **eDiscovery** require **Delegated Authentication**. On the login screen, you can check the box to enable Delegated Authentication. +> **⚠️ IMPORTANT WARNING**: If you enable this, your Entra App Registration MUST have `http://localhost` registered as a redirect URI. Otherwise, the interactive browser login popup will fail! + +### 2. PowerShell & Certificate-Based Authentication +The Deal Assistant uses **Microsoft Exchange Online PowerShell** to fetch Data Security & Governance metrics (like Sensitivity Labels, Retention Policies, Shared/Public mailbox statistics, detailed Calendar settings, and Connectors). Ensure you have installed PowerShell Core (`pwsh`) as detailed in the Prerequisites section. + +**Certificate Authentication**: +A standard Client Secret cannot authorize PowerShell modules—a certificate is required. +1. When you connect, the tool checks if a valid certificate is configured for your tenant/client pair. +2. If absent, the tool securely generates a self-signed certificate (`certificate.pem`) and an encrypted bundle (`passkey.pfx`) under the `certificate/{tenantId}_{clientId}` directory. +3. You will be prompted to upload `certificate.pem` to your Azure App Registration (**Certificates & secrets > Certificates > Upload certificate**). +4. **Optionality**: If you choose to skip this step, sections relying on certificate-based authentication will simply be skipped and marked as unavailable in the report. + +--- + +## Running the Tool + +1. **Download the Code**: Click the green **Code** button at the top of this GitHub repository page, select **Download ZIP**, and extract the contents to a directory of your choice on your system. +2. Open your terminal or command prompt. +3. **Navigate to the Folder**: Change directory (`cd`) to the folder where you extracted the files: ```bash - cd path/to/migration_planner + cd path/to/extracted/folder ``` -3. Run the script: - * Windows: `python migration_planner.py` - * Mac/Linux: `python3 migration_planner.py` +4. **Run the Deal Assistant**: + * Windows: `python deal_assistant.py` + * Mac/Linux: `python3 deal_assistant.py` *(Ensure you are in your virtual environment if you created one).* -### Process Decoupling Mechanics -The launcher window (`migration_planner.py`) acts as a lightweight CustomTkinter `SelectorApp`, which can be used to select which estimations need to be run (Exchange Online, Chats or Files). -All workload planners feature a top navigation bar with a `← Back to Selector` button. Clicking this cleanly terminates the active planner process and respawns a fresh `migration_planner.py` selector session. +--- + +## Unified Interface Navigation + +Upon launching, you are greeted with the **Unified Login Screen**: +* Enter your **Tenant ID**, **Client ID**, and **Client Secret**. +* Select if you wish to use **Delegated Authentication**. +* Click **Connect & Continue**. Follow any certificate upload instructions if prompted. + +Once authenticated, the tool provides a left-hand navigation sidebar with two main tabs: + +## Tab 1: Usage and Adoption +This tab loads the M365 Telemetry dashboard. +* **Fetch Report**: Queries Microsoft Graph APIs and PowerShell in parallel across 10 telemetry modules to audit licenses, usage trends, security policies, and mobile endpoints. +* **Download PDF**: Once telemetry fetching is complete, exports a high-fidelity, comprehensive PDF usage report (`m365_usage_report_.pdf`) detailing the entire tenant's footprint with inline tables and charts. + +### Telemetry Modules & Technical Mechanisms + +The Usage and Adoption tab scans the following 10 functional modules of a Microsoft 365 tenant: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ModuleFunctional ScopeMechanism & Endpoints
1. Subscribed SKUsRetrieves purchased subscriptions (SKUs), listing prepaid unit allocations (Enabled, Warning, Suspended) against active consumed units.Graph REST API: GET /subscribedSkus
(Cache: subscribed_skus table in telemetry_cache.db)
2. Directory (Entra ID Core)• Domains: Verified tenant domains, auth types (Managed/Federated), and services.
• Organization: Global org config details, provisioned plans, and Hybrid Sync status.
• Groups & Users: Aggregate counts for active/disabled users, members/guests, and static/dynamic groups.
• User Creation Logs: Directory audits of added/deleted users.
• Provisioning Logs: Inbound sync logs from HR databases.		• Domains: GET /domains (directory_domains table)
• Organization: GET /organization (directory_organization table)
• Groups & Users: POST /$batch (directory_users_groups table)
• User Creation: GET /auditLogs/directoryAudits (user_logs table)
• Provisioning: GET /auditLogs/provisioning (provisioning_logs table)
3. M365 Apps Adoption• Active Users: Daily active user counts across Exchange, SharePoint, Teams, and OneDrive (30/90/180 days).
• Trends: Adoption curves and growth trends.
• App Usage: Active user counts segmented by desktop Office applications (Word, Excel, PowerPoint, Outlook, OneNote).		Graph Reports API (streaming download):
• Active Users: GET /reports/getOffice365ActiveUserDetail(period='D180')
• Trends: GET /reports/getOffice365ActiveUserCounts(period='D30')
• App Usage: GET /reports/getM365AppUserDetail and getM365AppUserCounts
4. Exchange Online• Mailbox Usage: Total mailbox counts, sizes, items, and shared mailbox/public folder statistics.
• Calendar Telemetry: Tenant-wide scheduling rules, room resources, and calendar processing.
• Email Clients: Email clients usage (OWA Web, Outlook Desktop/Mobile, Apple Mail, legacy IMAP/POP).
• Exchange Connectors: Inbound/outbound connectors, smart hosts, and TLS requirements.
• Exchange Apps: Installed Outlook add-ins and organization apps.
• PST Discovery: Search for .pst file archives stored inside OneDrive/SharePoint libraries.		Hybrid fetch using Graph Reports, Graph Search, and PowerShell (Connect-ExchangeOnline):
• Mailbox Usage: GET /reports/getMailboxUsageDetail and get_mailbox_and_folder_stats.ps1
• Calendar: exchange_calendar_metadata.ps1
• Email Clients: GET /reports/getEmailAppUsageUserDetail(period='D180')
• Connectors: exchange_connectors.ps1
• Exchange Apps: exchange_organization_apps.ps1
• PST Discovery: POST /search/query (Graph Search)
5. Files (SharePoint & OneDrive)• SharePoint Site Usage: Total sites, total storage consumed, file count, and active file metrics.
• SharePoint Data Types: Aggregate hit counts for Document Libraries, Lists, and Web Pages across the entire tenant.
• OneDrive Usage: Personal OneDrive storage sizes, active accounts, and file sync activity.		Graph API:
• SharePoint Site Usage: GET /reports/getSharePointSiteUsageDetail(period='D180')
• SharePoint Data Types: POST /search/query (for contentclass STS_List, STS_List_DocumentLibrary, STS_ListItem_WebPageLibrary)
• OneDrive: GET /reports/getOneDriveUsageAccountDetail and getOneDriveActivityUserDetail
6. Entra ID Auth & Sign-ins• Auth Methods: MFA, SMS, Passkey, etc. registration and adoption counts.
• App Registrations: Application detail entries, Client IDs, registration timestamps, and secrets/certificates.
• App Sign-ins: Login volumes segmented by integrated target applications.
• User Sign-ins: Interactive vs. non-interactive login logs, devices, and browser types.		Graph Reports, Audit Logs, and Directory APIs:
• Auth Methods: GET /reports/authenticationMethods/usersRegisteredByMethod
• App Registrations: GET /applications (app_registrations table)
• App Sign-ins: GET /reports/credentialUserActivity
• User Sign-ins: GET /auditLogs/signIns
7. Intune (Endpoint Management)• Mobile Apps: Managed application packages pushed and maintained via Intune.
• Detected Apps: Software packages detected on managed devices.
• Managed Devices: Details of devices managed or pre-enrolled through Intune.
• Device Configurations: Device configuration profiles, platform policies, and compliance rates.
• Device Compliance Policies: Compliance policies configured to monitor device health and status.
• Mobile Device Management Policies: Mobile Device Management (MDM) policies configured for automatic enrollment and settings.
• Mobile BYOD Configurations: Device enrollment configurations restricting platform compliance (iOS, Windows Mobile, Android).		Graph Intune / Device Management endpoints:
• Mobile Apps: GET /deviceAppManagement/mobileApps
• Detected Apps: GET /deviceManagement/detectedApps
• Managed Devices: GET /deviceManagement/managedDevices
• Device Configurations: GET /deviceManagement/deviceConfigurations and /configurationPolicies
• Device Compliance Policies: GET /deviceManagement/deviceCompliancePolicies
• MDM Policies: GET /beta/policies/mobileDeviceManagementPolicies
• BYOD Configs: GET /deviceManagement/deviceEnrollmentConfigurations
8. Network Security• Secure Access Filtering: Global Secure Access filtering policies.
• Conditional Access: Network-based Conditional Access security policies.
• Firewall/Proxy: Firewall and proxy configuration profiles.		Graph Global Secure Access and Identity endpoints:
• Secure Access Filtering: GET /networkAccess/filteringPolicies
• Conditional Access: GET /identity/conditionalAccess/policies (network locations filtering)
• Firewall/Proxy: GET /deviceManagement/deviceConfigurations` (proxy/firewall profiles)
9. Data Security & Governance• Sensitivity Labels: Sensitivity labels and active label policies.
• Retention Policies: Retention and records compliance policies.
• DLP Policies: Active Data Loss Prevention policies.
• Sensitive Info Types (SIT): Custom and built-in SIT templates.
• Authentication Mechanics: General Entra ID Conditional Access security policies.
• Service Principal SSO: SSO modes (SAML, OIDC, Password, None) configured on Enterprise Applications.
• eDiscovery Cases: Active and closed Microsoft Purview eDiscovery cases.		Hybrid fetch running Graph APIs and PowerShell compliance context:
• Sensitivity Labels: fetch_sensitivity_labels.ps1 (Get-Label / Get-LabelPolicy) (PowerShell)
• Retention Policies: fetch_retention_policies.ps1 (Get-RetentionCompliancePolicy / Get-RetentionComplianceRule) (PowerShell)
• DLP Policies: fetch_dlp_policies.ps1 (Get-DlpCompliancePolicy / Get-DlpComplianceRule) (PowerShell)
• SIT: fetch_sensitive_info_types.ps1 (Get-ClassificationRuleCollection) (PowerShell)
• Authentication Mechanics: GET /identity/conditionalAccess/policies (auth_policies table)
• Service Principal SSO Modes: GET /servicePrincipals (service_principals_sso table)
• eDiscovery Cases: GET /security/cases/ediscoveryCases (Delegated Graph; ediscovery_cases table)
10. Power Automate ScansCloud Flows and Desktop Flows configurations, listing active/inactive flows and checking for complex connectors or premium triggers.Power Platform Admin / CRM Dataverse API requests:
• Environments: GET https://api.bap.microsoft.com/providers/Microsoft.BusinessAppPlatform/environments
• Cloud Flows: GET https://service.flow.microsoft.com/providers/Microsoft.ProcessSimple/environments/{env}/flows
• Desktop Flows: CRM Dataverse GET /api/data/v9.0/workflows (Category 6)
11. Microsoft Teams Overview• Teams Activity: Active users, guests, meetings organized, and channel messages over the last 180 days across all teams.Graph Reports API:
• Teams Activity: GET /reports/getTeamsTeamActivityDetail(period='D180')
+ +### Tab 1 Outputs: Usage and Adoption +* **PDF Report**: Accessible via the "Download PDF" button in the UI. Upon clicking, you can choose a save location on your system. The PDF is a comprehensive document (`m365_usage_report_.pdf`) containing detailed charts, graphs, data tables, and metrics for all successfully audited modules, providing a unified holistic view of the tenant. +* **Raw Data (CSVs & SQLite Database)**: All raw telemetry data fetched directly from Microsoft Graph APIs and PowerShell scripts is dumped into the local `telemetry/reports/_/` directory as CSV files (e.g., `sensitivity_labels.csv`, `ediscovery_cases.csv`). These files serve as the application's source of truth and are preserved for direct human audit if required. Additionally, an SQLite cache database (`telemetry_cache.db`) is automatically built inside this folder to optimize UI rendering performance during paginated lookups. +* **Logs**: Execution and debug logs are stored securely in the local `telemetry/logs/_` directory (specifically `telemetry_log.txt`). --- -## Tool Configuration & Scanning +## Tab 2: Migration Planner +This tab brings you to the workload selector, mirroring the standalone Migration Planner functionality. +* Select between **Exchange Online**, **Chat (Teams)**, or **Files (SharePoint/OneDrive)**. +* Configure workload-specific sources (e.g. Scan all, or Upload CSV), advanced settings (parallel threads, batches, heuristics vs. deep scan). +* **Process Decoupling**: Each workload planner runs isolated to prevent API throttling and memory clashes. A `← Back to Selector` button lets you easily switch workloads. -### Workflow A: Exchange Online Planner +--- + +### Tool Configuration & Scanning -#### 1. Connect & Source Selection (Exchange) +#### Workflow A: Exchange Online Planner + +##### 1. Connect & Source Selection (Exchange) * **Connect with Microsoft**: Enter your Tenant ID, Client ID, and Client Secret. * **User Source**: * **Scan All Users**: Automatically fetches every user in your tenant. @@ -237,7 +453,7 @@ All workload planners feature a top navigation bar with a `← Back to Selector` * **CSV Format**: Must contain a header **Email Id** (e.g., `user@domain.com`). Also if Group Mailbox estimation is required then a column called `"Type"` is needed to segregate group mailbox IDs from user mailbox IDs. The correct values for Type column are `"User"`, `"Group Mailbox"`. * **Smart Delta Scan**: If your CSV already contains columns like `Email Count`, `Contact Count`, `Calendar Count`, `Calendar Event Count`, `In-Place Archive Count` or `Group Mail Count`, `Group Thread Count`, the tool will skip scanning those specific items and use your provided numbers, speeding up the process significantly. -#### 2. Advanced Settings (Exchange) +##### 2. Advanced Settings (Exchange) Click **"Show Advanced Settings"** to tune the performance: * **Sources**: Check/Uncheck Emails, Contacts, Calendars, In-Place Archives and Group Mails to define what you want to scan. * **Concurrency**: Controls how many parallel threads the tool runs. @@ -246,9 +462,9 @@ Click **"Show Advanced Settings"** to tune the performance: --- -### Workflow B: Microsoft OneDrive / SharePoint +#### Workflow B: Microsoft OneDrive / SharePoint -#### 1. Connect & Source Selection +##### 1. Connect & Source Selection * **Connect with Microsoft**: Enter your Tenant ID, Client ID, and Client Secret. * **User Source**: * **Scan All Sites**: Scans all the Sites in the tenant. @@ -272,19 +488,19 @@ Click **"Show Advanced Settings"** to tune the performance: DISCLAIMER: If using the "Upload CSV" feature to re-calculate ETA, the final output would be missing some metadata like "Large Resources", "File Size Distribution", etc.. So the original corpus report should be used as the source of truth and the generated report without scan should be used only for batch planning. -#### 2. Advanced Settings +##### 2. Advanced Settings Click **"Show Advanced Settings"** to tune the performance and select your estimation mode: * **Site Types to Scan**: * **Personal Sites (OneDrive)**: Scans all the Personal / OneDrive sites in the tenant. * **SharePoint Sites**: Scans all the SharePoint sites in the tenant. * **Concurrency**: Controls how many parallel threads the tool runs. Note that this number is not the exact number of threads spawned but is a guidance on the thread count. -#### 3. Starting the Scan +##### 3. Starting the Scan Click **"Get Migration Estimates"**. * A disclaimer will appear noting that results are estimates. Click **OK** to proceed. * The tool will verify your credentials and permissions before starting. -#### 4. The Scan Page +##### 4. The Scan Page Once started, you will see a real-time progress screen: * **Spinners**: Indicate active scanning phases. * **Progress Bars**: Show percentage completion for Site and Drive Discovery along with Metrics Calculations. @@ -292,9 +508,9 @@ Once started, you will see a real-time progress screen: --- -### Workflow C: Microsoft Teams & Chat Planner +#### Workflow C: Microsoft Teams & Chat Planner -#### 1. Connect & Source Selection (Chat) +##### 1. Connect & Source Selection (Chat) * **Connect with Microsoft**: Enter your Tenant ID, Client ID, and Client Secret. * **User Source**: * **Scan all teams and users**: Automatically fetches every team and user in your tenant. @@ -308,7 +524,7 @@ Once started, you will see a real-time progress screen: ``` * **User/Team Resolution**: If a CSV of users is supplied without teams, the tool automatically resolves all Teams these users are members of using the `/users/{id}/joinedTeams` MS Graph API endpoint in batch requests and performs the scan on these teams only. Note that this covers all private and shared channels hosted within those resolved Teams. Shared channels hosted in external teams the user does not belong to are not resolved, as MS Graph does not support querying external shared channel memberships directly. -#### 2. Advanced Settings & Estimation Modes (Chat) +##### 2. Advanced Settings & Estimation Modes (Chat) Click **"Show Advanced Settings"** to tune the performance and select your estimation mode: * **Estimation Modes**: * **Last 6 Months (Heuristics)**: Leverages automated M365 activity reports (`Reports.Read.All`) for instant, high-level tenant estimations without deep scanning. *Note: Sizing projections are computed using statistical multipliers based on standard enterprise averages. For precise customer-specific densities, use Deep Scan mode.* @@ -316,12 +532,12 @@ Click **"Show Advanced Settings"** to tune the performance and select your estim * **Scan Options**: Check or uncheck Private Chats to control scan boundaries. * **Concurrency**: Sets async thread limits for channel message extrapolation. -#### 3. Starting the Scan (Chat) +##### 3. Starting the Scan (Chat) Click **"Get Migration Estimates"**. * A disclaimer will appear noting that results are estimates. Click **OK** to proceed. * The tool will verify your credentials and permissions before starting. -#### 4. The Scan Page (Chat) +##### 4. The Scan Page (Chat) Once started, you will see a real-time progress screen: * **Spinners**: Indicate active scanning phases. * **Progress Bars**: Show percentage completion for Private Chats and Channels. @@ -329,18 +545,18 @@ Once started, you will see a real-time progress screen: --- -## Understanding the Results +### Understanding the Results -### Workflow A Results: Microsoft Exchange Online +#### Workflow A Results: Microsoft Exchange Online -#### 1. Top Level Metrics +##### 1. Top Level Metrics The top cards display the total scope of the migration: * **Users**: Total distinct users identified/scanned. * **Emails / Events / Contacts / In-Place Archives / Group Mailboxes**: The aggregate sum of items across all users. The CSV report would include additional details at the more granular site level. -### 2. Timeline Estimates & Parallel Batches +#### 2. Timeline Estimates & Parallel Batches The tool calculates an Estimated Completion Time (ETA) based on the email corpus using a heuristic based logic: * **User Ordering**: Users are sorted in Ascending Order (Lightest users -> Heaviest users). The lightest users are packed into Batch 1, while the heaviest users usually end up in the final batches. * Max(Emails , (Calendar Events + Contacts), In-Place Archives, Group Mails) determines the sorting logic. @@ -351,9 +567,9 @@ The tool calculates an Estimated Completion Time (ETA) based on the email corpus --- -### Workflow B Results: Microsoft OneDrive / SharePoint +#### Workflow B Results: Microsoft OneDrive / SharePoint -#### 1. Top Level Metrics +##### 1. Top Level Metrics For OneDrive we show the following metrics in the UI report. * **Total Corpus Size**: Total size of the files discovered in the scan. * **Site Collection Count**: Number of sites discovered in the scan. @@ -370,23 +586,23 @@ The CSV report would include the above mentioned details along with the granular --- -### Workflow C Results: Microsoft Teams & Chat Planner +#### Workflow C Results: Microsoft Teams & Chat Planner -#### 1. Top Level Metrics (Chat & Teams) +##### 1. Top Level Metrics (Chat & Teams) The top cards display the total scope of the Chat/Teams migration: * **Users & Private Chats**: Total distinct Users, Private Chats, and Private Chat Messages. * **Teams & Channels**: Total distinct Teams, Channels, and Channel Messages. -#### 2. Timeline Estimates & Pagination Controls (Chat & Teams) +##### 2. Timeline Estimates & Pagination Controls (Chat & Teams) * **User & Team Ordering**: Entities are sorted in Ascending Order (Lightest entities -> Heaviest entities) to optimize packing. * **Pagination Controls**: The results Gantt chart and batch tables include interactive pagination dropdowns supporting 50, 100, 200, or All items per view. * **Total ETA** = The duration of the single longest bucket (lane). --- -## Outputs & Artifacts +### Outputs & Artifacts -### Workflow A Outputs: Exchange Online Planner +#### Workflow A Outputs: Exchange Online Planner Once the scan completes, the tool creates a folder in the `/outputs` directory named with the current timestamp (e.g., `/outputs/20240520_143000/`). @@ -399,7 +615,7 @@ You can also download just the log file via the **"Export logs"** button or the --- -### Workflow B Outputs: Microsoft OneDrive / SharePoint +#### Workflow B Outputs: Microsoft OneDrive / SharePoint Once the scan completes, the artifacts (CSV report and logs) can be downloaded via the "Export logs" and "Export full report" buttons in the UI. @@ -409,7 +625,7 @@ The artifacts include: --- -### Workflow C Outputs: Microsoft Teams & Chat Planner +#### Workflow C Outputs: Microsoft Teams & Chat Planner Once the scan completes, the tool creates a folder in the `/outputs` directory named with the current timestamp. @@ -429,3 +645,4 @@ You can also download just the log file via the **"Export logs"** button or the * **Local Database & Storage**: To enable scan progress persistence and delta tracking, the tool creates and maintains local SQLite databases in the `data/` directory (`data/chat_migration_v2.db` and `data/scan_progress.db`). These databases store metadata, estimated item counts, and progress checkpoints entirely locally. No actual chat message contents or emails are stored. The `data/` directory can be safely deleted or purged at any time after your estimations are completed to reclaim local disk space. * **License & Additional Terms**: Use of this tool is governed by the Apache 2.0 license. * This is not an officially supported Google product. This project is not eligible for the [Google Open Source Software Vulnerability Rewards Program](https://bughunters.google.com/open-source-security). + diff --git a/core/cert_auth.py b/core/cert_auth.py new file mode 100644 index 00000000..46df8aa7 --- /dev/null +++ b/core/cert_auth.py @@ -0,0 +1,149 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Certificate helper management functions for local authentication mapping to Microsoft Graph API.""" + +import os +import logging +import datetime +from typing import Tuple +from cryptography import x509 +from cryptography.x509.oid import NameOID +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.serialization import pkcs12 + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + +def update_log_directory(tenant_id: str = None, client_id: str = None) -> None: + """No-op as logging is handled centrally by the root logger.""" + pass + + +def get_project_root() -> str: + """Helper to locate the root path of migration-planner.""" + return os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +def get_cert_paths(cert_dir: str = "certificate", tenant_id: str = None, client_id: str = None) -> Tuple[str, str, str]: + """Returns absolute paths for cert directory, certificate.pem, and passkey.pfx.""" + root = get_project_root() + if tenant_id and client_id: + dir_path = os.path.join(root, cert_dir, f"{tenant_id}_{client_id}") + else: + dir_path = os.path.join(root, cert_dir) + pem_path = os.path.join(dir_path, "certificate.pem") + pfx_path = os.path.join(dir_path, "passkey.pfx") + return dir_path, pem_path, pfx_path + +def check_certificate_exists(cert_dir: str = "certificate", tenant_id: str = None, client_id: str = None) -> bool: + """Checks if the certificate directory exists and contains a valid cert.pfx file.""" + _, _, pfx_path = get_cert_paths(cert_dir, tenant_id, client_id) + exists = os.path.exists(pfx_path) + logger.info("Checking if certificate exists at %s: %s", pfx_path, exists) + return exists + +def generate_certificate(client_secret: str, cert_dir: str = "certificate", common_name: str = "LocalAppHybridAuth", tenant_id: str = None, client_id: str = None) -> Tuple[str, str]: + """Generates a self-signed certificate and PFX bundle using client_secret as the password. + + Returns: + Tuple containing absolute paths to the generated cert.pem and cert.pfx files. + """ + logger.info("Initializing certificate generation flow...") + dir_path, pem_path, pfx_path = get_cert_paths(cert_dir, tenant_id, client_id) + + os.makedirs(dir_path, exist_ok=True) + logger.info("Certificate directory confirmed: %s", dir_path) + + secret_bytes = client_secret.encode('utf-8') + + logger.info("Generating secure RSA private key...") + private_key = rsa.generate_private_key( + public_exponent=65537, + key_size=2048, + ) + + logger.info("Generating self-signed certificate with CN: %s...", common_name) + subject = issuer = x509.Name([ + x509.NameAttribute(NameOID.COMMON_NAME, common_name), + ]) + + cert = x509.CertificateBuilder().subject_name( + subject + ).issuer_name( + issuer + ).public_key( + private_key.public_key() + ).serial_number( + x509.random_serial_number() + ).not_valid_before( + datetime.datetime.now(datetime.timezone.utc) + ).not_valid_after( + # Valid for 2 years + datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(days=730) + ).sign(private_key, hashes.SHA256()) + + logger.info("Writing PEM public certificate to %s...", pem_path) + with open(pem_path, "wb") as f: + f.write(cert.public_bytes(serialization.Encoding.PEM)) + + logger.info("Serializing and writing PFX encrypted with client secret to %s...", pfx_path) + pfx_bytes = pkcs12.serialize_key_and_certificates( + name=common_name.encode('utf-8'), + key=private_key, + cert=cert, + cas=None, + encryption_algorithm=serialization.BestAvailableEncryption(secret_bytes) + ) + with open(pfx_path, "wb") as f: + f.write(pfx_bytes) + + logger.info("Certificate generation complete! Files written successfully.") + return pem_path, pfx_path + +def load_certificate(client_secret: str, cert_dir: str = "certificate", tenant_id: str = None, client_id: str = None) -> Tuple[str, str]: + """Decrypts PFX bundle using client_secret, extracting private key PEM and SHA1 thumbprint. + + Returns: + Tuple containing private_key_pem (str) and thumbprint (str). + """ + logger.info("Loading certificate files...") + _, _, pfx_path = get_cert_paths(cert_dir, tenant_id, client_id) + + if not os.path.exists(pfx_path): + raise FileNotFoundError(f"PFX certificate file not found at {pfx_path}") + + logger.info("Unlocking PFX file with client secret...") + with open(pfx_path, "rb") as f: + pfx_data = f.read() + + password_bytes = client_secret.encode('utf-8') + private_key, certificate, _ = pkcs12.load_key_and_certificates( + pfx_data, + password_bytes + ) + + logger.info("Extracting private key PEM...") + private_key_pem = private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.NoEncryption() + ).decode("utf-8") + + logger.info("Calculating certificate SHA1 fingerprint...") + thumbprint = certificate.fingerprint(hashes.SHA1()).hex() + logger.info("Loaded certificate successfully. Thumbprint: %s", thumbprint) + + return private_key_pem, thumbprint diff --git a/core/graph/__init__.py b/core/graph/__init__.py new file mode 100644 index 00000000..3f068e81 --- /dev/null +++ b/core/graph/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unified Microsoft Graph API service layer.""" diff --git a/core/graph/client.py b/core/graph/client.py new file mode 100644 index 00000000..c61f1b47 --- /dev/null +++ b/core/graph/client.py @@ -0,0 +1,84 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unified client managing connection pools, authentication, and sessions for Microsoft Graph.""" + +import logging +from typing import List, Dict, Any, Union, Optional +import requests +from util.auth_manager import TokenManager +from util.connectors import UrlInvoker + +logger = logging.getLogger(__name__) + +class GraphClient: + """Unified client managing credentials validation and connection slots to Microsoft Graph.""" + + def __init__( + self, + tenant_id: str, + client_ids: Union[str, List[str]], + client_secrets: Union[str, List[str]], + concurrency: int = 1, + retries: int = 5, + backoff: int = 2 + ) -> None: + self.tenant_id = tenant_id + + # Normalize inputs to lists to perfectly match TokenManager parameters + self.client_ids = [client_ids] if isinstance(client_ids, str) else client_ids + self.client_secrets = [client_secrets] if isinstance(client_secrets, str) else client_secrets + + self.token_manager = TokenManager( + tenant_id=self.tenant_id, + client_ids=self.client_ids, + client_secrets=self.client_secrets, + concurrency=concurrency, + retries=retries, + backoff=backoff + ) + + self.url_invoker = UrlInvoker( + token_manager=self.token_manager, + batch_retry_count=retries, + batch_backoff=backoff, + initial_delay=1, + jitter=0.5 + ) + + def authenticate(self, required_scopes: Optional[List[str]] = None) -> None: + """Validates Entra ID scopes and fetches active tokens.""" + self.token_manager.authenticate_all(required_scopes=required_scopes) + + def get_session(self) -> requests.Session: + """Returns the unified requests Session pool.""" + return self.token_manager.get_session() + + def get_active_token(self) -> Dict[str, Any]: + """Acquires an active, refreshed token slot from TokenManager lease queue.""" + return self.token_manager.get_valid_token_slot() + + def release_token(self, token_slot: Dict[str, Any]) -> None: + """Returns token slot to the queue, completing the lease cycle.""" + self.token_manager.return_token_slot(token_slot) + + def close(self) -> None: + """Releases all connection pool and socket resources.""" + self.token_manager.close() + + def __enter__(self) -> "GraphClient": + return self + + def __exit__(self, exc_type, exc_val, exc_tb) -> None: + self.close() diff --git a/core/graph/db.py b/core/graph/db.py new file mode 100644 index 00000000..71f37eca --- /dev/null +++ b/core/graph/db.py @@ -0,0 +1,253 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Asynchronous SQLite cache manager for telemetry datasets.""" + +import os +import csv +import logging +import sqlite3 +import aiosqlite +from typing import List, Dict, Any, Tuple, Optional + +logger = logging.getLogger(__name__) + +# --- Centralized Case-Insensitive SQLite Row Factory & Connection Patch --- + +class CaseInsensitiveRow(sqlite3.Row): + """SQLite Row factory enabling case-insensitive, space-insensitive, and punctuation-insensitive key retrieval.""" + + def __getitem__(self, key): + if isinstance(key, str): + norm_key = key.strip().lower().replace(" ", "").replace("_", "").replace("-", "") + for real_key in self.keys(): + norm_real = real_key.strip().lower().replace(" ", "").replace("_", "").replace("-", "") + if norm_real == norm_key: + return super().__getitem__(real_key) + return super().__getitem__(key) + + def get(self, key, default=None): + try: + return self[key] + except KeyError: + return default + +class CaseInsensitiveConnection(sqlite3.Connection): + """Custom SQLite Connection that intercepts and forces the CaseInsensitiveRow factory.""" + + def __setattr__(self, name, value): + if name == "row_factory": + super().__setattr__(name, CaseInsensitiveRow) + else: + super().__setattr__(name, value) + +# Patch sqlite3.connect to automatically inject our CaseInsensitiveConnection factory +_original_connect = sqlite3.connect + +def _case_insensitive_connect(*args, **kwargs): + kwargs["factory"] = CaseInsensitiveConnection + conn = _original_connect(*args, **kwargs) + conn.row_factory = CaseInsensitiveRow + return conn + +sqlite3.connect = _case_insensitive_connect + +class CaseInsensitiveDict(dict): + """Case-insensitive dictionary wrapper supporting spacing, case, and underscore insensitivity.""" + def __init__(self, data=None, **kwargs): + super().__init__() + if data: + for k, v in data.items(): + self[k] = v + for k, v in kwargs.items(): + self[k] = v + + def _normalize(self, key): + return str(key).lower().replace(" ", "").replace("_", "").replace("-", "") if isinstance(key, str) else key + + def __getitem__(self, key): + norm_key = self._normalize(key) + for k in self.keys(): + if self._normalize(k) == norm_key: + return super().__getitem__(k) + raise KeyError(key) + + def __setitem__(self, key, value): + norm_key = self._normalize(key) + for k in list(self.keys()): + if self._normalize(k) == norm_key: + super().__delitem__(k) + super().__setitem__(key, value) + + def get(self, key, default=None): + try: + return self[key] + except KeyError: + return default + + def __contains__(self, key): + norm_key = self._normalize(key) + for k in self.keys(): + if self._normalize(k) == norm_key: + return True + return False + +# ------------------------------------------------------------------------ + + +def _sanitize_col(name: str) -> str: + return name.strip().replace(" ", "_").replace("(", "").replace(")", "").replace("/", "_").replace("-", "_") + +async def import_csv_to_sqlite(csv_path: str, db_path: str, table_name: str, index_column: Optional[str] = None) -> None: + """Parses CSV chunk-by-chunk and inserts it into SQLite database cache asynchronously.""" + if not os.path.exists(csv_path): + logger.warning(f"CSV file not found for import: {csv_path}") + return + + logger.info(f"Starting async SQLite import of {csv_path} to table {table_name}...") + + # Read headers + with open(csv_path, 'r', encoding='utf-8') as f: + reader = csv.reader(f) + headers = next(reader, None) + if not headers: + logger.warning(f"Empty CSV headers for: {csv_path}") + return + + sanitized_headers = [_sanitize_col(h) for h in headers] + cols_def = ", ".join(f"[{h}] TEXT" for h in sanitized_headers) + + async with aiosqlite.connect(db_path) as db: + await db.execute(f"DROP TABLE IF EXISTS {table_name}") + await db.execute(f"CREATE TABLE {table_name} ({cols_def})") + await db.commit() + + insert_sql = f"INSERT INTO {table_name} VALUES ({', '.join('?' for _ in sanitized_headers)})" + + with open(csv_path, 'r', encoding='utf-8') as f: + reader = csv.reader(f) + next(reader, None) # skip header + + chunk = [] + for row in reader: + if row: + # Pad or truncate row to match headers count to avoid sqlite schema mismatches + if len(row) < len(headers): + row = row + [""] * (len(headers) - len(row)) + elif len(row) > len(headers): + row = row[:len(headers)] + chunk.append(row) + if len(chunk) >= 5000: + await db.executemany(insert_sql, chunk) + await db.commit() + chunk = [] + + if chunk: + await db.executemany(insert_sql, chunk) + await db.commit() + + if index_column: + sanitized_idx = _sanitize_col(index_column) + if sanitized_idx in sanitized_headers: + logger.info(f"Creating SQL Index on column {sanitized_idx} of table {table_name}...") + await db.execute(f"CREATE INDEX IF NOT EXISTS idx_{table_name}_{sanitized_idx} ON {table_name}([{sanitized_idx}])") + await db.commit() + + logger.info(f"Successfully completed importing {csv_path} into table {table_name}.") + +async def query_page_async( + db_path: str, + table_name: str, + page_idx: int, + page_size: int, + search_filter: Optional[str] = None, + search_column: Optional[str] = None +) -> Tuple[List[Dict[str, Any]], int]: + """Retrieves a single paginated chunk of rows asynchronously and counts the total matching entries.""" + if not os.path.exists(db_path): + return [], 0 + + offset = page_idx * page_size + where_clause = "" + params = [] + + if search_filter and search_column: + sanitized_col = _sanitize_col(search_column) + where_clause = f" WHERE [{sanitized_col}] LIKE ?" + params.append(f"%{search_filter}%") + + async with aiosqlite.connect(db_path) as db: + db.row_factory = aiosqlite.Row + + # 1. Get total count + count_sql = f"SELECT COUNT(*) FROM {table_name}{where_clause}" + async with db.execute(count_sql, params) as cursor: + row = await cursor.fetchone() + total_count = row[0] if row else 0 + + # 2. Get rows page + query_sql = f"SELECT * FROM {table_name}{where_clause} LIMIT ? OFFSET ?" + query_params = params + [page_size, offset] + + async with db.execute(query_sql, query_params) as cursor: + rows = await cursor.fetchall() + items = [CaseInsensitiveDict(dict(r)) for r in rows] + + return items, total_count + +def query_page_sync( + db_path: str, + table_name: str, + page_idx: int, + page_size: int, + search_filter: Optional[str] = None, + search_column: Optional[str] = None +) -> Tuple[List[Dict[str, Any]], int]: + """Retrieves a single paginated chunk of rows synchronously using built-in sqlite3 client.""" + import sqlite3 + if not os.path.exists(db_path): + return [], 0 + + offset = page_idx * page_size + where_clause = "" + params = [] + + if search_filter and search_column: + sanitized_col = _sanitize_col(search_column) + where_clause = f" WHERE [{sanitized_col}] LIKE ?" + params.append(f"%{search_filter}%") + + conn = sqlite3.connect(db_path) + try: + conn.row_factory = sqlite3.Row + cursor = conn.cursor() + + # 1. Get total count + count_sql = f"SELECT COUNT(*) FROM {table_name}{where_clause}" + cursor.execute(count_sql, params) + row = cursor.fetchone() + total_count = row[0] if row else 0 + + # 2. Get rows page + query_sql = f"SELECT * FROM {table_name}{where_clause} LIMIT ? OFFSET ?" + query_params = params + [page_size, offset] + + cursor.execute(query_sql, query_params) + rows = cursor.fetchall() + items = [CaseInsensitiveDict(dict(r)) for r in rows] + + return items, total_count + finally: + conn.close() + diff --git a/core/graph/delegated_auth.py b/core/graph/delegated_auth.py new file mode 100644 index 00000000..37e87242 --- /dev/null +++ b/core/graph/delegated_auth.py @@ -0,0 +1,112 @@ +import msal +import logging +import threading +import socket +from typing import List, Optional + +from msal.oauth2cli.authcode import AuthCodeReceiver + +logger = logging.getLogger(__name__) + +def get_free_port(): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.bind(('localhost', 0)) + port = s.getsockname()[1] + s.close() + return port + +class DelegatedAuthClient: + """Manages Microsoft Graph API authentication using MSAL native capabilities.""" + + # Shared in-memory token cache to persist tokens across UI reloads + # without writing sensitive tokens to disk. + _shared_token_cache = msal.TokenCache() + + def __init__(self, tenant_id: str, client_id: str, client_secret: str): + self.tenant_id = tenant_id + self.client_id = client_id + self.client_secret = client_secret + self.authority = f"https://login.microsoftonline.com/{tenant_id}" + + try: + self.app = msal.ConfidentialClientApplication( + client_id, + client_credential=client_secret, + authority=self.authority, + token_cache=self._shared_token_cache + ) + except Exception as e: + if "Unable to get authority configuration for" in str(e): + raise Exception("Incorrect Tenant ID.") + raise e + + self._lock = threading.Lock() + + @staticmethod + def _parse_error(desc: str) -> str: + if "AADSTS7000215" in desc or "AADSTS7000222" in desc: + return "Incorrect Client Secret." + if "AADSTS700016" in desc: + return "Incorrect Client ID." + if "AADSTS90002" in desc or "AADSTS900023" in desc: + return "Incorrect Tenant ID." + return desc.split('\n')[0].split('\r')[0] if desc else "Unknown error." + + def get_token(self, scopes: List[str], force_interactive: bool = False) -> Optional[str]: + try: + with self._lock: + result = None + accounts = self.app.get_accounts() + + if accounts and not force_interactive: + result = self.app.acquire_token_silent(scopes, account=accounts[0]) + + if not result: + logger.info("No valid token found in cache. Using MSAL native AuthCodeReceiver popup flow.") + + port = get_free_port() + redirect_uri = f"http://localhost:{port}" + + auth_url = self.app.get_authorization_request_url( + scopes, + redirect_uri=redirect_uri, + prompt="select_account" + ) + auth_url += "&response_mode=form_post" + + with AuthCodeReceiver(port=port) as receiver: + auth_response = receiver.get_auth_response( + auth_uri=auth_url, + timeout=120 + ) + + if not auth_response: + raise Exception("Delegated Auth Cancelled or timed out waiting for browser popup.") + + if "error" in auth_response: + graceful_err = self._parse_error(auth_response.get('error_description', '')) + logger.error(f"Delegated Auth Failed in browser: {auth_response['error']} - {graceful_err}") + raise Exception(f"Delegated Auth Failed: {graceful_err}") + + if "code" not in auth_response: + raise Exception("Delegated Auth Failed: No authorization code received.") + + auth_code = auth_response["code"] + + result = self.app.acquire_token_by_authorization_code( + auth_code, + scopes=scopes, + redirect_uri=redirect_uri + ) + + if "access_token" in result: + return result["access_token"] + else: + graceful_err = self._parse_error(result.get('error_description', '')) + logger.error(f"Failed to acquire delegated token: {result.get('error')} - {graceful_err}") + raise Exception(f"Delegated Auth Failed: {graceful_err}") + except Exception as e: + err_msg = str(e) + if "Unable to get authority configuration for" in err_msg: + raise Exception("Incorrect Tenant ID.") + raise e diff --git a/core/graph/directory.py b/core/graph/directory.py new file mode 100644 index 00000000..33f3d67f --- /dev/null +++ b/core/graph/directory.py @@ -0,0 +1,572 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""DirectoryService for querying tenant details and subscribed SKUs config.""" + +import os +import csv +import logging +from typing import Dict, Any +from core.graph.client import GraphClient + +logger = logging.getLogger(__name__) + +class DirectoryService: + """Service to query Entra ID directory configuration details.""" + + def __init__(self, client: GraphClient) -> None: + self.client = client + + def get_subscribed_skus(self) -> Dict[str, Any]: + """Queries the Microsoft Graph /subscribedSkus endpoint with active retries.""" + token_slot = self.client.get_active_token() + session = self.client.get_session() + + headers = { + "Authorization": f"Bearer {token_slot['token']}", + "ConsistencyLevel": "eventual" + } + try: + url = "https://graph.microsoft.com/v1.0/subscribedSkus" + logger.info("Querying Graph API configuration endpoint: %s", url) + resp = session.get(url, headers=headers, timeout=30.0) + resp.raise_for_status() + return resp.json() + finally: + self.client.release_token(token_slot) + + def get_tenant_primary_domain(self) -> str: + """Queries the /organization endpoint to retrieve the tenant's default or initial domain name.""" + token_slot = self.client.get_active_token() + session = self.client.get_session() + + headers = { + "Authorization": f"Bearer {token_slot['token']}" + } + try: + url = "https://graph.microsoft.com/v1.0/organization" + logger.info("Querying Graph API organization endpoint: %s", url) + resp = session.get(url, headers=headers, timeout=30.0) + resp.raise_for_status() + data = resp.json() + values = data.get("value", []) + if not values: + raise ValueError("No organization details found in Microsoft Graph.") + + domains = values[0].get("verifiedDomains", []) + + # 1. Find default domain + for d in domains: + if d.get("isDefault"): + return d.get("name") + + # 2. Fallback to initial domain (.onmicrosoft.com) + for d in domains: + if d.get("isInitial"): + return d.get("name") + + # 3. Fallback to first verified domain + if domains: + return domains[0].get("name") + + raise ValueError("No verified domains found in organization details.") + finally: + self.client.release_token(token_slot) + + def get_directory_telemetry(self, log_callback=None) -> Dict[str, Any]: + """Queries Microsoft Graph API in a single batch to fetch both domains list, user counts, and group counts.""" + logger.info("Fetching directory telemetry data using Graph API batch...") + if log_callback: + log_callback("Querying Microsoft Graph API for directory domains, users, and groups...") + + batch_requests = [ + { + "id": "organization", + "method": "GET", + "url": "/organization", + }, + { + "id": "domains", + "method": "GET", + "url": "/domains", + }, + { + "id": "total", + "method": "GET", + "url": "/groups?$count=true&$top=1&$select=id", + "headers": {"ConsistencyLevel": "eventual"} + }, + { + "id": "security", + "method": "GET", + "url": "/groups?$filter=securityEnabled eq true and mailEnabled eq false&$count=true&$top=1&$select=id", + "headers": {"ConsistencyLevel": "eventual"} + }, + { + "id": "distribution", + "method": "GET", + "url": "/groups?$filter=mailEnabled eq true and securityEnabled eq false and NOT groupTypes/any(c:c eq 'Unified')&$count=true&$top=1&$select=id", + "headers": {"ConsistencyLevel": "eventual"} + }, + { + "id": "mail_enabled_security", + "method": "GET", + "url": "/groups?$filter=mailEnabled eq true and securityEnabled eq true and NOT groupTypes/any(c:c eq 'Unified')&$count=true&$top=1&$select=id", + "headers": {"ConsistencyLevel": "eventual"} + }, + { + "id": "m365", + "method": "GET", + "url": "/groups?$filter=groupTypes/any(c:c eq 'Unified')&$count=true&$top=1&$select=id", + "headers": {"ConsistencyLevel": "eventual"} + }, + { + "id": "dynamic", + "method": "GET", + "url": "/groups?$filter=groupTypes/any(s:s eq 'DynamicMembership')&$count=true&$top=1&$select=id", + "headers": {"ConsistencyLevel": "eventual"} + }, + { + "id": "users_total", + "method": "GET", + "url": "/users?$count=true&$top=1&$select=id", + "headers": {"ConsistencyLevel": "eventual"} + }, + { + "id": "users_enabled", + "method": "GET", + "url": "/users?$filter=accountEnabled eq true&$count=true&$top=1&$select=id", + "headers": {"ConsistencyLevel": "eventual"} + }, + { + "id": "users_disabled", + "method": "GET", + "url": "/users?$filter=accountEnabled eq false&$count=true&$top=1&$select=id", + "headers": {"ConsistencyLevel": "eventual"} + }, + { + "id": "users_member", + "method": "GET", + "url": "/users?$filter=userType eq 'Member'&$count=true&$top=1&$select=id", + "headers": {"ConsistencyLevel": "eventual"} + }, + { + "id": "users_guest", + "method": "GET", + "url": "/users?$filter=userType eq 'Guest'&$count=true&$top=1&$select=id", + "headers": {"ConsistencyLevel": "eventual"} + } + ] + + # Invoke the batch query via client's UrlInvoker + responses = self.client.url_invoker.invoke( + url="https://graph.microsoft.com/v1.0", + batch=batch_requests, + logger=log_callback or (lambda x: None), + context="DirectoryTelemetry" + ) + + organization_list = [] + domains_list = [] + counts = { + "total": 0, + "security": 0, + "distribution": 0, + "mail_enabled_security": 0, + "m365": 0, + "dynamic": 0 + } + + user_counts = { + "users_total": 0, + "users_enabled": 0, + "users_disabled": 0, + "users_member": 0, + "users_guest": 0 + } + + for resp in responses: + resp_id = resp.get("id") + if resp.get("status", 0) != 200: + error_msg = resp.get("body", {}).get("error", {}).get("message", "Unknown error") + logger.error("Failed to fetch directory telemetry for %s: status %s, message: %s", resp_id, resp.get("status"), error_msg) + raise Exception(f"Failed to fetch directory telemetry for '{resp_id}': {error_msg}") + + body = resp.get("body", {}) + if resp_id == "organization": + organization_list = body.get("value", []) + elif resp_id == "domains": + domains_list = body.get("value", []) + elif resp_id in counts: + count_val = body.get("@odata.count", 0) + counts[resp_id] = count_val + elif resp_id in user_counts: + count_val = body.get("@odata.count", 0) + user_counts[resp_id] = count_val + + # Fetch federation configuration details for federated domains + federated_domains = [d for d in domains_list if str(d.get("authenticationType", "")).lower() == "federated"] + if federated_domains: + token_slot = self.client.get_active_token() + session = self.client.get_session() + headers = { + "Authorization": f"Bearer {token_slot['token']}", + "Accept": "application/json" + } + try: + for domain in federated_domains: + domain_id = domain.get("id") + fed_url = f"https://graph.microsoft.com/v1.0/domains/{domain_id}/federationConfiguration" + logger.info("Fetching federation configuration for domain: %s", domain_id) + try: + fed_resp = session.get(fed_url, headers=headers, timeout=30.0) + if fed_resp.status_code == 200: + fed_vals = fed_resp.json().get("value", []) + if fed_vals: + domain["federationDisplayName"] = fed_vals[0].get("displayName") or "N/A" + domain["federationIssuerUri"] = fed_vals[0].get("issuerUri") or "N/A" + else: + domain["federationDisplayName"] = "N/A" + domain["federationIssuerUri"] = "N/A" + else: + try: + err_msg = fed_resp.json().get("error", {}).get("message", f"HTTP {fed_resp.status_code}") + except Exception: + err_msg = f"HTTP {fed_resp.status_code}" + if len(err_msg) > 50: + err_msg = err_msg[:47] + "..." + domain["federationDisplayName"] = err_msg + domain["federationIssuerUri"] = err_msg + except Exception as err: + logger.warning("Error fetching federation configuration for %s: %s", domain_id, err) + err_msg = str(err) + if "timeout" in err_msg.lower(): + err_msg = "Timeout Error" + elif "connection" in err_msg.lower(): + err_msg = "Connection Error" + else: + err_msg = "Request Error" + domain["federationDisplayName"] = err_msg + domain["federationIssuerUri"] = err_msg + finally: + self.client.release_token(token_slot) + + # Normalize user counts dictionary keys for the UI + normalized_user_counts = { + "total": user_counts["users_total"], + "enabled": user_counts["users_enabled"], + "disabled": user_counts["users_disabled"], + "member": user_counts["users_member"], + "guest": user_counts["users_guest"] + } + + return { + "organization": organization_list, + "domains": domains_list, + "group_counts": counts, + "user_counts": normalized_user_counts + } + + def fetch_service_principals_sso(self, csv_path: str = None, on_page_callback=None, is_cancelled_callback=None) -> None: + """Fetches all Service Principals (Enterprise Apps) and their SSO modes, streaming to CSV.""" + logger.info("Fetching Service Principals and SSO modes...") + token_slot = self.client.get_active_token() + session = self.client.get_session() + headers = { + "Authorization": f"Bearer {token_slot['token']}", + "ConsistencyLevel": "eventual" + } + + url = "https://graph.microsoft.com/v1.0/servicePrincipals?$select=id,appId,displayName,preferredSingleSignOnMode" + + try: + import csv, os + if csv_path: + f = open(csv_path, 'w', encoding='utf-8', newline='') + writer = csv.writer(f) + writer.writerow(["displayName", "preferredSingleSignOnMode"]) + else: + f = None + writer = None + + while url: + if is_cancelled_callback and is_cancelled_callback(): break + resp = session.get(url, headers=headers, timeout=30.0) + resp.raise_for_status() + data = resp.json() + value_list = data.get("value", []) + + if writer: + for sp in value_list: + writer.writerow([ + sp.get("displayName", ""), + sp.get("preferredSingleSignOnMode", "") + ]) + + if on_page_callback: + on_page_callback(value_list) + + url = data.get("@odata.nextLink") + finally: + if 'f' in locals() and f: f.close() + self.client.release_token(token_slot) + + def fetch_user_creation_logs(self, csv_path: str, max_rows: int = 50, on_page_callback=None, is_cancelled_callback=None) -> None: + """Queries Microsoft Graph API /auditLogs/directoryAudits to fetch successful Add user and Delete user logs, + flattens, and appends/saves to CSV. + """ + token_slot = self.client.get_active_token() + session = self.client.get_session() + + headers = { + "Authorization": f"Bearer {token_slot['token']}", + "Accept": "application/json" + } + + urls = [ + ("Add user", "https://graph.microsoft.com/v1.0/auditLogs/directoryAudits?$select=activityDisplayName,initiatedBy&$filter=activityDisplayName eq 'Add user' and result eq 'success'&$top=50"), + ("Delete user", "https://graph.microsoft.com/v1.0/auditLogs/directoryAudits?$select=activityDisplayName,initiatedBy&$filter=activityDisplayName eq 'Delete user' and result eq 'success'&$top=50") + ] + + import csv + + rows_written = 0 + try: + logger.info("Starting User Creation logs fetch...") + # We want to overwrite or initialize the CSV file with headers if it's the start + with open(csv_path, 'w', encoding='utf-8', newline='') as f: + writer = csv.writer(f) + writer.writerow(["Activity", "Initiated By"]) + + with open(csv_path, 'a', encoding='utf-8', newline='') as f: + writer = csv.writer(f) + + try: + for activity_type, url in urls: + if is_cancelled_callback and is_cancelled_callback(): + logger.info("User Creation logs fetch cancelled in-flight.") + break + + next_url = url + activity_rows_written = 0 + + while next_url and activity_rows_written < max_rows: + if is_cancelled_callback and is_cancelled_callback(): + break + + logger.info("Querying MSFT Graph directory audits for '%s'...", activity_type) + try: + resp = session.get(next_url, headers=headers, timeout=60.0) + except Exception as get_err: + logger.warning("Query attempt for '%s' failed with exception: %s. Displaying data obtained till now.", activity_type, get_err) + break + + if not resp or resp.status_code != 200: + if resp and resp.status_code in [401, 403]: + logger.error("Directory audits endpoint permission error: %d %s", resp.status_code, resp.text) + raise PermissionError("AuditLog.Read.All permission required. Please ensure this permission is granted to the application registration in Microsoft Entra ID.") + else: + status_str = f"status {resp.status_code}" if resp else "connection/timeout error" + logger.warning("Directory audits query for '%s' failed (%s). Displaying data obtained till now.", activity_type, status_str) + break + + data = resp.json() + value_list = data.get("value", []) + + page_rows = [] + for log in value_list: + activity = log.get("activityDisplayName") or "" + initiated_by_obj = log.get("initiatedBy") or {} + + import json + initiated_by_str = json.dumps(initiated_by_obj) + + writer.writerow([activity, initiated_by_str]) + activity_rows_written += 1 + rows_written += 1 + + page_rows.append({ + "activity": activity, + "initiatedBy": initiated_by_str + }) + + if activity_rows_written >= max_rows: + break + + if on_page_callback: + try: + on_page_callback(page_rows) + except Exception as cb_err: + logger.warning("Error in User Creation logs page callback: %s", cb_err) + + if activity_rows_written >= max_rows: + break + + next_url = data.get("@odata.nextLink") + except PermissionError as pe: + logger.error("Permission error during User Creation logs query: %s", pe) + with open(csv_path, 'w', encoding='utf-8', newline='') as f_err: + w_err = csv.writer(f_err) + w_err.writerow(["Activity", "Initiated By"]) + w_err.writerow(["ERROR", str(pe)]) + if on_page_callback: + on_page_callback([{"activity": "ERROR", "initiatedBy": str(pe)}]) + except Exception as ex: + logger.error("Unexpected error during User Creation logs query: %s", ex) + with open(csv_path, 'w', encoding='utf-8', newline='') as f_err: + w_err = csv.writer(f_err) + w_err.writerow(["Activity", "Initiated By"]) + w_err.writerow(["ERROR", f"Failed to query User Creation logs: {ex}"]) + if on_page_callback: + on_page_callback([{"activity": "ERROR", "initiatedBy": f"Failed to query User Creation logs: {ex}"}]) + + logger.info("Finished fetching User Creation logs. Rows written: %d", rows_written) + finally: + self.client.release_token(token_slot) + + def fetch_provisioning_logs(self, csv_path: str, max_rows: int = 200, on_page_callback=None, is_cancelled_callback=None) -> None: + """Queries Microsoft Graph API /auditLogs/provisioning to fetch provisioning logs, + flattens, and appends/saves to CSV. + """ + token_slot = self.client.get_active_token() + session = self.client.get_session() + + headers = { + "Authorization": f"Bearer {token_slot['token']}", + "Accept": "application/json" + } + + url = "https://graph.microsoft.com/v1.0/auditLogs/provisioning?$select=initiatedBy,provisioningAction,provisioningSteps,servicePrincipal,sourceSystem,targetSystem,tenantId,provisioningStatusInfo&$top=100" + + import csv + import json + + def to_raw_str(val): + if val is None: + return "" + if isinstance(val, (dict, list)): + return json.dumps(val) + return str(val) + + rows_written = 0 + try: + logger.info("Starting Provisioning logs fetch...") + # We want to overwrite or initialize the CSV file with headers if it's the start + with open(csv_path, 'w', encoding='utf-8', newline='') as f: + writer = csv.writer(f) + writer.writerow(["initiatedBy", "provisioningAction", "provisioningSteps", "servicePrincipal", "sourceSystem", "targetSystem", "tenantId", "provisioningStatusInfo"]) + + with open(csv_path, 'a', encoding='utf-8', newline='') as f: + writer = csv.writer(f) + + next_url = url + while next_url and rows_written < max_rows: + if is_cancelled_callback and is_cancelled_callback(): + logger.info("Provisioning logs fetch cancelled in-flight.") + break + + logger.info("Querying MSFT Graph directory provisioning logs...") + try: + resp = session.get(next_url, headers=headers, timeout=60.0) + except Exception as get_err: + logger.warning("Query attempt failed with exception: %s. Displaying data obtained till now.", get_err) + break + + if not resp or resp.status_code != 200: + if resp and resp.status_code in [401, 403]: + logger.error("Provisioning audits endpoint permission error: %d %s", resp.status_code, resp.text) + raise PermissionError("AuditLog.Read.All permission required. Please ensure this permission is granted to the application registration in Microsoft Entra ID.") + else: + status_str = f"status {resp.status_code}" if resp else "connection/timeout error" + logger.warning("Provisioning audits query failed (%s). Displaying data obtained till now.", status_str) + break + + data = resp.json() + value_list = data.get("value", []) + + page_rows = [] + for log in value_list: + initiated_by = to_raw_str(log.get("initiatedBy")) + action = to_raw_str(log.get("provisioningAction")) + steps = to_raw_str(log.get("provisioningSteps")) + service_principal = to_raw_str(log.get("servicePrincipal")) + source_system = to_raw_str(log.get("sourceSystem")) + target_system = to_raw_str(log.get("targetSystem")) + tenant_id = to_raw_str(log.get("tenantId")) + status_info = to_raw_str(log.get("provisioningStatusInfo")) + + writer.writerow([initiated_by, action, steps, service_principal, source_system, target_system, tenant_id, status_info]) + rows_written += 1 + + page_rows.append({ + "initiatedBy": initiated_by, + "provisioningAction": action, + "provisioningSteps": steps, + "servicePrincipal": service_principal, + "sourceSystem": source_system, + "targetSystem": target_system, + "tenantId": tenant_id, + "provisioningStatusInfo": status_info + }) + + if rows_written >= max_rows: + break + + if on_page_callback: + try: + on_page_callback(page_rows) + except Exception as cb_err: + logger.warning("Error in Provisioning logs page callback: %s", cb_err) + + if rows_written >= max_rows: + break + + next_url = data.get("@odata.nextLink") + except PermissionError as pe: + logger.error("Permission error during Provisioning logs query: %s", pe) + with open(csv_path, 'w', encoding='utf-8', newline='') as f_err: + w_err = csv.writer(f_err) + w_err.writerow(["initiatedBy", "provisioningAction", "provisioningSteps", "servicePrincipal", "sourceSystem", "targetSystem", "tenantId", "provisioningStatusInfo"]) + w_err.writerow(["ERROR", str(pe), "", "", "", "", "", ""]) + if on_page_callback: + on_page_callback([{ + "initiatedBy": "ERROR", + "provisioningAction": str(pe), + "provisioningSteps": "", + "servicePrincipal": "", + "sourceSystem": "", + "targetSystem": "", + "tenantId": "", + "provisioningStatusInfo": "" + }]) + except Exception as ex: + logger.error("Unexpected error during Provisioning logs query: %s", ex) + with open(csv_path, 'w', encoding='utf-8', newline='') as f_err: + w_err = csv.writer(f_err) + w_err.writerow(["initiatedBy", "provisioningAction", "provisioningSteps", "servicePrincipal", "sourceSystem", "targetSystem", "tenantId", "provisioningStatusInfo"]) + w_err.writerow(["ERROR", f"Failed to query Provisioning logs: {ex}", "", "", "", "", "", ""]) + if on_page_callback: + on_page_callback([{ + "initiatedBy": "ERROR", + "provisioningAction": f"Failed to query Provisioning logs: {ex}", + "provisioningSteps": "", + "servicePrincipal": "", + "sourceSystem": "", + "targetSystem": "", + "tenantId": "", + "provisioningStatusInfo": "" + }]) + + logger.info("Finished fetching Provisioning logs. Rows written: %d", rows_written) + self.client.release_token(token_slot) diff --git a/core/graph/directory/__init__.py b/core/graph/directory/__init__.py new file mode 100644 index 00000000..89d06ac6 --- /dev/null +++ b/core/graph/directory/__init__.py @@ -0,0 +1,260 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""DirectoryService backward compatibility facade and sub-module exports.""" + +import logging +from typing import Dict, Any, List + +from core.graph.client import GraphClient +from core.graph.directory.organization import OrganizationService +from core.graph.directory.domains import DomainsService +from core.graph.directory.user_logs import UserLogsService +from core.graph.directory.provisioning_logs import ProvisioningLogsService +from core.graph.directory.users_groups import UsersGroupsService +from core.graph.directory.subscribed_skus import SubscribedSKUsService +from core.graph.directory.service_principals import ServicePrincipalsService + +logger = logging.getLogger(__name__) + +class DirectoryService: + """Facade for querying Entra ID directory configuration details (for backward compatibility).""" + + def __init__(self, client: GraphClient) -> None: + self.client = client + + def get_subscribed_skus(self) -> Dict[str, Any]: + return SubscribedSKUsService(self.client).get_subscribed_skus() + + def get_tenant_primary_domain(self) -> str: + return OrganizationService(self.client).get_tenant_primary_domain() + + def get_directory_telemetry(self, log_callback=None) -> Dict[str, Any]: + """Queries Microsoft Graph API in a single batch to fetch both domains list, user counts, and group counts. + Maintained for exact backward compatibility with test suites and batch invocations. + """ + logger.info("Fetching directory telemetry data using Graph API batch...") + if log_callback: + log_callback("Querying Microsoft Graph API for directory domains, users, and groups...") + + batch_requests = [ + { + "id": "organization", + "method": "GET", + "url": "/organization", + }, + { + "id": "domains", + "method": "GET", + "url": "/domains", + }, + { + "id": "total", + "method": "GET", + "url": "/groups?$count=true&$top=1&$select=id", + "headers": {"ConsistencyLevel": "eventual"} + }, + { + "id": "security", + "method": "GET", + "url": "/groups?$filter=securityEnabled eq true and mailEnabled eq false&$count=true&$top=1&$select=id", + "headers": {"ConsistencyLevel": "eventual"} + }, + { + "id": "distribution", + "method": "GET", + "url": "/groups?$filter=mailEnabled eq true and securityEnabled eq false and NOT groupTypes/any(c:c eq 'Unified')&$count=true&$top=1&$select=id", + "headers": {"ConsistencyLevel": "eventual"} + }, + { + "id": "mail_enabled_security", + "method": "GET", + "url": "/groups?$filter=mailEnabled eq true and securityEnabled eq true and NOT groupTypes/any(c:c eq 'Unified')&$count=true&$top=1&$select=id", + "headers": {"ConsistencyLevel": "eventual"} + }, + { + "id": "m365", + "method": "GET", + "url": "/groups?$filter=groupTypes/any(c:c eq 'Unified')&$count=true&$top=1&$select=id", + "headers": {"ConsistencyLevel": "eventual"} + }, + { + "id": "dynamic", + "method": "GET", + "url": "/groups?$filter=groupTypes/any(s:s eq 'DynamicMembership')&$count=true&$top=1&$select=id", + "headers": {"ConsistencyLevel": "eventual"} + }, + { + "id": "users_total", + "method": "GET", + "url": "/users?$count=true&$top=1&$select=id", + "headers": {"ConsistencyLevel": "eventual"} + }, + { + "id": "users_enabled", + "method": "GET", + "url": "/users?$filter=accountEnabled eq true&$count=true&$top=1&$select=id", + "headers": {"ConsistencyLevel": "eventual"} + }, + { + "id": "users_disabled", + "method": "GET", + "url": "/users?$filter=accountEnabled eq false&$count=true&$top=1&$select=id", + "headers": {"ConsistencyLevel": "eventual"} + }, + { + "id": "users_member", + "method": "GET", + "url": "/users?$filter=userType eq 'Member'&$count=true&$top=1&$select=id", + "headers": {"ConsistencyLevel": "eventual"} + }, + { + "id": "users_guest", + "method": "GET", + "url": "/users?$filter=userType eq 'Guest'&$count=true&$top=1&$select=id", + "headers": {"ConsistencyLevel": "eventual"} + } + ] + + # Invoke the batch query via client's UrlInvoker + responses = self.client.url_invoker.invoke( + url="https://graph.microsoft.com/v1.0", + batch=batch_requests, + logger=log_callback or (lambda x: None), + context="DirectoryTelemetry" + ) + + organization_list = [] + domains_list = [] + counts = { + "total": 0, + "security": 0, + "distribution": 0, + "mail_enabled_security": 0, + "m365": 0, + "dynamic": 0 + } + + user_counts = { + "users_total": 0, + "users_enabled": 0, + "users_disabled": 0, + "users_member": 0, + "users_guest": 0 + } + + for resp in responses: + resp_id = resp.get("id") + if resp.get("status", 0) != 200: + error_msg = resp.get("body", {}).get("error", {}).get("message", "Unknown error") + logger.error("Failed to fetch directory telemetry for %s: status %s, message: %s", resp_id, resp.get("status"), error_msg) + raise Exception(f"Failed to fetch directory telemetry for '{resp_id}': {error_msg}") + + body = resp.get("body", {}) + if resp_id == "organization": + organization_list = body.get("value", []) + elif resp_id == "domains": + domains_list = body.get("value", []) + elif resp_id in counts: + count_val = body.get("@odata.count", 0) + counts[resp_id] = count_val + elif resp_id in user_counts: + count_val = body.get("@odata.count", 0) + user_counts[resp_id] = count_val + + # Fetch federation configuration details for federated domains + federated_domains = [d for d in domains_list if str(d.get("authenticationType", "")).lower() == "federated"] + if federated_domains: + token_slot = self.client.get_active_token() + session = self.client.get_session() + headers = { + "Authorization": f"Bearer {token_slot['token']}", + "Accept": "application/json" + } + try: + for domain in federated_domains: + domain_id = domain.get("id") + fed_url = f"https://graph.microsoft.com/v1.0/domains/{domain_id}/federationConfiguration" + logger.info("Fetching federation configuration for domain: %s", domain_id) + try: + fed_resp = session.get(fed_url, headers=headers, timeout=30.0) + if fed_resp.status_code == 200: + fed_vals = fed_resp.json().get("value", []) + if fed_vals: + domain["federationDisplayName"] = fed_vals[0].get("displayName") or "N/A" + domain["federationIssuerUri"] = fed_vals[0].get("issuerUri") or "N/A" + else: + domain["federationDisplayName"] = "N/A" + domain["federationIssuerUri"] = "N/A" + else: + try: + err_msg = fed_resp.json().get("error", {}).get("message", f"HTTP {fed_resp.status_code}") + except Exception: + err_msg = f"HTTP {fed_resp.status_code}" + if len(err_msg) > 50: + err_msg = err_msg[:47] + "..." + domain["federationDisplayName"] = err_msg + domain["federationIssuerUri"] = err_msg + except Exception as err: + logger.warning("Error fetching federation configuration for %s: %s", domain_id, err) + err_msg = str(err) + if "timeout" in err_msg.lower(): + err_msg = "Timeout Error" + elif "connection" in err_msg.lower(): + err_msg = "Connection Error" + else: + err_msg = "Request Error" + domain["federationDisplayName"] = err_msg + domain["federationIssuerUri"] = err_msg + finally: + self.client.release_token(token_slot) + + # Normalize user counts dictionary keys for the UI + normalized_user_counts = { + "total": user_counts["users_total"], + "enabled": user_counts["users_enabled"], + "disabled": user_counts["users_disabled"], + "member": user_counts["users_member"], + "guest": user_counts["users_guest"] + } + + return { + "organization": organization_list, + "domains": domains_list, + "group_counts": counts, + "user_counts": normalized_user_counts + } + + def fetch_service_principals_sso(self, csv_path: str = None, on_page_callback=None, is_cancelled_callback=None) -> None: + ServicePrincipalsService(self.client).fetch_service_principals_sso( + csv_path=csv_path, + on_page_callback=on_page_callback, + is_cancelled_callback=is_cancelled_callback + ) + + def fetch_user_creation_logs(self, csv_path: str, max_rows: int = 50, on_page_callback=None, is_cancelled_callback=None) -> None: + UserLogsService(self.client).fetch_user_creation_logs( + csv_path=csv_path, + max_rows=max_rows, + on_page_callback=on_page_callback, + is_cancelled_callback=is_cancelled_callback + ) + + def fetch_provisioning_logs(self, csv_path: str, max_rows: int = 200, on_page_callback=None, is_cancelled_callback=None) -> None: + ProvisioningLogsService(self.client).fetch_provisioning_logs( + csv_path=csv_path, + max_rows=max_rows, + on_page_callback=on_page_callback, + is_cancelled_callback=is_cancelled_callback + ) diff --git a/core/graph/directory/domains.py b/core/graph/directory/domains.py new file mode 100644 index 00000000..2433eb4c --- /dev/null +++ b/core/graph/directory/domains.py @@ -0,0 +1,93 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Service for querying Entra ID domains list and federation configurations.""" + +import logging +from typing import List, Dict, Any +from core.graph.client import GraphClient + +logger = logging.getLogger(__name__) + +class DomainsService: + def __init__(self, client: GraphClient) -> None: + self.client = client + + def get_domains(self, log_callback=None) -> List[Dict[str, Any]]: + """Queries /domains and resolves federation configurations for federated domains.""" + if log_callback: + log_callback("Querying domains from Microsoft Graph...") + token_slot = self.client.get_active_token() + session = self.client.get_session() + + headers = { + "Authorization": f"Bearer {token_slot['token']}" + } + try: + url = "https://graph.microsoft.com/v1.0/domains" + logger.info("Querying Graph API domains endpoint: %s", url) + resp = session.get(url, headers=headers, timeout=30.0) + resp.raise_for_status() + domains_list = resp.json().get("value", []) + finally: + self.client.release_token(token_slot) + + # Fetch federation configuration details for federated domains + federated_domains = [d for d in domains_list if str(d.get("authenticationType", "")).lower() == "federated"] + if federated_domains: + token_slot = self.client.get_active_token() + session = self.client.get_session() + headers = { + "Authorization": f"Bearer {token_slot['token']}", + "Accept": "application/json" + } + try: + for domain in federated_domains: + domain_id = domain.get("id") + fed_url = f"https://graph.microsoft.com/v1.0/domains/{domain_id}/federationConfiguration" + logger.info("Fetching federation configuration for domain: %s", domain_id) + try: + fed_resp = session.get(fed_url, headers=headers, timeout=30.0) + if fed_resp.status_code == 200: + fed_vals = fed_resp.json().get("value", []) + if fed_vals: + domain["federationDisplayName"] = fed_vals[0].get("displayName") or "N/A" + domain["federationIssuerUri"] = fed_vals[0].get("issuerUri") or "N/A" + else: + domain["federationDisplayName"] = "N/A" + domain["federationIssuerUri"] = "N/A" + else: + try: + err_msg = fed_resp.json().get("error", {}).get("message", f"HTTP {fed_resp.status_code}") + except Exception: + err_msg = f"HTTP {fed_resp.status_code}" + if len(err_msg) > 50: + err_msg = err_msg[:47] + "..." + domain["federationDisplayName"] = err_msg + domain["federationIssuerUri"] = err_msg + except Exception as err: + logger.warning("Error fetching federation configuration for %s: %s", domain_id, err) + err_msg = str(err) + if "timeout" in err_msg.lower(): + err_msg = "Timeout Error" + elif "connection" in err_msg.lower(): + err_msg = "Connection Error" + else: + err_msg = "Request Error" + domain["federationDisplayName"] = err_msg + domain["federationIssuerUri"] = err_msg + finally: + self.client.release_token(token_slot) + + return domains_list diff --git a/core/graph/directory/organization.py b/core/graph/directory/organization.py new file mode 100644 index 00000000..ffd63682 --- /dev/null +++ b/core/graph/directory/organization.py @@ -0,0 +1,68 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Service for querying Entra ID organization configuration details.""" + +import logging +from typing import List, Dict, Any +from core.graph.client import GraphClient + +logger = logging.getLogger(__name__) + +class OrganizationService: + def __init__(self, client: GraphClient) -> None: + self.client = client + + def get_organization_info(self, log_callback=None) -> List[Dict[str, Any]]: + """Queries the Microsoft Graph /organization endpoint.""" + if log_callback: + log_callback("Querying organization details from Microsoft Graph...") + token_slot = self.client.get_active_token() + session = self.client.get_session() + + headers = { + "Authorization": f"Bearer {token_slot['token']}" + } + try: + url = "https://graph.microsoft.com/v1.0/organization" + logger.info("Querying Graph API organization endpoint: %s", url) + resp = session.get(url, headers=headers, timeout=30.0) + resp.raise_for_status() + return resp.json().get("value", []) + finally: + self.client.release_token(token_slot) + + def get_tenant_primary_domain(self) -> str: + """Queries the /organization endpoint to retrieve the tenant's default or initial domain name.""" + organization_list = self.get_organization_info() + if not organization_list: + raise ValueError("No organization details found in Microsoft Graph.") + + domains = organization_list[0].get("verifiedDomains", []) + + # 1. Find default domain + for d in domains: + if d.get("isDefault"): + return d.get("name") + + # 2. Fallback to initial domain (.onmicrosoft.com) + for d in domains: + if d.get("isInitial"): + return d.get("name") + + # 3. Fallback to first verified domain + if domains: + return domains[0].get("name") + + raise ValueError("No verified domains found in organization details.") diff --git a/core/graph/directory/provisioning_logs.py b/core/graph/directory/provisioning_logs.py new file mode 100644 index 00000000..28752773 --- /dev/null +++ b/core/graph/directory/provisioning_logs.py @@ -0,0 +1,159 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Service for querying Entra ID provisioning logs.""" + +import logging +import csv +import json +from core.graph.client import GraphClient + +logger = logging.getLogger(__name__) + +class ProvisioningLogsService: + def __init__(self, client: GraphClient) -> None: + self.client = client + + def fetch_provisioning_logs(self, csv_path: str, max_rows: int = 200, on_page_callback=None, is_cancelled_callback=None) -> None: + """Queries Microsoft Graph API /auditLogs/provisioning to fetch provisioning logs, + flattens, and appends/saves to CSV. + """ + token_slot = self.client.get_active_token() + session = self.client.get_session() + + headers = { + "Authorization": f"Bearer {token_slot['token']}", + "Accept": "application/json" + } + + url = "https://graph.microsoft.com/v1.0/auditLogs/provisioning?$select=initiatedBy,provisioningAction,provisioningSteps,servicePrincipal,sourceSystem,targetSystem,tenantId,provisioningStatusInfo&$top=100" + + def to_raw_str(val): + if val is None: + return "" + if isinstance(val, (dict, list)): + return json.dumps(val) + return str(val) + + rows_written = 0 + try: + logger.info("Starting Provisioning logs fetch...") + # Initialize/overwrite CSV + with open(csv_path, 'w', encoding='utf-8', newline='') as f: + writer = csv.writer(f) + writer.writerow(["initiatedBy", "provisioningAction", "provisioningSteps", "servicePrincipal", "sourceSystem", "targetSystem", "tenantId", "provisioningStatusInfo"]) + + with open(csv_path, 'a', encoding='utf-8', newline='') as f: + writer = csv.writer(f) + + next_url = url + while next_url and rows_written < max_rows: + if is_cancelled_callback and is_cancelled_callback(): + logger.info("Provisioning logs fetch cancelled in-flight.") + break + + logger.info("Querying MSFT Graph directory provisioning logs...") + try: + resp = session.get(next_url, headers=headers, timeout=60.0) + except Exception as get_err: + logger.warning("Query attempt failed with exception: %s. Displaying data obtained till now.", get_err) + break + + if not resp or resp.status_code != 200: + if resp and resp.status_code in [401, 403]: + logger.error("Provisioning audits endpoint permission error: %d %s", resp.status_code, resp.text) + raise PermissionError("AuditLog.Read.All permission required. Please ensure this permission is granted to the application registration in Microsoft Entra ID.") + else: + status_str = f"status {resp.status_code}" if resp else "connection/timeout error" + logger.warning("Provisioning audits query failed (%s). Displaying data obtained till now.", status_str) + break + + data = resp.json() + value_list = data.get("value", []) + + page_rows = [] + for log in value_list: + initiated_by = to_raw_str(log.get("initiatedBy")) + action = to_raw_str(log.get("provisioningAction")) + steps = to_raw_str(log.get("provisioningSteps")) + service_principal = to_raw_str(log.get("servicePrincipal")) + source_system = to_raw_str(log.get("sourceSystem")) + target_system = to_raw_str(log.get("targetSystem")) + tenant_id = to_raw_str(log.get("tenantId")) + status_info = to_raw_str(log.get("provisioningStatusInfo")) + + writer.writerow([initiated_by, action, steps, service_principal, source_system, target_system, tenant_id, status_info]) + rows_written += 1 + + page_rows.append({ + "initiatedBy": initiated_by, + "provisioningAction": action, + "provisioningSteps": steps, + "servicePrincipal": service_principal, + "sourceSystem": source_system, + "targetSystem": target_system, + "tenantId": tenant_id, + "provisioningStatusInfo": status_info + }) + + if rows_written >= max_rows: + break + + if on_page_callback: + try: + on_page_callback(page_rows) + except Exception as cb_err: + logger.warning("Error in Provisioning logs page callback: %s", cb_err) + + if rows_written >= max_rows: + break + + next_url = data.get("@odata.nextLink") + except PermissionError as pe: + logger.error("Permission error during Provisioning logs query: %s", pe) + with open(csv_path, 'w', encoding='utf-8', newline='') as f_err: + w_err = csv.writer(f_err) + w_err.writerow(["initiatedBy", "provisioningAction", "provisioningSteps", "servicePrincipal", "sourceSystem", "targetSystem", "tenantId", "provisioningStatusInfo"]) + w_err.writerow(["ERROR", str(pe), "", "", "", "", "", ""]) + if on_page_callback: + on_page_callback([{ + "initiatedBy": "ERROR", + "provisioningAction": str(pe), + "provisioningSteps": "", + "servicePrincipal": "", + "sourceSystem": "", + "targetSystem": "", + "tenantId": "", + "provisioningStatusInfo": "" + }]) + except Exception as ex: + logger.error("Unexpected error during Provisioning logs query: %s", ex) + with open(csv_path, 'w', encoding='utf-8', newline='') as f_err: + w_err = csv.writer(f_err) + w_err.writerow(["initiatedBy", "provisioningAction", "provisioningSteps", "servicePrincipal", "sourceSystem", "targetSystem", "tenantId", "provisioningStatusInfo"]) + w_err.writerow(["ERROR", f"Failed to query Provisioning logs: {ex}", "", "", "", "", "", ""]) + if on_page_callback: + on_page_callback([{ + "initiatedBy": "ERROR", + "provisioningAction": f"Failed to query Provisioning logs: {ex}", + "provisioningSteps": "", + "servicePrincipal": "", + "sourceSystem": "", + "targetSystem": "", + "tenantId": "", + "provisioningStatusInfo": "" + }]) + + logger.info("Finished fetching Provisioning logs. Rows written: %d", rows_written) + self.client.release_token(token_slot) diff --git a/core/graph/directory/service_principals.py b/core/graph/directory/service_principals.py new file mode 100644 index 00000000..560ca555 --- /dev/null +++ b/core/graph/directory/service_principals.py @@ -0,0 +1,68 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Service for querying Service Principals (Enterprise Apps) and SSO configuration details.""" + +import logging +import csv +from core.graph.client import GraphClient + +logger = logging.getLogger(__name__) + +class ServicePrincipalsService: + def __init__(self, client: GraphClient) -> None: + self.client = client + + def fetch_service_principals_sso(self, csv_path: str = None, on_page_callback=None, is_cancelled_callback=None) -> None: + """Fetches all Service Principals (Enterprise Apps) and their SSO modes, streaming to CSV.""" + logger.info("Fetching Service Principals and SSO modes...") + token_slot = self.client.get_active_token() + session = self.client.get_session() + headers = { + "Authorization": f"Bearer {token_slot['token']}", + "ConsistencyLevel": "eventual" + } + + url = "https://graph.microsoft.com/v1.0/servicePrincipals?$select=id,appId,displayName,preferredSingleSignOnMode" + + try: + if csv_path: + f = open(csv_path, 'w', encoding='utf-8', newline='') + writer = csv.writer(f) + writer.writerow(["displayName", "preferredSingleSignOnMode"]) + else: + f = None + writer = None + + while url: + if is_cancelled_callback and is_cancelled_callback(): break + resp = session.get(url, headers=headers, timeout=30.0) + resp.raise_for_status() + data = resp.json() + value_list = data.get("value", []) + + if writer: + for sp in value_list: + writer.writerow([ + sp.get("displayName", ""), + sp.get("preferredSingleSignOnMode", "") + ]) + + if on_page_callback: + on_page_callback(value_list) + + url = data.get("@odata.nextLink") + finally: + if 'f' in locals() and f: f.close() + self.client.release_token(token_slot) diff --git a/core/graph/directory/subscribed_skus.py b/core/graph/directory/subscribed_skus.py new file mode 100644 index 00000000..8a28d663 --- /dev/null +++ b/core/graph/directory/subscribed_skus.py @@ -0,0 +1,43 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Service for querying Entra ID subscribed SKUs.""" + +import logging +from typing import Dict, Any +from core.graph.client import GraphClient + +logger = logging.getLogger(__name__) + +class SubscribedSKUsService: + def __init__(self, client: GraphClient) -> None: + self.client = client + + def get_subscribed_skus(self) -> Dict[str, Any]: + """Queries the Microsoft Graph /subscribedSkus endpoint with active retries.""" + token_slot = self.client.get_active_token() + session = self.client.get_session() + + headers = { + "Authorization": f"Bearer {token_slot['token']}", + "ConsistencyLevel": "eventual" + } + try: + url = "https://graph.microsoft.com/v1.0/subscribedSkus" + logger.info("Querying Graph API configuration endpoint: %s", url) + resp = session.get(url, headers=headers, timeout=30.0) + resp.raise_for_status() + return resp.json() + finally: + self.client.release_token(token_slot) diff --git a/core/graph/directory/user_logs.py b/core/graph/directory/user_logs.py new file mode 100644 index 00000000..81ee9a7a --- /dev/null +++ b/core/graph/directory/user_logs.py @@ -0,0 +1,136 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Service for querying Entra ID user creation/deletion logs.""" + +import logging +import csv +import json +from core.graph.client import GraphClient + +logger = logging.getLogger(__name__) + +class UserLogsService: + def __init__(self, client: GraphClient) -> None: + self.client = client + + def fetch_user_creation_logs(self, csv_path: str, max_rows: int = 50, on_page_callback=None, is_cancelled_callback=None) -> None: + """Queries Microsoft Graph API /auditLogs/directoryAudits to fetch successful Add user and Delete user logs, + flattens, and appends/saves to CSV. + """ + token_slot = self.client.get_active_token() + session = self.client.get_session() + + headers = { + "Authorization": f"Bearer {token_slot['token']}", + "Accept": "application/json" + } + + urls = [ + ("Add user", "https://graph.microsoft.com/v1.0/auditLogs/directoryAudits?$select=activityDisplayName,initiatedBy&$filter=activityDisplayName eq 'Add user' and result eq 'success'&$top=50"), + ("Delete user", "https://graph.microsoft.com/v1.0/auditLogs/directoryAudits?$select=activityDisplayName,initiatedBy&$filter=activityDisplayName eq 'Delete user' and result eq 'success'&$top=50") + ] + + rows_written = 0 + try: + logger.info("Starting User Creation logs fetch...") + # Initialize/overwrite CSV + with open(csv_path, 'w', encoding='utf-8', newline='') as f: + writer = csv.writer(f) + writer.writerow(["Activity", "Initiated By"]) + + with open(csv_path, 'a', encoding='utf-8', newline='') as f: + writer = csv.writer(f) + + try: + for activity_type, url in urls: + if is_cancelled_callback and is_cancelled_callback(): + logger.info("User Creation logs fetch cancelled in-flight.") + break + + next_url = url + activity_rows_written = 0 + + while next_url and activity_rows_written < max_rows: + if is_cancelled_callback and is_cancelled_callback(): + break + + logger.info("Querying MSFT Graph directory audits for '%s'...", activity_type) + try: + resp = session.get(next_url, headers=headers, timeout=60.0) + except Exception as get_err: + logger.warning("Query attempt for '%s' failed with exception: %s. Displaying data obtained till now.", activity_type, get_err) + break + + if not resp or resp.status_code != 200: + if resp and resp.status_code in [401, 403]: + logger.error("Directory audits endpoint permission error: %d %s", resp.status_code, resp.text) + raise PermissionError("AuditLog.Read.All permission required. Please ensure this permission is granted to the application registration in Microsoft Entra ID.") + else: + status_str = f"status {resp.status_code}" if resp else "connection/timeout error" + logger.warning("Directory audits query for '%s' failed (%s). Displaying data obtained till now.", activity_type, status_str) + break + + data = resp.json() + value_list = data.get("value", []) + + page_rows = [] + for log in value_list: + activity = log.get("activityDisplayName") or "" + initiated_by_obj = log.get("initiatedBy") or {} + + initiated_by_str = json.dumps(initiated_by_obj) + + writer.writerow([activity, initiated_by_str]) + activity_rows_written += 1 + rows_written += 1 + + page_rows.append({ + "activity": activity, + "initiatedBy": initiated_by_str + }) + + if activity_rows_written >= max_rows: + break + + if on_page_callback: + try: + on_page_callback(page_rows) + except Exception as cb_err: + logger.warning("Error in User Creation logs page callback: %s", cb_err) + + if activity_rows_written >= max_rows: + break + + next_url = data.get("@odata.nextLink") + except PermissionError as pe: + logger.error("Permission error during User Creation logs query: %s", pe) + with open(csv_path, 'w', encoding='utf-8', newline='') as f_err: + w_err = csv.writer(f_err) + w_err.writerow(["Activity", "Initiated By"]) + w_err.writerow(["ERROR", str(pe)]) + if on_page_callback: + on_page_callback([{"activity": "ERROR", "initiatedBy": str(pe)}]) + except Exception as ex: + logger.error("Unexpected error during User Creation logs query: %s", ex) + with open(csv_path, 'w', encoding='utf-8', newline='') as f_err: + w_err = csv.writer(f_err) + w_err.writerow(["Activity", "Initiated By"]) + w_err.writerow(["ERROR", f"Failed to query User Creation logs: {ex}"]) + if on_page_callback: + on_page_callback([{"activity": "ERROR", "initiatedBy": f"Failed to query User Creation logs: {ex}"}]) + + logger.info("Finished fetching User Creation logs. Rows written: %d", rows_written) + finally: + self.client.release_token(token_slot) diff --git a/core/graph/directory/users_groups.py b/core/graph/directory/users_groups.py new file mode 100644 index 00000000..890cbbd2 --- /dev/null +++ b/core/graph/directory/users_groups.py @@ -0,0 +1,153 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Service for querying Entra ID Users & Groups counts.""" + +import logging +from typing import Dict, Any +from core.graph.client import GraphClient + +logger = logging.getLogger(__name__) + +class UsersGroupsService: + def __init__(self, client: GraphClient) -> None: + self.client = client + + def get_users_groups_counts(self, log_callback=None) -> Dict[str, Any]: + """Queries Microsoft Graph API in batch to fetch group counts and user counts.""" + if log_callback: + log_callback("Querying users and groups counts from Microsoft Graph...") + logger.info("Fetching users & groups count telemetry data using Graph API batch...") + + batch_requests = [ + { + "id": "total", + "method": "GET", + "url": "/groups?$count=true&$top=1&$select=id", + "headers": {"ConsistencyLevel": "eventual"} + }, + { + "id": "security", + "method": "GET", + "url": "/groups?$filter=securityEnabled eq true and mailEnabled eq false&$count=true&$top=1&$select=id", + "headers": {"ConsistencyLevel": "eventual"} + }, + { + "id": "distribution", + "method": "GET", + "url": "/groups?$filter=mailEnabled eq true and securityEnabled eq false and NOT groupTypes/any(c:c eq 'Unified')&$count=true&$top=1&$select=id", + "headers": {"ConsistencyLevel": "eventual"} + }, + { + "id": "mail_enabled_security", + "method": "GET", + "url": "/groups?$filter=mailEnabled eq true and securityEnabled eq true and NOT groupTypes/any(c:c eq 'Unified')&$count=true&$top=1&$select=id", + "headers": {"ConsistencyLevel": "eventual"} + }, + { + "id": "m365", + "method": "GET", + "url": "/groups?$filter=groupTypes/any(c:c eq 'Unified')&$count=true&$top=1&$select=id", + "headers": {"ConsistencyLevel": "eventual"} + }, + { + "id": "dynamic", + "method": "GET", + "url": "/groups?$filter=groupTypes/any(s:s eq 'DynamicMembership')&$count=true&$top=1&$select=id", + "headers": {"ConsistencyLevel": "eventual"} + }, + { + "id": "users_total", + "method": "GET", + "url": "/users?$count=true&$top=1&$select=id", + "headers": {"ConsistencyLevel": "eventual"} + }, + { + "id": "users_enabled", + "method": "GET", + "url": "/users?$filter=accountEnabled eq true&$count=true&$top=1&$select=id", + "headers": {"ConsistencyLevel": "eventual"} + }, + { + "id": "users_disabled", + "method": "GET", + "url": "/users?$filter=accountEnabled eq false&$count=true&$top=1&$select=id", + "headers": {"ConsistencyLevel": "eventual"} + }, + { + "id": "users_member", + "method": "GET", + "url": "/users?$filter=userType eq 'Member'&$count=true&$top=1&$select=id", + "headers": {"ConsistencyLevel": "eventual"} + }, + { + "id": "users_guest", + "method": "GET", + "url": "/users?$filter=userType eq 'Guest'&$count=true&$top=1&$select=id", + "headers": {"ConsistencyLevel": "eventual"} + } + ] + + responses = self.client.url_invoker.invoke( + url="https://graph.microsoft.com/v1.0", + batch=batch_requests, + logger=log_callback or (lambda x: None), + context="UsersGroupsTelemetry" + ) + + group_counts = { + "total": 0, + "security": 0, + "distribution": 0, + "mail_enabled_security": 0, + "m365": 0, + "dynamic": 0 + } + + user_counts = { + "users_total": 0, + "users_enabled": 0, + "users_disabled": 0, + "users_member": 0, + "users_guest": 0 + } + + for resp in responses: + resp_id = resp.get("id") + if resp.get("status", 0) != 200: + error_msg = resp.get("body", {}).get("error", {}).get("message", "Unknown error") + logger.error("Failed to fetch Users/Groups count for %s: status %s, message: %s", resp_id, resp.get("status"), error_msg) + raise Exception(f"Failed to fetch Users/Groups count for '{resp_id}': {error_msg}") + + body = resp.get("body", {}) + if resp_id in group_counts: + count_val = body.get("@odata.count", 0) + group_counts[resp_id] = count_val + elif resp_id in user_counts: + count_val = body.get("@odata.count", 0) + user_counts[resp_id] = count_val + + # Normalize keys + normalized_user_counts = { + "total": user_counts["users_total"], + "enabled": user_counts["users_enabled"], + "disabled": user_counts["users_disabled"], + "member": user_counts["users_member"], + "guest": user_counts["users_guest"] + } + + return { + "group_counts": group_counts, + "user_counts": normalized_user_counts + } diff --git a/core/graph/ediscovery.py b/core/graph/ediscovery.py new file mode 100644 index 00000000..042d57fe --- /dev/null +++ b/core/graph/ediscovery.py @@ -0,0 +1,61 @@ +import requests +import logging +from typing import Dict, Any + +logger = logging.getLogger(__name__) + +class EDiscoveryFetcher: + """Fetches eDiscovery cases using Delegated Authentication.""" + + def __init__(self, token: str): + self.token = token + self.base_url = "https://graph.microsoft.com/v1.0" + self.session = requests.Session() + self.session.headers.update({ + "Authorization": f"Bearer {self.token}", + "Accept": "application/json" + }) + + def fetch_cases(self, csv_path: str = None, on_page_callback=None) -> Dict[str, Any]: + """Fetches the list of eDiscovery cases.""" + endpoint = f"{self.base_url}/security/cases/ediscoveryCases" + all_cases = [] + try: + if csv_path: + import csv + with open(csv_path, 'w', newline='', encoding='utf-8') as f: + writer = csv.DictWriter(f, fieldnames=["id", "displayName", "status", "createdDateTime", "closedBy"]) + writer.writeheader() + + while endpoint: + response = self.session.get(endpoint) + response.raise_for_status() + data = response.json() + page_items = data.get("value", []) + + if csv_path and page_items: + import csv + with open(csv_path, 'a', newline='', encoding='utf-8') as f: + writer = csv.DictWriter(f, fieldnames=["id", "displayName", "status", "createdDateTime", "closedBy"], extrasaction='ignore') + for item in page_items: + cb = item.get("closedBy", {}) + cb_user = cb.get("user", {}) if isinstance(cb, dict) else {} + item["closedBy"] = cb_user.get("displayName", "") + writer.writerow(item) + + all_cases.extend(page_items) + if on_page_callback: + on_page_callback(page_items) + + endpoint = data.get("@odata.nextLink") + + return {"success": True, "data": all_cases} + except requests.exceptions.RequestException as e: + logger.error(f"Failed to fetch eDiscovery cases: {e}") + error_details = str(e) + if hasattr(e, 'response') and e.response is not None: + try: + error_details += " - " + e.response.json().get("error", {}).get("message", e.response.text) + except Exception: + error_details += " - " + e.response.text + return {"success": False, "error": error_details} diff --git a/core/graph/entra/__init__.py b/core/graph/entra/__init__.py new file mode 100644 index 00000000..3e2416c1 --- /dev/null +++ b/core/graph/entra/__init__.py @@ -0,0 +1,203 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Facade exposing Entra telemetry pipelines and consolidated runner.""" + +import os +import csv +import logging +import threading + +from core.graph.client import GraphClient +from core.graph.reports import ReportsService + +from core.graph.entra.auth_methods import run_auth_methods_pipeline +from core.graph.entra.app_signins import run_app_signins_pipeline +from core.graph.entra.user_signins import run_user_signins_pipeline +from core.graph.entra.app_registrations import run_app_registrations_pipeline + +logger = logging.getLogger(__name__) + +def run_devices_apps_pipeline( + client_id: str, + client_secret: str, + tenant_id: str, + on_app_signins_page_callback=None, + on_auth_methods_page_callback=None, + on_user_signins_page_callback=None, + on_app_registrations_page_callback=None, + is_cancelled_callback=None +) -> dict: + """Pipeline to fetch app sign-in summaries, auth methods, user signins, and app registrations in parallel.""" + logger.info("Starting Microsoft Entra Data Telemetry Pipeline in parallel...") + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + telemetry_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(script_dir))), "telemetry") + reports_dir = os.path.join(telemetry_dir, "reports", f"{tenant_id}_{client_id}") + os.makedirs(reports_dir, exist_ok=True) + + csv_path_app_signins = os.path.join(reports_dir, "entra_app_signins.csv") + csv_path_auth_methods = os.path.join(reports_dir, "entra_auth_methods.csv") + csv_path_user_signins = os.path.join(reports_dir, "entra_user_signins.csv") + csv_path_app_registrations = os.path.join(reports_dir, "entra_app_registrations.csv") + + with open(csv_path_app_signins, 'w', encoding='utf-8', newline='') as f: + writer = csv.writer(f) + writer.writerow(["appDisplayName", "successSignInCount"]) + + with open(csv_path_auth_methods, 'w', encoding='utf-8', newline='') as f: + writer = csv.writer(f) + writer.writerow(["authenticationMethod", "successActivityCount"]) + + with open(csv_path_user_signins, 'w', encoding='utf-8', newline='') as f: + writer = csv.writer(f) + writer.writerow(["appDisplayName", "operatingSystem", "browser", "isInteractive"]) + + with open(csv_path_app_registrations, 'w', encoding='utf-8', newline='') as f: + writer = csv.writer(f) + writer.writerow(["displayName", "appId", "createdDateTime", "signInAudience", "credentials"]) + + client = GraphClient( + tenant_id=tenant_id, + client_ids=client_id, + client_secrets=client_secret, + concurrency=4, + retries=5, + backoff=2 + ) + client.authenticate() + reports_service = ReportsService(client) + + errors = [] + + def run_fetch_app_signins(path): + try: + reports_service.fetch_app_signin_summary( + csv_path=path, + max_rows=5000, + on_page_callback=on_app_signins_page_callback, + is_cancelled_callback=is_cancelled_callback + ) + except Exception as thread_err: + logger.error(f"Error in thread fetching app sign-ins: {thread_err}", exc_info=True) + errors.append(thread_err) + + def run_fetch_auth_methods(path): + try: + reports_service.fetch_auth_methods_summary( + csv_path=path, + max_rows=5000, + on_page_callback=on_auth_methods_page_callback, + is_cancelled_callback=is_cancelled_callback + ) + except Exception as thread_err: + logger.error(f"Error in thread fetching auth methods: {thread_err}", exc_info=True) + errors.append(thread_err) + + def run_fetch_user_signins(path): + try: + reports_service.fetch_user_signins( + csv_path=path, + max_rows=20000, + on_page_callback=on_user_signins_page_callback, + is_cancelled_callback=is_cancelled_callback + ) + except Exception as thread_err: + logger.error(f"Error in thread fetching user sign-ins: {thread_err}", exc_info=True) + errors.append(thread_err) + + def run_fetch_app_registrations(path): + try: + reports_service.fetch_app_registrations( + csv_path=path, + max_rows=5000, + on_page_callback=on_app_registrations_page_callback, + is_cancelled_callback=is_cancelled_callback + ) + except Exception as thread_err: + logger.error(f"Error in thread fetching app registrations: {thread_err}", exc_info=True) + errors.append(thread_err) + + try: + t3 = threading.Thread(target=run_fetch_app_signins, args=(csv_path_app_signins,), daemon=True) + t4 = threading.Thread(target=run_fetch_auth_methods, args=(csv_path_auth_methods,), daemon=True) + t5 = threading.Thread(target=run_fetch_user_signins, args=(csv_path_user_signins,), daemon=True) + t6 = threading.Thread(target=run_fetch_app_registrations, args=(csv_path_app_registrations,), daemon=True) + + t3.start() + t4.start() + t5.start() + t6.start() + + t3.join() + t4.join() + t5.join() + t6.join() + + if len(errors) == 4: + raise errors[0] + + app_signins = [] + auth_methods = [] + app_registrations = [] + unique_apps = set() + unique_os = set() + unique_browsers = set() + + if os.path.exists(csv_path_app_signins): + with open(csv_path_app_signins, 'r', encoding='utf-8') as f: + reader = csv.reader(f) + next(reader, None) + for row in reader: + if len(row) >= 2: + app_signins.append((row[0], row[1])) + + if os.path.exists(csv_path_auth_methods): + with open(csv_path_auth_methods, 'r', encoding='utf-8') as f: + reader = csv.reader(f) + next(reader, None) + for row in reader: + if len(row) >= 2: + auth_methods.append((row[0], row[1])) + + if os.path.exists(csv_path_app_registrations): + with open(csv_path_app_registrations, 'r', encoding='utf-8') as f: + reader = csv.reader(f) + next(reader, None) + for row in reader: + if len(row) >= 5: + app_registrations.append((row[0], row[1], row[2], row[3], row[4])) + + if os.path.exists(csv_path_user_signins): + with open(csv_path_user_signins, 'r', encoding='utf-8') as f: + reader = csv.reader(f) + next(reader, None) + for row in reader: + if len(row) >= 4: + if row[0]: unique_apps.add(row[0]) + if row[1]: unique_os.add(row[1]) + if row[2]: unique_browsers.add(row[2]) + + return { + "app_signins": app_signins, + "auth_methods": auth_methods, + "app_registrations": app_registrations, + "user_signins": { + "apps": sorted(list(unique_apps)), + "os": sorted(list(unique_os)), + "browsers": sorted(list(unique_browsers)) + } + } + finally: + client.close() diff --git a/core/graph/entra/app_registrations.py b/core/graph/entra/app_registrations.py new file mode 100644 index 00000000..c22eb3ee --- /dev/null +++ b/core/graph/entra/app_registrations.py @@ -0,0 +1,51 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Microsoft Entra App Registrations telemetry scanner data pipeline.""" + +import logging +from core.graph.client import GraphClient +from core.graph.reports import ReportsService + +logger = logging.getLogger(__name__) + +def run_app_registrations_pipeline( + client_id: str, + client_secret: str, + tenant_id: str, + csv_path: str, + max_rows: int = 5000, + on_page_callback=None, + is_cancelled_callback=None +): + """Fetch app registrations details.""" + client = GraphClient( + tenant_id=tenant_id, + client_ids=client_id, + client_secrets=client_secret, + concurrency=1, + retries=5, + backoff=2 + ) + client.authenticate() + reports_service = ReportsService(client) + try: + reports_service.fetch_app_registrations( + csv_path=csv_path, + max_rows=max_rows, + on_page_callback=on_page_callback, + is_cancelled_callback=is_cancelled_callback + ) + finally: + client.close() diff --git a/core/graph/entra/app_signins.py b/core/graph/entra/app_signins.py new file mode 100644 index 00000000..674ed150 --- /dev/null +++ b/core/graph/entra/app_signins.py @@ -0,0 +1,51 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Microsoft Entra App Sign-ins telemetry scanner data pipeline.""" + +import logging +from core.graph.client import GraphClient +from core.graph.reports import ReportsService + +logger = logging.getLogger(__name__) + +def run_app_signins_pipeline( + client_id: str, + client_secret: str, + tenant_id: str, + csv_path: str, + max_rows: int = 5000, + on_page_callback=None, + is_cancelled_callback=None +): + """Fetch app sign-in activity summary.""" + client = GraphClient( + tenant_id=tenant_id, + client_ids=client_id, + client_secrets=client_secret, + concurrency=1, + retries=5, + backoff=2 + ) + client.authenticate() + reports_service = ReportsService(client) + try: + reports_service.fetch_app_signin_summary( + csv_path=csv_path, + max_rows=max_rows, + on_page_callback=on_page_callback, + is_cancelled_callback=is_cancelled_callback + ) + finally: + client.close() diff --git a/core/graph/entra/auth_methods.py b/core/graph/entra/auth_methods.py new file mode 100644 index 00000000..d4ac180e --- /dev/null +++ b/core/graph/entra/auth_methods.py @@ -0,0 +1,53 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Microsoft Entra Authentication Methods telemetry scanner data pipeline.""" + +import logging +from core.graph.client import GraphClient +from core.graph.reports import ReportsService + +logger = logging.getLogger(__name__) + +def run_auth_methods_pipeline( + client_id: str, + client_secret: str, + tenant_id: str, + csv_path: str, + period: str = "D7", + max_rows: int = 5000, + on_page_callback=None, + is_cancelled_callback=None +): + """Fetch authentication methods usage activity summary.""" + client = GraphClient( + tenant_id=tenant_id, + client_ids=client_id, + client_secrets=client_secret, + concurrency=1, + retries=5, + backoff=2 + ) + client.authenticate() + reports_service = ReportsService(client) + try: + reports_service.fetch_auth_methods_summary( + csv_path=csv_path, + period=period, + max_rows=max_rows, + on_page_callback=on_page_callback, + is_cancelled_callback=is_cancelled_callback + ) + finally: + client.close() diff --git a/core/graph/entra/user_signins.py b/core/graph/entra/user_signins.py new file mode 100644 index 00000000..93d5ad2a --- /dev/null +++ b/core/graph/entra/user_signins.py @@ -0,0 +1,51 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Microsoft Entra User Sign-ins telemetry scanner data pipeline.""" + +import logging +from core.graph.client import GraphClient +from core.graph.reports import ReportsService + +logger = logging.getLogger(__name__) + +def run_user_signins_pipeline( + client_id: str, + client_secret: str, + tenant_id: str, + csv_path: str, + max_rows: int = 20000, + on_page_callback=None, + is_cancelled_callback=None +): + """Fetch interactive/non-interactive user sign-in details.""" + client = GraphClient( + tenant_id=tenant_id, + client_ids=client_id, + client_secrets=client_secret, + concurrency=1, + retries=5, + backoff=2 + ) + client.authenticate() + reports_service = ReportsService(client) + try: + reports_service.fetch_user_signins( + csv_path=csv_path, + max_rows=max_rows, + on_page_callback=on_page_callback, + is_cancelled_callback=is_cancelled_callback + ) + finally: + client.close() diff --git a/core/graph/exchange/__init__.py b/core/graph/exchange/__init__.py new file mode 100644 index 00000000..c927141d --- /dev/null +++ b/core/graph/exchange/__init__.py @@ -0,0 +1,24 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Facade exposing Exchange data pipeline functions.""" + +from core.graph.exchange.mailbox import run_mailbox_usage_pipeline, format_bytes +from core.graph.exchange.calendar import run_calendar_telemetry_pipeline +from core.graph.exchange.integrated_apps import run_exchange_apps_pipeline +from core.graph.exchange.mail_security import run_mail_security_pipeline +from core.graph.exchange.transport_rules import run_transport_rules_pipeline +from core.graph.exchange.connectors import fetch_exchange_connectors_data +from core.graph.exchange.email_clients import run_email_client_usage_pipeline +from core.graph.exchange.pst_files import run_pst_discovery_pipeline diff --git a/core/graph/exchange/calendar.py b/core/graph/exchange/calendar.py new file mode 100644 index 00000000..a7147952 --- /dev/null +++ b/core/graph/exchange/calendar.py @@ -0,0 +1,122 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Exchange Online Calendar environment telemetry scanner data pipeline.""" + +import logging +from core.graph.client import GraphClient +from core.graph.directory import DirectoryService +from core.powershell.client import PowerShellClient +from core.powershell.calendar import CalendarStatsService + +logger = logging.getLogger(__name__) + +def run_calendar_telemetry_pipeline(client_id: str, client_secret: str, tenant_id: str) -> dict: + """Consolidated orchestration pipeline to download and audit Exchange Calendar telemetry config.""" + logger.info("Starting PowerShell Calendar Telemetry Pipeline...") + + tenant_domain = tenant_id + client = None + try: + client = GraphClient( + tenant_id=tenant_id, + client_ids=client_id, + client_secrets=client_secret, + concurrency=1, + retries=5, + backoff=2 + ) + client.authenticate() + dir_svc = DirectoryService(client) + tenant_domain = dir_svc.get_tenant_primary_domain() + logger.info(f"Retrieved primary tenant domain: {tenant_domain}") + except Exception as e: + logger.warning(f"Could not retrieve tenant domain via Graph. Falling back to Tenant ID Guid: {e}") + finally: + if client: + client.close() + + rooms_count = 0 + rooms_error = None + rooms_naming = None + equipment_count = 0 + equipment_error = None + can_share_attachments = True + owa_policy_error = None + org_apps = [] + apps_error = None + powershell_error = None + + try: + logger.info("Connecting to Exchange Online PowerShell for calendar metadata...") + ps_client = PowerShellClient( + tenant_id=tenant_domain, + client_id=client_id, + client_secret=client_secret, + cert_tenant_id=tenant_id + ) + cal_service = CalendarStatsService(ps_client) + metadata = cal_service.fetch_calendar_attachments_policy() + + rooms_count = metadata.get("RoomsCount", 0) + rooms_error = metadata.get("RoomsError") + rooms_naming = metadata.get("RoomsNaming") + equipment_count = metadata.get("EquipmentCount", 0) + equipment_error = metadata.get("EquipmentError") + can_share_attachments = metadata.get("CanShareAttachments", True) + owa_policy_error = metadata.get("OwaPolicyError") + org_apps = metadata.get("OrganizationApps", []) + apps_error = metadata.get("AppsError") + + except Exception as e: + logger.warning(f"Could not connect to Exchange Online PowerShell: {e}") + powershell_error = str(e) + + if "pwsh" in str(e).lower() or "powershell" in str(e).lower(): + err_msg = "pwsh not available" + elif "module" in str(e).lower(): + err_msg = "ExchangeOnlineManagement module not installed" + else: + err_msg = "Not Permitted (Exchange Permission Issue)" + + rooms_error = err_msg + equipment_error = err_msg + owa_policy_error = err_msg + apps_error = err_msg + + if rooms_error: + logger.error(f"Exchange PowerShell error querying Room Mailboxes: {rooms_error}") + if equipment_error: + logger.error(f"Exchange PowerShell error querying Equipment Mailboxes: {equipment_error}") + if owa_policy_error: + logger.error(f"Exchange PowerShell error querying OWA Mailbox Policy: {owa_policy_error}") + if apps_error: + logger.error(f"Exchange PowerShell error querying Organization Apps: {apps_error}") + + total_resources = rooms_count + equipment_count + + return { + "CanUsersReserveRooms": rooms_error if rooms_error else (total_resources > 0), + "TotalCalendarResources": total_resources, + "RoomsCount": rooms_count, + "EquipmentCount": equipment_count, + "RoomsError": rooms_error, + "DevicesError": equipment_error, + "OrganizationApps": org_apps, + "AppsError": apps_error, + "NamingConvention": rooms_error if rooms_error else (rooms_naming if rooms_naming else "None found"), + "CanShareAttachments": owa_policy_error if owa_policy_error else can_share_attachments, + "RoomsList": metadata.get("RoomsList", []) if not rooms_error else [], + "powershell_error": powershell_error + } diff --git a/core/graph/exchange/connectors.py b/core/graph/exchange/connectors.py new file mode 100644 index 00000000..de37e5c7 --- /dev/null +++ b/core/graph/exchange/connectors.py @@ -0,0 +1,58 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Exchange Online Inbound and Outbound Connectors telemetry scanner data pipeline.""" + +import logging +from core.graph.client import GraphClient +from core.graph.directory import DirectoryService +from core.powershell.client import PowerShellClient +from core.powershell.exchange_connectors import ExchangeConnectorsService + +logger = logging.getLogger(__name__) + +def fetch_exchange_connectors_data(client_id: str, client_secret: str, tenant_id: str) -> dict: + """Fetch Exchange Online Inbound and Outbound Connectors.""" + logger.info("Starting Exchange Connectors fetch...") + + tenant_domain = tenant_id + try: + client = GraphClient( + tenant_id=tenant_id, + client_ids=client_id, + client_secrets=client_secret, + concurrency=1, + retries=3, + backoff=2 + ) + client.authenticate() + dir_svc = DirectoryService(client) + tenant_domain = dir_svc.get_tenant_primary_domain() + logger.info(f"Retrieved primary tenant domain for Connectors: {tenant_domain}") + except Exception as e: + logger.warning(f"Could not retrieve tenant domain. Falling back to Tenant ID Guid: {e}") + finally: + try: + client.close() + except Exception: + pass + + try: + ps_client = PowerShellClient(tenant_id=tenant_domain, client_id=client_id, client_secret=client_secret, cert_tenant_id=tenant_id) + connector_svc = ExchangeConnectorsService(ps_client) + data = connector_svc.fetch_exchange_connectors() + return {"connectors": data, "error": None} + except Exception as e: + logger.error("Failed to fetch Exchange Connectors via PowerShell", exc_info=True) + return {"connectors": None, "error": str(e)} diff --git a/core/graph/exchange/email_clients.py b/core/graph/exchange/email_clients.py new file mode 100644 index 00000000..605948c9 --- /dev/null +++ b/core/graph/exchange/email_clients.py @@ -0,0 +1,101 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Exchange Online Supported Email Clients telemetry scanner data pipeline.""" + +import os +import logging +import pandas as pd + +from core.graph.client import GraphClient +from core.graph.reports import ReportsService + +logger = logging.getLogger(__name__) + +def parse_email_client_support_csv(filepath: str) -> dict: + """Parses the Email App Usage Counts CSV to categorize Browser vs Non-Browser client adoption.""" + logger.info(f"Processing Email App Usage Counts file: {os.path.basename(filepath)}") + if not os.path.exists(filepath): + logger.warning(f"Email App Usage report {filepath} not found. Skipping client breakdown.") + return {} + + try: + df = pd.read_csv(filepath) + for col in df.columns: + if col not in ["Report Refresh Date", "User Principal Name", "Display Name", "Last Activity Date", "Is Deleted"]: + df[col] = df[col].astype(str).str.strip().str.upper().isin(["TRUE", "1", "UNDETERMINED"]) + + if "Is Deleted" in df.columns: + df = df[~df["Is Deleted"].astype(str).str.strip().str.upper().isin(["TRUE", "1"])] + + if df.empty: + logger.warning(f"Email App Usage report {filepath} is empty.") + return {} + + owa_users = int(df["Outlook For Web"].sum()) if "Outlook For Web" in df.columns else 0 + + win_users = int(df["Outlook For Windows"].sum()) if "Outlook For Windows" in df.columns else 0 + mac_users = int(df["Outlook For Mac"].sum()) if "Outlook For Mac" in df.columns else 0 + mail_mac = int(df["Mail For Mac"].sum()) if "Mail For Mac" in df.columns else 0 + + mobile_users = int(df["Outlook For Mobile"].sum()) if "Outlook For Mobile" in df.columns else 0 + other_mobile = int(df["Other For Mobile"].sum()) if "Other For Mobile" in df.columns else 0 + + pop3_users = int(df["POP3 App"].sum()) if "POP3 App" in df.columns else 0 + imap4_users = int(df["IMAP4 App"].sum()) if "IMAP4 App" in df.columns else 0 + smtp_users = int(df["SMTP App"].sum()) if "SMTP App" in df.columns else 0 + + total_desktop = win_users + mac_users + mail_mac + total_mobile = mobile_users + other_mobile + total_protocols = pop3_users + imap4_users + smtp_users + total_non_browser = total_desktop + total_mobile + total_protocols + + return { + "browser_users": owa_users, + "desktop_win": win_users, + "desktop_mac": mac_users, + "desktop_mail_mac": mail_mac, + "mobile_outlook": mobile_users, + "mobile_other": other_mobile, + "protocol_pop3": pop3_users, + "protocol_imap4": imap4_users, + "protocol_smtp": smtp_users, + "total_desktop": total_desktop, + "total_mobile": total_mobile, + "total_protocols": total_protocols, + "total_non_browser": total_non_browser + } + except Exception as e: + logger.error(f"Error parsing Email App Usage CSV: {e}") + return {} + +def run_email_client_usage_pipeline(client_id: str, client_secret: str, tenant_id: str) -> dict: + """Executes the pipeline to download and parse Supported Email Clients usage data.""" + client = GraphClient(tenant_id=tenant_id, client_ids=client_id, client_secrets=client_secret, concurrency=1, retries=3, backoff=2) + client.authenticate() + service = ReportsService(client) + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + telemetry_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(script_dir))), "telemetry") + reports_dir = os.path.join(telemetry_dir, "reports", f"{tenant_id}_{client_id}") + + client_stats = {} + client_error = None + try: + service.download_email_app_usage_detail(reports_dir) + client_stats = parse_email_client_support_csv(os.path.join(reports_dir, "EmailAppUsageUserDetail(180d).csv")) + except Exception as e: + client_error = str(e) + client.close() + return {"client_adoption": client_stats, "client_error": client_error} diff --git a/core/graph/exchange/integrated_apps.py b/core/graph/exchange/integrated_apps.py new file mode 100644 index 00000000..72044653 --- /dev/null +++ b/core/graph/exchange/integrated_apps.py @@ -0,0 +1,21 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Exchange Online Integrated Apps (Organization-wide Apps) telemetry scanner data pipeline.""" + +from core.graph.exchange.calendar import run_calendar_telemetry_pipeline + +def run_exchange_apps_pipeline(client_id: str, client_secret: str, tenant_id: str) -> dict: + """Executes the pipeline to fetch Exchange Online Integrated Apps.""" + return run_calendar_telemetry_pipeline(client_id, client_secret, tenant_id) diff --git a/core/graph/exchange/mail_security.py b/core/graph/exchange/mail_security.py new file mode 100644 index 00000000..e62ce922 --- /dev/null +++ b/core/graph/exchange/mail_security.py @@ -0,0 +1,90 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Exchange Online Mail Security telemetry scanner data pipeline.""" + +import logging +import requests +from util.auth_manager import TokenManager + +logger = logging.getLogger(__name__) + +def run_mail_security_pipeline(client_id: str, client_secret: str, tenant_id: str) -> dict: + """Standalone pipeline to query Graph API for mail security SKUs and user counts.""" + if not tenant_id or not client_id or not client_secret: + raise ValueError("Missing credentials.") + + tm = TokenManager(tenant_id=tenant_id, client_ids=[client_id], client_secrets=[client_secret], concurrency=1, retries=1, backoff=0) + tm.authenticate_all() + + slot = tm.get_valid_token_slot() + if not slot: + raise ConnectionError("Authentication failed: No valid token.") + + token = slot["token"] + headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"} + + url = "https://graph.microsoft.com/v1.0/subscribedSkus" + data = [] + + while url: + res = requests.get(url, headers=headers, timeout=15) + if res.status_code != 200: + tm.return_token_slot(slot) + raise ConnectionError(f"Graph API Error {res.status_code}: {res.text}") + + json_data = res.json() + data.extend(json_data.get("value", [])) + url = json_data.get("@odata.nextLink") + + tm.return_token_slot(slot) + + defender_skus_set = set() + eop_skus_set = set() + + defender_users = 0 + eop_users = 0 + + for sku in data: + raw_part_num = sku.get("skuPartNumber", "Unknown") + if isinstance(raw_part_num, list): + part_num = ", ".join([str(x) for x in raw_part_num]) + else: + part_num = str(raw_part_num) + + consumed = int(sku.get("consumedUnits", 0)) + plans = sku.get("servicePlans", []) + + has_defender = False + has_eop = False + + for p in plans: + if p.get("provisioningStatus") == "Success": + name = p.get("servicePlanName", "").upper() + if "DEFENDER_PLATFORM_FOR_OFFICE" in name or "ATP_ENTERPRISE" in name: + has_defender = True + elif "EXCHANGE_S_ENTERPRISE" in name or "EXCHANGE_S_STANDARD" in name or "EXCHANGE_S_FOUNDATION" in name: + has_eop = True + + if has_defender: + defender_skus_set.add(part_num) + defender_users += consumed + elif has_eop: + eop_skus_set.add(part_num) + eop_users += consumed + + return { + "defender": {"skus": list(defender_skus_set), "users": defender_users}, + "eop": {"skus": list(eop_skus_set), "users": eop_users} + } diff --git a/core/graph/exchange/mailbox.py b/core/graph/exchange/mailbox.py new file mode 100644 index 00000000..d8ee21ac --- /dev/null +++ b/core/graph/exchange/mailbox.py @@ -0,0 +1,164 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Exchange Online Mailbox usage telemetry scanner data pipeline.""" + +import os +import logging +import pandas as pd + +from core.graph.client import GraphClient +from core.graph.reports import ReportsService +from core.graph.directory import DirectoryService +from core.powershell.client import PowerShellClient +from core.powershell.mailbox import MailboxStatsService + +logger = logging.getLogger(__name__) + +def format_bytes(num_bytes: float) -> str: + """Formats raw byte values into highly readable string equivalents (e.g., GB, TB).""" + if num_bytes is None: + return "0.00 Bytes" + + for unit in ['Bytes', 'KB', 'MB', 'GB', 'TB', 'PB']: + if num_bytes < 1024.0: + return f"{num_bytes:,.2f} {unit}" + num_bytes /= 1024.0 + return f"{num_bytes:,.2f} EB" + +def parse_mailbox_usage_csv(filepath: str) -> dict: + """Streams the Mailbox Usage Detail CSV and aggregates metrics in chunks using pandas.""" + logger.info(f"Processing Mailbox Usage file: {os.path.basename(filepath)}") + if not os.path.exists(filepath): + logger.error(f"Error: Could not find Mailbox report {filepath}") + raise FileNotFoundError("Mailbox Usage report not found.") + + cols = ["Storage Used (Byte)", "Item Count"] + headers = pd.read_csv(filepath, nrows=0).columns.tolist() + if "Is Deleted" in headers: + cols.append("Is Deleted") + + total_mailboxes = 0 + total_bytes = 0 + total_emails = 0 + + for chunk in pd.read_csv(filepath, usecols=cols, chunksize=10000): + if "Is Deleted" in chunk.columns: + active_chunk = chunk[~chunk["Is Deleted"].astype(str).str.strip().str.upper().isin(["TRUE", "1"])] + else: + active_chunk = chunk + + active_chunk = active_chunk.dropna(subset=['Storage Used (Byte)', 'Item Count']) + + total_mailboxes += len(active_chunk) + total_bytes += int(active_chunk['Storage Used (Byte)'].sum()) + total_emails += int(active_chunk['Item Count'].sum()) + + avg_bytes = (total_bytes / total_mailboxes) if total_mailboxes > 0 else 0.0 + avg_emails = (total_emails / total_mailboxes) if total_mailboxes > 0 else 0.0 + + logger.info( + f"Mailbox parsing complete: mailboxes={total_mailboxes}, " + f"storage={format_bytes(total_bytes)}, items={total_emails}" + ) + + return { + "total_mailboxes": total_mailboxes, + "total_storage_bytes": total_bytes, + "total_storage_formatted": format_bytes(total_bytes), + "average_mailbox_size_bytes": avg_bytes, + "average_mailbox_size_formatted": format_bytes(avg_bytes), + "total_emails": total_emails, + "average_emails": avg_emails + } + +def run_mailbox_usage_pipeline(client_id: str, client_secret: str, tenant_id: str) -> dict: + """Pipeline specifically for Mailbox Usage telemetry data collection.""" + logger.info("Starting Mailbox Usage Telemetry Pipeline...") + + client = GraphClient( + tenant_id=tenant_id, + client_ids=client_id, + client_secrets=client_secret, + concurrency=1, + retries=5, + backoff=2 + ) + client.authenticate() + service = ReportsService(client) + + tenant_domain = tenant_id + try: + dir_svc = DirectoryService(client) + tenant_domain = dir_svc.get_tenant_primary_domain() + logger.info(f"Retrieved primary tenant domain for Connect-ExchangeOnline: {tenant_domain}") + except Exception as e: + logger.warning(f"Could not retrieve tenant domain via Graph. Falling back to Tenant ID Guid: {e}") + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + telemetry_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(script_dir))), "telemetry") + reports_dir = os.path.join(telemetry_dir, "reports", f"{tenant_id}_{client_id}") + + service.download_mailbox_usage_detail(reports_dir) + logger.info("Mailbox Usage CSV download completed. Initiating parser...") + client.close() + + data = parse_mailbox_usage_csv(os.path.join(reports_dir, "MailboxUsageDetail(180d).csv")) + + shared_count = None + shared_bytes = None + pf_count = None + pf_bytes = None + mail_pf_count = None + powershell_error = None + + try: + logger.info("Running PowerShell script for Shared Mailboxes and Public Folders stats...") + ps_client = PowerShellClient( + tenant_id=tenant_domain, + client_id=client_id, + client_secret=client_secret, + cert_tenant_id=tenant_id + ) + pb_service = MailboxStatsService(ps_client) + stats = pb_service.fetch_mailbox_and_folder_stats() + + shared_count = stats.get("SharedMailboxesCount") + shared_bytes = stats.get("SharedMailboxesTotalBytes") + pf_count = stats.get("PublicFoldersCount") + pf_bytes = stats.get("PublicFoldersTotalBytes") + mail_pf_count = stats.get("MailPublicFoldersCount") + + errors = stats.get("Errors", {}) + if errors: + for component, err_msg in errors.items(): + logger.error(f"PowerShell error querying {component}: {err_msg}") + powershell_error = ", ".join(f"{k}: {v}" for k, v in errors.items()) + except Exception as e: + logger.error("Failed to fetch Shared Mailbox / Public Folder stats via PowerShell", exc_info=True) + powershell_error = str(e) + + data.update({ + "shared_mailboxes_count": shared_count, + "shared_mailboxes_total_bytes": shared_bytes, + "shared_mailboxes_total_formatted": format_bytes(shared_bytes) if shared_bytes is not None else "Error/Unavailable", + "public_folders_count": pf_count, + "public_folders_total_bytes": pf_bytes, + "public_folders_total_formatted": format_bytes(pf_bytes) if pf_bytes is not None else "Error/Unavailable", + "mail_public_folders_count": mail_pf_count, + "powershell_error": powershell_error + }) + + logger.info("Mailbox Usage Telemetry Pipeline completed successfully.") + return data diff --git a/core/graph/exchange/pst_files.py b/core/graph/exchange/pst_files.py new file mode 100644 index 00000000..e8f1d823 --- /dev/null +++ b/core/graph/exchange/pst_files.py @@ -0,0 +1,36 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Exchange Online PST files telemetry scanner data pipeline.""" + +import logging +from core.graph.client import GraphClient +from core.graph.reports import ReportsService + +logger = logging.getLogger(__name__) + +def run_pst_discovery_pipeline(client_id: str, client_secret: str, tenant_id: str) -> dict: + """Executes search discovery of cloud stored PST files.""" + client = GraphClient(tenant_id=tenant_id, client_ids=client_id, client_secrets=client_secret, concurrency=1, retries=3, backoff=2) + client.authenticate() + service = ReportsService(client) + + pst_cloud = {} + pst_error = None + try: + pst_cloud = service.search_cloud_pst_files() + except Exception as e: + pst_error = str(e) + client.close() + return {"pst_cloud_data": pst_cloud, "pst_error": pst_error} diff --git a/core/graph/exchange/transport_rules.py b/core/graph/exchange/transport_rules.py new file mode 100644 index 00000000..e2b260ab --- /dev/null +++ b/core/graph/exchange/transport_rules.py @@ -0,0 +1,41 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Exchange Online Transport Rules telemetry scanner data pipeline.""" + +import os +import logging +from core.powershell.transport_rules import TransportRulesFetcher + +logger = logging.getLogger(__name__) + +def run_transport_rules_pipeline(client_id: str, client_secret: str, tenant_id: str) -> dict: + """Orchestrates PowerShell client to fetch Exchange Transport Rules and output to CSV.""" + logger.info("Starting Exchange Transport Rules Telemetry Pipeline...") + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + telemetry_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(script_dir))), "telemetry") + reports_dir = os.path.join(telemetry_dir, "reports", f"{tenant_id}_{client_id}") + os.makedirs(reports_dir, exist_ok=True) + csv_path = os.path.join(reports_dir, "exchange_transport_rules.csv") + + fetcher = TransportRulesFetcher(tenant_id, client_id, client_secret) + res = fetcher.fetch_rules(csv_path) + + if not res.get("success", False): + errs = res.get("errors", {}) + first_err = list(errs.values())[0] if errs else "Unknown Script Error" + raise ConnectionError(f"PowerShell Execution Error: {first_err}") + + return {"csv_path": csv_path, "success": True} diff --git a/core/graph/files/__init__.py b/core/graph/files/__init__.py new file mode 100644 index 00000000..d09e6960 --- /dev/null +++ b/core/graph/files/__init__.py @@ -0,0 +1,18 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Facade exposing Files data pipeline functions.""" + +from core.graph.files.sharepoint import run_sharepoint_pipeline, parse_sharepoint_csv +from core.graph.files.onedrive import run_onedrive_pipeline, parse_onedrive_csv, format_bytes diff --git a/core/graph/files/msteams_overview.py b/core/graph/files/msteams_overview.py new file mode 100644 index 00000000..1ab679ea --- /dev/null +++ b/core/graph/files/msteams_overview.py @@ -0,0 +1,40 @@ +import os +import logging +from core.graph.client import GraphClient +from core.graph.reports import ReportsService + +logger = logging.getLogger(__name__) + +def run_msteams_pipeline(client_id, client_secret, tenant_id) -> str: + """Pipeline specifically for MsTeams Overview telemetry data collection.""" + logger.info("Starting MsTeams Overview Telemetry Pipeline...") + + client = GraphClient( + tenant_id=tenant_id, + client_ids=client_id, + client_secrets=client_secret, + concurrency=2, + retries=5, + backoff=2 + ) + client.authenticate() + service = ReportsService(client) + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + telemetry_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(script_dir))), "telemetry") + reports_dir = os.path.join(telemetry_dir, "reports", f"{tenant_id}_{client_id}") + + output_filename = "msteams_activity.csv" + temp_filename = output_filename + ".tmp" + url = "https://graph.microsoft.com/v1.0/reports/getTeamsTeamActivityDetail(period='D180')" + + service.download_report(url, temp_filename, reports_dir) + + final_path = os.path.join(reports_dir, output_filename) + tmp_path = os.path.join(reports_dir, temp_filename) + if os.path.exists(tmp_path): + os.replace(tmp_path, final_path) + + logger.info("MsTeams Overview Telemetry Pipeline completed successfully.") + client.close() + return final_path diff --git a/core/graph/files/onedrive.py b/core/graph/files/onedrive.py new file mode 100644 index 00000000..c515fe5b --- /dev/null +++ b/core/graph/files/onedrive.py @@ -0,0 +1,171 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""OneDrive usage telemetry scanner data pipeline.""" + +import os +import logging +import pandas as pd + +from core.graph.client import GraphClient +from core.graph.reports import ReportsService + +logger = logging.getLogger(__name__) + +def format_bytes(num_bytes: int) -> str: + """Formats raw byte values into highly readable string equivalents (e.g., GB, TB).""" + if num_bytes is None: + return "0.00 Bytes" + + for unit in ['Bytes', 'KB', 'MB', 'GB', 'TB', 'PB']: + if num_bytes < 1024.0: + return f"{num_bytes:,.2f} {unit}" + num_bytes /= 1024.0 + return f"{num_bytes:,.2f} EB" + +def parse_onedrive_csv(filepath): + """Streams the OneDrive Account Usage Detail CSV and aggregates metrics in chunks.""" + logger.info(f"Processing OneDrive Account Usage file: {os.path.basename(filepath)}") + if not os.path.exists(filepath): + logger.error(f"Error: Could not find OneDrive report {filepath}") + raise FileNotFoundError(f"OneDrive Account Usage report not found.") + + headers = pd.read_csv(filepath, nrows=0).columns.tolist() + expected = ["Is Deleted", "Storage Used (Byte)", "File Count", "Active File Count"] + cols = [c for c in expected if c in headers] + + total_accounts = 0 + total_storage = 0 + total_files = 0 + active_files = 0 + + for chunk in pd.read_csv(filepath, usecols=cols, chunksize=10000, encoding="utf-8-sig"): + if "Is Deleted" in chunk.columns: + mask = chunk["Is Deleted"].astype(str).str.strip().str.upper() != "TRUE" + active_chunk = chunk[mask] + else: + active_chunk = chunk + + total_accounts += len(active_chunk) + if "Storage Used (Byte)" in active_chunk.columns: + total_storage += int(pd.to_numeric(active_chunk["Storage Used (Byte)"], errors='coerce').fillna(0).sum()) + if "File Count" in active_chunk.columns: + total_files += int(pd.to_numeric(active_chunk["File Count"], errors='coerce').fillna(0).sum()) + if "Active File Count" in active_chunk.columns: + active_files += int(pd.to_numeric(active_chunk["Active File Count"], errors='coerce').fillna(0).sum()) + + logger.info(f"OneDrive parsing complete: accounts={total_accounts}, storage={format_bytes(total_storage)}, files={total_files}, active_files={active_files}") + return { + "total_accounts": total_accounts, + "total_storage_bytes": total_storage, + "total_storage_formatted": format_bytes(total_storage), + "total_files": total_files, + "active_files": active_files, + "active_files_pct": (active_files / total_files * 100) if total_files > 0 else 0.0 + } + +def parse_onedrive_activity_csv(filepath): + """Streams the OneDrive Activity User Detail CSV and aggregates active sync client users in chunks.""" + logger.info(f"Processing OneDrive Activity User Detail file: {os.path.basename(filepath)}") + if not os.path.exists(filepath): + logger.error(f"Error: Could not find OneDrive Activity report {filepath}") + raise FileNotFoundError(f"OneDrive Activity User Detail report not found.") + + headers = pd.read_csv(filepath, nrows=0).columns.tolist() + expected = ["Is Deleted", "Synced File Count"] + cols = [c for c in expected if c in headers] + + sync_users = 0 + + for chunk in pd.read_csv(filepath, usecols=cols, chunksize=10000, encoding="utf-8-sig"): + if "Is Deleted" in chunk.columns: + mask = chunk["Is Deleted"].astype(str).str.strip().str.upper() != "TRUE" + active_chunk = chunk[mask] + else: + active_chunk = chunk + + if "Synced File Count" in active_chunk.columns: + synced_series = pd.to_numeric(active_chunk["Synced File Count"], errors='coerce').fillna(0) + sync_users += int((synced_series > 0).sum()) + + logger.info(f"OneDrive Activity parsing complete: sync_users={sync_users}") + return { + "sync_users": sync_users + } + +def parse_onenote_users_csv(filepath): + """Streams the M365 App User Detail CSV and counts unique active OneNote users in chunks.""" + logger.info(f"Processing OneNote Users file: {os.path.basename(filepath)}") + if not os.path.exists(filepath): + logger.error(f"Error: Could not find M365 App User Detail report {filepath}") + raise FileNotFoundError(f"M365 App User Detail report not found.") + + headers = pd.read_csv(filepath, nrows=0).columns.tolist() + expected = ["Is Deleted", "OneNote"] + cols = [c for c in expected if c in headers] + + onenote_users = 0 + + for chunk in pd.read_csv(filepath, usecols=cols, chunksize=10000, encoding="utf-8-sig"): + if "Is Deleted" in chunk.columns: + mask = chunk["Is Deleted"].astype(str).str.strip().str.upper() != "TRUE" + active_chunk = chunk[mask] + else: + active_chunk = chunk + + if "OneNote" in active_chunk.columns: + onenote_series = active_chunk["OneNote"].astype(str).str.strip().str.lower() + onenote_users += int(onenote_series.isin(["yes", "true"]).sum()) + + logger.info(f"OneNote Users parsing complete: onenote_users={onenote_users}") + return { + "onenote_users": onenote_users + } + +def run_onedrive_pipeline(client_id, client_secret, tenant_id) -> dict: + """Pipeline specifically for OneDrive telemetry data collection.""" + logger.info("Starting OneDrive Telemetry Pipeline...") + + client = GraphClient( + tenant_id=tenant_id, + client_ids=client_id, + client_secrets=client_secret, + concurrency=2, + retries=5, + backoff=2 + ) + client.authenticate() + service = ReportsService(client) + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + telemetry_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(script_dir))), "telemetry") + reports_dir = os.path.join(telemetry_dir, "reports", f"{tenant_id}_{client_id}") + + service.download_onedrive_details(reports_dir) + logger.info("OneDrive account, activity, and OneNote CSV downloads completed. Initiating parsers...") + client.close() + + od_data = parse_onedrive_csv(os.path.join(reports_dir, "OneDriveUsageAccountDetail(180d).csv")) + od_act_data = parse_onedrive_activity_csv(os.path.join(reports_dir, "OneDriveActivityUserDetail(180d).csv")) + onenote_data = parse_onenote_users_csv(os.path.join(reports_dir, "M365AppUserDetail_sp_od(180d).csv")) + + # Merge active sync client user data into od_data + od_data["sync_users"] = od_act_data["sync_users"] + od_data["sync_users_pct"] = (od_act_data["sync_users"] / od_data["total_accounts"] * 100) if od_data["total_accounts"] > 0 else 0.0 + + # Merge OneNote user data + od_data["onenote_users"] = onenote_data["onenote_users"] + + logger.info("OneDrive Telemetry Pipeline completed successfully.") + return od_data diff --git a/core/graph/files/sharepoint.py b/core/graph/files/sharepoint.py new file mode 100644 index 00000000..0cac64a8 --- /dev/null +++ b/core/graph/files/sharepoint.py @@ -0,0 +1,93 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""SharePoint Site usage telemetry scanner data pipeline.""" + +import os +import logging +import pandas as pd + +from core.graph.client import GraphClient +from core.graph.reports import ReportsService +from core.graph.files.onedrive import format_bytes + +logger = logging.getLogger(__name__) + +def parse_sharepoint_csv(filepath): + """Streams the SharePoint Site Usage Detail CSV and aggregates metrics in chunks.""" + logger.info(f"Processing SharePoint Site Usage file: {os.path.basename(filepath)}") + if not os.path.exists(filepath): + logger.error(f"Error: Could not find SharePoint report {filepath}") + raise FileNotFoundError(f"SharePoint Site Usage report not found.") + + headers = pd.read_csv(filepath, nrows=0).columns.tolist() + expected = ["Is Deleted", "Storage Used (Byte)", "File Count", "Active File Count"] + cols = [c for c in expected if c in headers] + + total_sites = 0 + total_storage = 0 + total_files = 0 + active_files = 0 + + for chunk in pd.read_csv(filepath, usecols=cols, chunksize=10000, encoding="utf-8-sig"): + if "Is Deleted" in chunk.columns: + mask = chunk["Is Deleted"].astype(str).str.strip().str.upper() != "TRUE" + active_chunk = chunk[mask] + else: + active_chunk = chunk + + total_sites += len(active_chunk) + if "Storage Used (Byte)" in active_chunk.columns: + total_storage += int(pd.to_numeric(active_chunk["Storage Used (Byte)"], errors='coerce').fillna(0).sum()) + if "File Count" in active_chunk.columns: + total_files += int(pd.to_numeric(active_chunk["File Count"], errors='coerce').fillna(0).sum()) + if "Active File Count" in active_chunk.columns: + active_files += int(pd.to_numeric(active_chunk["Active File Count"], errors='coerce').fillna(0).sum()) + + logger.info(f"SharePoint parsing complete: sites={total_sites}, storage={format_bytes(total_storage)}, files={total_files}, active_files={active_files}") + return { + "total_sites": total_sites, + "total_storage_bytes": total_storage, + "total_storage_formatted": format_bytes(total_storage), + "total_files": total_files, + "active_files": active_files, + "active_files_pct": (active_files / total_files * 100) if total_files > 0 else 0.0 + } + +def run_sharepoint_pipeline(client_id, client_secret, tenant_id) -> dict: + """Pipeline specifically for SharePoint telemetry data collection.""" + logger.info("Starting SharePoint Telemetry Pipeline...") + + client = GraphClient( + tenant_id=tenant_id, + client_ids=client_id, + client_secrets=client_secret, + concurrency=1, + retries=5, + backoff=2 + ) + client.authenticate() + service = ReportsService(client) + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + telemetry_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(script_dir))), "telemetry") + reports_dir = os.path.join(telemetry_dir, "reports", f"{tenant_id}_{client_id}") + + service.download_sharepoint_details(reports_dir) + logger.info("SharePoint Site Usage CSV download completed. Initiating parser...") + client.close() + + sp_data = parse_sharepoint_csv(os.path.join(reports_dir, "SharePointSiteUsageDetail(180d).csv")) + logger.info("SharePoint Telemetry Pipeline completed successfully.") + return sp_data diff --git a/core/graph/files/sharepoint_data_types.py b/core/graph/files/sharepoint_data_types.py new file mode 100644 index 00000000..a641dc28 --- /dev/null +++ b/core/graph/files/sharepoint_data_types.py @@ -0,0 +1,122 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import csv +import logging +import os +import requests + +from core.graph.client import GraphClient + +logger = logging.getLogger(__name__) + +class SharePointDataTypesService: + def __init__(self, client: GraphClient) -> None: + self.client = client + + def get_data_types_summary(self, reports_dir: str) -> dict: + """Runs the search queries to count document libraries, lists, and web pages.""" + token_slot = self.client.get_active_token() + session = self.client.get_session() + headers = { + "Authorization": f"Bearer {token_slot['token']}", + "Content-Type": "application/json" + } + + url = "https://graph.microsoft.com/v1.0/search/query" + + # Base queries without region + base_queries = { + "Document Libraries": "contentclass:STS_List_DocumentLibrary", + "Lists": "contentclass:STS_List", + "Web Pages": "contentclass:STS_ListItem_WebPageLibrary" + } + + results = {k: 0 for k in base_queries.keys()} + regions = ["NAM", "EUR", "APC"] + + try: + for region in regions: + logger.info(f"Querying region: {region}") + for name, query_str in base_queries.items(): + payload = { + "requests": [{ + "entityTypes": ["listItem"], + "query": {"queryString": query_str}, + "size": 1, + "region": region + }] + } + + try: + logger.info(f"Submitting Graph Search Query for: {name} in {region}") + resp = session.post(url, headers=headers, json=payload, timeout=30.0) + + if resp.status_code != 200: + logger.warning(f"Error Response Body for {region}: {resp.text}") + resp.raise_for_status() + + data = resp.json() + hits_containers = data.get("value", [{}])[0].get("hitsContainers", [{}]) + total = hits_containers[0].get("total", 0) + results[name] += total + + except requests.exceptions.HTTPError as e: + # If a multi-geo region doesn't exist for this tenant, log warning and continue + logger.warning(f"Failed to query {name} in region {region}. It may not be provisioned: {e}") + continue + + # Write to CSV in accordance with telemetry scaling guidelines + os.makedirs(reports_dir, exist_ok=True) + csv_path = os.path.join(reports_dir, "sharepoint_data_types.csv") + tmp_path = csv_path + ".tmp" + + with open(tmp_path, "w", newline="", encoding="utf-8") as f: + writer = csv.writer(f) + writer.writerow(["Data_Type", "Count"]) + for k, v in results.items(): + writer.writerow([k, v]) + + os.replace(tmp_path, csv_path) + + return results + finally: + self.client.release_token(token_slot) + +def run_sharepoint_data_types_pipeline(client_id, client_secret, tenant_id) -> dict: + """Pipeline entrypoint for SharePoint data types telemetry.""" + logger.info("Starting SharePoint Data Types Telemetry Pipeline...") + + client = GraphClient( + tenant_id=tenant_id, + client_ids=client_id, + client_secrets=client_secret, + concurrency=1, + retries=5, + backoff=2 + ) + logger.info("Attempting authenticate with Sites.Read.All...") + client.authenticate(required_scopes=["Sites.Read.All"]) + logger.info("Authentication successful.") + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + telemetry_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(script_dir))), "telemetry") + reports_dir = os.path.join(telemetry_dir, "reports", f"{tenant_id}_{client_id}") + + service = SharePointDataTypesService(client) + sp_data = service.get_data_types_summary(reports_dir) + + logger.info("SharePoint Data Types Pipeline completed successfully.") + client.close() + return sp_data diff --git a/core/graph/intune/__init__.py b/core/graph/intune/__init__.py new file mode 100644 index 00000000..6158c856 --- /dev/null +++ b/core/graph/intune/__init__.py @@ -0,0 +1,431 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Facade exposing Intune telemetry pipelines and legacy IntuneService class.""" + +import os +import csv +import logging +import threading +from collections import defaultdict +import pandas as pd + +from core.graph.client import GraphClient +from core.graph.intune.mobile_apps import run_mobile_apps_pipeline +from core.graph.intune.detected_apps import run_detected_apps_pipeline +from core.graph.intune.device_configs import run_device_configs_pipeline +from core.graph.intune.managed_devices import run_managed_devices_pipeline +from core.graph.intune.device_compliance import run_device_compliance_pipeline +from core.graph.intune.mdm_policies import run_mdm_policies_pipeline +from core.graph.intune.byod_configs import run_byod_configs_pipeline + +logger = logging.getLogger(__name__) + +def run_intune_policies_pipeline( + client_id: str, + client_secret: str, + tenant_id: str, + on_page_callback=None, + on_apps_page_callback=None, + is_cancelled_callback=None, + delegated_token: str = None +) -> dict: + """Consolidated pipeline to fetch Intune configuration policies, mobile apps, and detected apps in parallel (for backward compatibility).""" + logger.info("Starting Intune Policies Pipeline in parallel...") + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + telemetry_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(script_dir))), "telemetry") + reports_dir = os.path.join(telemetry_dir, "reports", f"{tenant_id}_{client_id}") + os.makedirs(reports_dir, exist_ok=True) + + csv_path_device_configs = os.path.join(reports_dir, "intune_device_configs.csv") + csv_path_config_policies = os.path.join(reports_dir, "intune_config_policies.csv") + csv_path_apps = os.path.join(reports_dir, "intune_apps.csv") + csv_path_detected_apps = os.path.join(reports_dir, "intune_detected_apps.csv") + csv_path_managed_devices = os.path.join(reports_dir, "intune_managed_devices.csv") + csv_path_device_compliance = os.path.join(reports_dir, "intune_device_compliance.csv") + csv_path_android_compliance = os.path.join(reports_dir, "intune_android_compliance.csv") + csv_path_ios_compliance = os.path.join(reports_dir, "intune_ios_compliance.csv") + csv_path_mdm_policies = os.path.join(reports_dir, "intune_mdm_policies.csv") + csv_path_byod_configs = os.path.join(reports_dir, "intune_byod_configs.csv") + + temp_path_device_configs = csv_path_device_configs + ".tmp" + temp_path_config_policies = csv_path_config_policies + ".tmp" + temp_path_apps = csv_path_apps + ".tmp" + temp_path_detected_apps = csv_path_detected_apps + ".tmp" + temp_path_managed_devices = csv_path_managed_devices + ".tmp" + temp_path_device_compliance = csv_path_device_compliance + ".tmp" + temp_path_android_compliance = csv_path_android_compliance + ".tmp" + temp_path_ios_compliance = csv_path_ios_compliance + ".tmp" + temp_path_mdm_policies = csv_path_mdm_policies + ".tmp" + temp_path_byod_configs = csv_path_byod_configs + ".tmp" + + for path in [temp_path_device_configs, temp_path_config_policies]: + with open(path, 'w', encoding='utf-8', newline='') as f: + writer = csv.writer(f) + writer.writerow(["displayName", "platform", "policyType"]) + + with open(temp_path_apps, 'w', encoding='utf-8', newline='') as f: + writer = csv.writer(f) + writer.writerow(["displayName"]) + + errors = [] + + def fetch_device_configs(): + try: + run_device_configs_pipeline( + client_id=client_id, + client_secret=client_secret, + tenant_id=tenant_id, + endpoint_name="deviceConfigurations", + csv_path=temp_path_device_configs, + on_page_callback=on_page_callback, + is_cancelled_callback=is_cancelled_callback + ) + except Exception as e: + errors.append(e) + + def fetch_config_policies(): + try: + run_device_configs_pipeline( + client_id=client_id, + client_secret=client_secret, + tenant_id=tenant_id, + endpoint_name="configurationPolicies", + csv_path=temp_path_config_policies, + on_page_callback=on_page_callback, + is_cancelled_callback=is_cancelled_callback + ) + except Exception as e: + errors.append(e) + + def fetch_mobile_apps(): + try: + run_mobile_apps_pipeline( + client_id=client_id, + client_secret=client_secret, + tenant_id=tenant_id, + csv_path=temp_path_apps, + on_page_callback=on_apps_page_callback, + is_cancelled_callback=is_cancelled_callback + ) + except Exception as e: + errors.append(e) + + def fetch_detected_apps(): + try: + run_detected_apps_pipeline( + client_id=client_id, + client_secret=client_secret, + tenant_id=tenant_id, + csv_path=temp_path_detected_apps, + is_cancelled_callback=is_cancelled_callback + ) + except Exception as e: + errors.append(e) + + def fetch_managed_devices(): + try: + run_managed_devices_pipeline( + client_id=client_id, + client_secret=client_secret, + tenant_id=tenant_id, + csv_path=temp_path_managed_devices, + is_cancelled_callback=is_cancelled_callback + ) + except Exception as e: + errors.append(e) + + def fetch_android_compliance(): + try: + run_device_compliance_pipeline( + client_id=client_id, + client_secret=client_secret, + tenant_id=tenant_id, + csv_path=temp_path_android_compliance, + filter_type='microsoft.graph.androidCompliancePolicy', + is_cancelled_callback=is_cancelled_callback + ) + except Exception as e: + errors.append(e) + + def fetch_ios_compliance(): + try: + run_device_compliance_pipeline( + client_id=client_id, + client_secret=client_secret, + tenant_id=tenant_id, + csv_path=temp_path_ios_compliance, + filter_type='microsoft.graph.iosCompliancePolicy', + is_cancelled_callback=is_cancelled_callback + ) + except Exception as e: + errors.append(e) + + def fetch_mdm_policies(): + try: + run_mdm_policies_pipeline( + client_id=client_id, + client_secret=client_secret, + tenant_id=tenant_id, + csv_path=temp_path_mdm_policies, + delegated_token=delegated_token, + is_cancelled_callback=is_cancelled_callback + ) + except Exception as e: + errors.append(e) + + def fetch_byod_configs(): + try: + run_byod_configs_pipeline( + client_id=client_id, + client_secret=client_secret, + tenant_id=tenant_id, + csv_path=temp_path_byod_configs, + is_cancelled_callback=is_cancelled_callback + ) + except Exception as e: + errors.append(e) + + t1 = threading.Thread(target=fetch_device_configs, daemon=True) + t2 = threading.Thread(target=fetch_config_policies, daemon=True) + t3 = threading.Thread(target=fetch_mobile_apps, daemon=True) + t4 = threading.Thread(target=fetch_detected_apps, daemon=True) + t5 = threading.Thread(target=fetch_managed_devices, daemon=True) + t6 = threading.Thread(target=fetch_android_compliance, daemon=True) + t7 = threading.Thread(target=fetch_ios_compliance, daemon=True) + t8 = threading.Thread(target=fetch_mdm_policies, daemon=True) + t9 = threading.Thread(target=fetch_byod_configs, daemon=True) + + t1.start() + t2.start() + t3.start() + t4.start() + t5.start() + t6.start() + t7.start() + t8.start() + t9.start() + + t1.join() + t2.join() + t3.join() + t4.join() + t5.join() + t6.join() + t7.join() + t8.join() + t9.join() + + if len(errors) == 9: + raise errors[0] + + for temp, final in [ + (temp_path_device_configs, csv_path_device_configs), + (temp_path_config_policies, csv_path_config_policies), + (temp_path_apps, csv_path_apps), + (temp_path_detected_apps, csv_path_detected_apps), + (temp_path_managed_devices, csv_path_managed_devices), + (temp_path_device_compliance, csv_path_device_compliance), + (temp_path_android_compliance, csv_path_android_compliance), + (temp_path_ios_compliance, csv_path_ios_compliance), + (temp_path_mdm_policies, csv_path_mdm_policies), + (temp_path_byod_configs, csv_path_byod_configs) + ]: + if os.path.exists(temp): + if os.path.exists(final): + os.remove(final) + os.rename(temp, final) + + counts = defaultdict(int) + total_dc = 0 + total_cp = 0 + unique_apps = set() + + if os.path.exists(csv_path_device_configs): + with open(csv_path_device_configs, 'r', encoding='utf-8') as f: + reader = csv.reader(f) + next(reader, None) + for row in reader: + if len(row) >= 3: + platform, policy_type = row[1], row[2] + if platform and policy_type: + counts[(platform, policy_type)] += 1 + total_dc += 1 + + if os.path.exists(csv_path_config_policies): + with open(csv_path_config_policies, 'r', encoding='utf-8') as f: + reader = csv.reader(f) + next(reader, None) + for row in reader: + if len(row) >= 3: + platform, policy_type = row[1], row[2] + if platform and policy_type: + counts[(platform, policy_type)] += 1 + total_cp += 1 + + if os.path.exists(csv_path_apps): + with open(csv_path_apps, 'r', encoding='utf-8') as f: + reader = csv.reader(f) + next(reader, None) + for row in reader: + if len(row) >= 1: + app_name = row[0] + if app_name: + unique_apps.add(app_name) + + rows = [] + for (platform, p_type), count in sorted(counts.items()): + rows.append((platform, p_type, str(count))) + + detected_rows_for_ui = [] + if os.path.exists(csv_path_detected_apps): + df_detected = pd.read_csv(csv_path_detected_apps) + df_slice = df_detected.head(200).fillna("N/A") + detected_rows_for_ui = df_slice.to_dict('records') + + managed_devices_rows_for_ui = [] + if os.path.exists(csv_path_managed_devices): + df_managed = pd.read_csv(csv_path_managed_devices) + df_slice = df_managed.head(200).fillna("N/A") + managed_devices_rows_for_ui = df_slice.to_dict('records') + + android_compliance_rows_for_ui = [] + if os.path.exists(csv_path_android_compliance): + df_android = pd.read_csv(csv_path_android_compliance) + df_slice = df_android.head(200).fillna("N/A") + android_compliance_rows_for_ui = df_slice.to_dict('records') + + ios_compliance_rows_for_ui = [] + if os.path.exists(csv_path_ios_compliance): + df_ios = pd.read_csv(csv_path_ios_compliance) + df_slice = df_ios.head(200).fillna("N/A") + ios_compliance_rows_for_ui = df_slice.to_dict('records') + + mdm_policies_rows_for_ui = [] + if os.path.exists(csv_path_mdm_policies): + df_mdm = pd.read_csv(csv_path_mdm_policies) + df_slice = df_mdm.head(200).fillna("N/A") + mdm_policies_rows_for_ui = df_slice.to_dict('records') + + byod_configs_rows_for_ui = [] + if os.path.exists(csv_path_byod_configs): + df_byod = pd.read_csv(csv_path_byod_configs) + df_slice = df_byod.head(200).fillna("N/A") + byod_configs_rows_for_ui = df_slice.to_dict('records') + + return { + "total_device_configs": total_dc, + "total_config_policies": total_cp, + "table_rows": rows, + "mobile_apps": sorted(list(unique_apps)), + "detected_apps": detected_rows_for_ui, + "managed_devices": managed_devices_rows_for_ui, + "android_compliance": android_compliance_rows_for_ui, + "ios_compliance": ios_compliance_rows_for_ui, + "mdm_policies": mdm_policies_rows_for_ui, + "byod_configs": byod_configs_rows_for_ui + } + + +class IntuneService: + """Legacy backward compatibility wrapper for Intune Graph query service.""" + def __init__(self, client: GraphClient): + self.client = client + + def fetch_configuration_records(self, endpoint_name: str, csv_path: str, max_rows: int = 10000, on_page_callback=None, is_cancelled_callback=None) -> None: + run_device_configs_pipeline( + client_id=self.client.client_ids[0] if isinstance(self.client.client_ids, list) else self.client.client_ids, + client_secret=self.client.client_secrets[0] if isinstance(self.client.client_secrets, list) else self.client.client_secrets, + tenant_id=self.client.tenant_id, + endpoint_name=endpoint_name, + csv_path=csv_path, + max_rows=max_rows, + on_page_callback=on_page_callback, + is_cancelled_callback=is_cancelled_callback + ) + + def fetch_mobile_apps(self, csv_path: str, max_rows: int = 5000, on_page_callback=None, is_cancelled_callback=None) -> None: + run_mobile_apps_pipeline( + client_id=self.client.client_ids[0] if isinstance(self.client.client_ids, list) else self.client.client_ids, + client_secret=self.client.client_secrets[0] if isinstance(self.client.client_secrets, list) else self.client.client_secrets, + tenant_id=self.client.tenant_id, + csv_path=csv_path, + max_rows=max_rows, + on_page_callback=on_page_callback, + is_cancelled_callback=is_cancelled_callback + ) + + def fetch_detected_apps(self, csv_path: str = None, max_rows: int = 10000, is_cancelled_callback=None) -> list: + return run_detected_apps_pipeline( + client_id=self.client.client_ids[0] if isinstance(self.client.client_ids, list) else self.client.client_ids, + client_secret=self.client.client_secrets[0] if isinstance(self.client.client_secrets, list) else self.client.client_secrets, + tenant_id=self.client.tenant_id, + csv_path=csv_path, + max_rows=max_rows, + is_cancelled_callback=is_cancelled_callback + ) + + def fetch_managed_devices(self, csv_path: str = None, max_rows: int = 10000, is_cancelled_callback=None) -> list: + return run_managed_devices_pipeline( + client_id=self.client.client_ids[0] if isinstance(self.client.client_ids, list) else self.client.client_ids, + client_secret=self.client.client_secrets[0] if isinstance(self.client.client_secrets, list) else self.client.client_secrets, + tenant_id=self.client.tenant_id, + csv_path=csv_path, + max_rows=max_rows, + is_cancelled_callback=is_cancelled_callback + ) + + def fetch_android_compliance(self, csv_path: str = None, max_rows: int = 10000, is_cancelled_callback=None) -> list: + return run_device_compliance_pipeline( + client_id=self.client.client_ids[0] if isinstance(self.client.client_ids, list) else self.client.client_ids, + client_secret=self.client.client_secrets[0] if isinstance(self.client.client_secrets, list) else self.client.client_secrets, + tenant_id=self.client.tenant_id, + csv_path=csv_path, + filter_type='microsoft.graph.androidCompliancePolicy', + max_rows=max_rows, + is_cancelled_callback=is_cancelled_callback + ) + + def fetch_ios_compliance(self, csv_path: str = None, max_rows: int = 10000, is_cancelled_callback=None) -> list: + return run_device_compliance_pipeline( + client_id=self.client.client_ids[0] if isinstance(self.client.client_ids, list) else self.client.client_ids, + client_secret=self.client.client_secrets[0] if isinstance(self.client.client_secrets, list) else self.client.client_secrets, + tenant_id=self.client.tenant_id, + csv_path=csv_path, + filter_type='microsoft.graph.iosCompliancePolicy', + max_rows=max_rows, + is_cancelled_callback=is_cancelled_callback + ) + + def fetch_mdm_policies(self, csv_path: str = None, delegated_token: str = None, max_rows: int = 1000, is_cancelled_callback=None) -> list: + return run_mdm_policies_pipeline( + client_id=self.client.client_ids[0] if isinstance(self.client.client_ids, list) else self.client.client_ids, + client_secret=self.client.client_secrets[0] if isinstance(self.client.client_secrets, list) else self.client.client_secrets, + tenant_id=self.client.tenant_id, + csv_path=csv_path, + delegated_token=delegated_token, + max_rows=max_rows, + is_cancelled_callback=is_cancelled_callback + ) + + def fetch_byod_configs(self, csv_path: str = None, max_rows: int = 1000, is_cancelled_callback=None) -> list: + return run_byod_configs_pipeline( + client_id=self.client.client_ids[0] if isinstance(self.client.client_ids, list) else self.client.client_ids, + client_secret=self.client.client_secrets[0] if isinstance(self.client.client_secrets, list) else self.client.client_secrets, + tenant_id=self.client.tenant_id, + csv_path=csv_path, + max_rows=max_rows, + is_cancelled_callback=is_cancelled_callback + ) diff --git a/core/graph/intune/byod_configs.py b/core/graph/intune/byod_configs.py new file mode 100644 index 00000000..6fc6f2d6 --- /dev/null +++ b/core/graph/intune/byod_configs.py @@ -0,0 +1,110 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Microsoft Intune Mobile BYOD Configurations data pipeline.""" + +import logging +import pandas as pd +from core.graph.client import GraphClient + +logger = logging.getLogger(__name__) + +def format_restriction(restriction: dict) -> str: + """Helper to format a restriction dict into a single line readable string.""" + if not restriction or not isinstance(restriction, dict): + return "N/A" + blocked = "Yes" if restriction.get("platformBlocked") else "No" + personal_blocked = "Yes" if restriction.get("personalDeviceEnrollmentBlocked") else "No" + min_ver = restriction.get("osMinimumVersion") or "None" + max_ver = restriction.get("osMaximumVersion") or "None" + return f"Blocked: {blocked}, Personal Blocked: {personal_blocked}, Min OS: {min_ver}, Max OS: {max_ver}" + +def run_byod_configs_pipeline( + client_id: str, + client_secret: str, + tenant_id: str, + csv_path: str = None, + max_rows: int = 1000, + is_cancelled_callback=None +) -> list: + """Fetch Mobile BYOD Configurations from deviceManagement/deviceEnrollmentConfigurations.""" + client = GraphClient( + tenant_id=tenant_id, + client_ids=client_id, + client_secrets=client_secret, + concurrency=1, + retries=5, + backoff=2 + ) + client.authenticate() + + url = "https://graph.microsoft.com/v1.0/deviceManagement/deviceEnrollmentConfigurations" + token_slot = client.get_active_token() + session = client.get_session() + + headers = { + "Authorization": f"Bearer {token_slot['token']}", + "Accept": "application/json" + } + + rows = [] + try: + logger.info("Querying Intune Device Enrollment Configurations for Mobile BYOD...") + while url and len(rows) < max_rows: + if is_cancelled_callback and is_cancelled_callback(): + logger.info("Mobile BYOD configurations scan cancelled. Aborting.") + break + resp = session.get(url, headers=headers) + if resp.status_code == 200: + data = resp.json() + value_list = data.get("value", []) + + for item in value_list: + if len(rows) >= max_rows: + break + + # Filter for platform restrictions configuration type + odata_type = item.get("@odata.type") + if odata_type != "#microsoft.graph.deviceEnrollmentPlatformRestrictionsConfiguration": + continue + + ios_rest = format_restriction(item.get("iosRestriction")) + win_mob_rest = format_restriction(item.get("windowsMobileRestriction")) + android_rest = format_restriction(item.get("androidRestriction")) + + rows.append({ + "displayName": item.get("displayName") or "N/A", + "description": item.get("description") or "N/A", + "priority": item.get("priority", 0), + "lastModifiedDateTime": item.get("lastModifiedDateTime") or "N/A", + "iosRestrictions": ios_rest, + "windowsMobileRestrictions": win_mob_rest, + "androidRestrictions": android_rest + }) + + url = data.get("@odata.nextLink") + elif resp.status_code in [401, 403]: + logger.error("BYOD configs access denied: %d %s", resp.status_code, resp.text) + raise PermissionError("DeviceManagementServiceConfig.Read.All permission required.") + else: + logger.error("BYOD configs endpoint failed: %d %s", resp.status_code, resp.text) + raise ConnectionError(f"API request failed with status {resp.status_code}") + + if csv_path and rows: + df = pd.DataFrame(rows) + df.to_csv(csv_path, index=False, encoding='utf-8') + return rows + finally: + client.release_token(token_slot) + client.close() diff --git a/core/graph/intune/detected_apps.py b/core/graph/intune/detected_apps.py new file mode 100644 index 00000000..3e0eac49 --- /dev/null +++ b/core/graph/intune/detected_apps.py @@ -0,0 +1,83 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Microsoft Intune Detected Apps telemetry scanner data pipeline.""" + +import logging +import pandas as pd +from core.graph.client import GraphClient + +logger = logging.getLogger(__name__) + +def run_detected_apps_pipeline( + client_id: str, + client_secret: str, + tenant_id: str, + csv_path: str = None, + max_rows: int = 10000, + is_cancelled_callback=None +) -> list: + """Fetch detected apps from Microsoft Graph /deviceManagement/detectedApps.""" + client = GraphClient( + tenant_id=tenant_id, + client_ids=client_id, + client_secrets=client_secret, + concurrency=1, + retries=5, + backoff=2 + ) + client.authenticate() + + url = "https://graph.microsoft.com/v1.0/deviceManagement/detectedApps" + token_slot = client.get_active_token() + session = client.get_session() + + headers = { + "Authorization": f"Bearer {token_slot['token']}", + "Accept": "application/json" + } + + rows = [] + try: + logger.info("Querying Intune Detected Apps...") + while url and len(rows) < max_rows: + if is_cancelled_callback and is_cancelled_callback(): break + resp = session.get(url, headers=headers) + if resp.status_code == 200: + data = resp.json() + value_list = data.get("value", []) + + for item in value_list: + if len(rows) >= max_rows: + break + rows.append({ + "displayName": item.get("displayName") or "N/A", + "version": item.get("version") or "N/A", + "publisher": item.get("publisher") or "N/A", + "platform": item.get("platform") or "unknown", + "deviceCount": item.get("deviceCount") or 0 + }) + + url = data.get("@odata.nextLink") + else: + logger.error("Detected apps endpoint failed: %d %s", resp.status_code, resp.text) + raise ConnectionError(f"API request failed with status {resp.status_code}") + + if csv_path and rows: + df = pd.DataFrame(rows) + df.to_csv(csv_path, index=False, encoding='utf-8') + return rows + finally: + client.release_token(token_slot) + client.close() diff --git a/core/graph/intune/device_compliance.py b/core/graph/intune/device_compliance.py new file mode 100644 index 00000000..cd54aeaf --- /dev/null +++ b/core/graph/intune/device_compliance.py @@ -0,0 +1,87 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Microsoft Intune Device Compliance Policies telemetry scanner data pipeline.""" + +import logging +import pandas as pd +from core.graph.client import GraphClient + +logger = logging.getLogger(__name__) + +def run_device_compliance_pipeline( + client_id: str, + client_secret: str, + tenant_id: str, + csv_path: str = None, + filter_type: str = None, + max_rows: int = 10000, + is_cancelled_callback=None +) -> list: + """Fetch device compliance policies from Microsoft Graph /deviceManagement/deviceCompliancePolicies.""" + client = GraphClient( + tenant_id=tenant_id, + client_ids=client_id, + client_secrets=client_secret, + concurrency=1, + retries=5, + backoff=2 + ) + client.authenticate() + + if filter_type: + url = f"https://graph.microsoft.com/v1.0/deviceManagement/deviceCompliancePolicies?$filter=isof('{filter_type}')" + else: + url = "https://graph.microsoft.com/v1.0/deviceManagement/deviceCompliancePolicies" + + token_slot = client.get_active_token() + session = client.get_session() + + headers = { + "Authorization": f"Bearer {token_slot['token']}", + "Accept": "application/json" + } + + rows = [] + try: + logger.info(f"Querying Intune Device Compliance Policies (filter: {filter_type or 'None'})...") + while url and len(rows) < max_rows: + if is_cancelled_callback and is_cancelled_callback(): + logger.info("Device compliance scan cancelled. Aborting.") + break + resp = session.get(url, headers=headers) + if resp.status_code == 200: + data = resp.json() + value_list = data.get("value", []) + + for item in value_list: + if len(rows) >= max_rows: + break + rows.append(item) + + url = data.get("@odata.nextLink") + elif resp.status_code in [401, 403]: + logger.error("Device compliance access denied: %d %s", resp.status_code, resp.text) + raise PermissionError("DeviceManagementConfiguration.Read.All permission required for compliance policies.") + else: + logger.error("Device compliance endpoint failed: %d %s", resp.status_code, resp.text) + raise ConnectionError(f"API request failed with status {resp.status_code}") + + if csv_path and rows: + df = pd.DataFrame(rows) + df.to_csv(csv_path, index=False, encoding='utf-8') + return rows + finally: + client.release_token(token_slot) + client.close() diff --git a/core/graph/intune/device_configs.py b/core/graph/intune/device_configs.py new file mode 100644 index 00000000..9a721195 --- /dev/null +++ b/core/graph/intune/device_configs.py @@ -0,0 +1,157 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Microsoft Intune Device Configurations telemetry scanner data pipeline.""" + +import csv +import re +import logging +import time +from core.graph.client import GraphClient + +logger = logging.getLogger(__name__) + +def run_device_configs_pipeline( + client_id: str, + client_secret: str, + tenant_id: str, + endpoint_name: str, + csv_path: str, + max_rows: int = 10000, + on_page_callback=None, + is_cancelled_callback=None +): + """Fetch Intune configuration policies from Graph beta endpoint.""" + client = GraphClient( + tenant_id=tenant_id, + client_ids=client_id, + client_secrets=client_secret, + concurrency=1, + retries=5, + backoff=2 + ) + client.authenticate() + + token_slot = client.get_active_token() + session = client.get_session() + + headers = { + "Authorization": f"Bearer {token_slot['token']}", + "Accept": "application/json" + } + + base_url = f"https://graph.microsoft.com/beta/deviceManagement/{endpoint_name}" + next_url = f"{base_url}?$top=100" + rows_written = 0 + page_number = 1 + + try: + logger.info("Fetching Intune configurations from endpoint %s...", endpoint_name) + with open(csv_path, 'a', encoding='utf-8', newline='') as f: + writer = csv.writer(f) + + while next_url and rows_written < max_rows: + if is_cancelled_callback and is_cancelled_callback(): + logger.info("Fetch cancelled in-flight for Intune %s. Aborting.", endpoint_name) + break + + retries_left = 2 + resp = None + while retries_left > 0: + try: + resp = session.get(next_url, headers=headers, timeout=40.0) + if resp.status_code == 200: + break + elif resp.status_code in [401, 403]: + break + except Exception as get_err: + logger.warning("Intune query attempt failed: %s", get_err) + + retries_left -= 1 + if retries_left > 0: + time.sleep(2) + + page_number += 1 + if resp and resp.status_code == 200: + data = resp.json() + value_list = data.get("value", []) + parsed_records = [] + + for item in value_list: + display_name = item.get("displayName", "") + + if endpoint_name == "deviceConfigurations": + odata_type = item.get("@odata.type", "") + type_str = odata_type.replace("#microsoft.graph.", "") + platform = "Unknown" + + if type_str.startswith("windows10"): + platform = "Windows 10" + policy_type = type_str.replace("windows10", "") + elif type_str.startswith("windows"): + platform = "Windows" + policy_type = type_str.replace("windows", "") + elif type_str.startswith("ios"): + platform = "iOS" + policy_type = type_str.replace("ios", "") + elif type_str.startswith("android"): + platform = "Android" + policy_type = type_str.replace("android", "") + elif type_str.startswith("macOS"): + platform = "macOS" + policy_type = type_str.replace("macOS", "") + else: + policy_type = type_str + + if not policy_type: + policy_type = "Configuration" + policy_type = re.sub(r"([A-Z])", r" \1", policy_type).strip() + else: # configurationPolicies + raw_platform = item.get("platforms", "Unknown") + if raw_platform == "windows10AndLater": + platform = "Windows 10" + elif raw_platform == "windows81AndLater": + platform = "Windows 8.1" + elif raw_platform == "macOS": + platform = "macOS" + else: + platform = raw_platform.capitalize() + policy_type = "Settings Catalog" + + writer.writerow([display_name, platform, policy_type]) + parsed_records.append({ + "displayName": display_name, + "platform": platform, + "policyType": policy_type + }) + rows_written += 1 + + if on_page_callback: + on_page_callback(parsed_records) + + if rows_written >= max_rows: + break + next_url = data.get("@odata.nextLink") + else: + if resp and resp.status_code in [401, 403]: + logger.error("Intune endpoint access denied: %d %s", resp.status_code, resp.text) + raise PermissionError("DeviceManagementConfiguration.Read.All permission required.") + else: + status_str = f"status {resp.status_code}" if resp else "connection/timeout error" + logger.warning("Intune query failed after 2 attempts (%s). Stopping pagination.", status_str) + break + logger.info("Successfully fetched %d records for Intune %s", rows_written, endpoint_name) + finally: + client.release_token(token_slot) + client.close() diff --git a/core/graph/intune/managed_devices.py b/core/graph/intune/managed_devices.py new file mode 100644 index 00000000..79113bb3 --- /dev/null +++ b/core/graph/intune/managed_devices.py @@ -0,0 +1,92 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Microsoft Intune Managed Devices telemetry scanner data pipeline.""" + +import logging +import pandas as pd +from core.graph.client import GraphClient + +logger = logging.getLogger(__name__) + +def run_managed_devices_pipeline( + client_id: str, + client_secret: str, + tenant_id: str, + csv_path: str = None, + max_rows: int = 10000, + is_cancelled_callback=None +) -> list: + """Fetch managed devices from Microsoft Graph /deviceManagement/managedDevices.""" + client = GraphClient( + tenant_id=tenant_id, + client_ids=client_id, + client_secrets=client_secret, + concurrency=1, + retries=5, + backoff=2 + ) + client.authenticate() + + url = "https://graph.microsoft.com/v1.0/deviceManagement/managedDevices" + token_slot = client.get_active_token() + session = client.get_session() + + headers = { + "Authorization": f"Bearer {token_slot['token']}", + "Accept": "application/json" + } + + rows = [] + try: + logger.info("Querying Intune Managed Devices...") + while url and len(rows) < max_rows: + if is_cancelled_callback and is_cancelled_callback(): + logger.info("Managed devices scan cancelled. Aborting.") + break + resp = session.get(url, headers=headers) + if resp.status_code == 200: + data = resp.json() + value_list = data.get("value", []) + + for item in value_list: + if len(rows) >= max_rows: + break + rows.append({ + "userId": item.get("userId") or "N/A", + "deviceName": item.get("deviceName") or "N/A", + "operatingSystem": item.get("operatingSystem") or "N/A", + "managementAgent": str(item.get("managementAgent") or "unknown"), + "deviceRegistrationState": str(item.get("deviceRegistrationState") or "unknown"), + "model": item.get("model") or "N/A", + "manufacturer": item.get("manufacturer") or "N/A", + "userPrincipalName": item.get("userPrincipalName") or "N/A", + "emailAddress": item.get("emailAddress") or "N/A" + }) + + url = data.get("@odata.nextLink") + elif resp.status_code in [401, 403]: + logger.error("Managed devices access denied: %d %s", resp.status_code, resp.text) + raise PermissionError("DeviceManagementManagedDevices.Read.All permission required.") + else: + logger.error("Managed devices endpoint failed: %d %s", resp.status_code, resp.text) + raise ConnectionError(f"API request failed with status {resp.status_code}") + + if csv_path and rows: + df = pd.DataFrame(rows) + df.to_csv(csv_path, index=False, encoding='utf-8') + return rows + finally: + client.release_token(token_slot) + client.close() diff --git a/core/graph/intune/mdm_policies.py b/core/graph/intune/mdm_policies.py new file mode 100644 index 00000000..d5d7ca42 --- /dev/null +++ b/core/graph/intune/mdm_policies.py @@ -0,0 +1,110 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Microsoft Intune Mobile Device Management (MDM) Policies telemetry scanner data pipeline.""" + +import logging +import requests +import pandas as pd +from core.graph.client import GraphClient + +logger = logging.getLogger(__name__) + +def run_mdm_policies_pipeline( + client_id: str, + client_secret: str, + tenant_id: str, + csv_path: str = None, + delegated_token: str = None, + max_rows: int = 1000, + is_cancelled_callback=None +) -> list: + """Fetch Mobile Device Management Policies from beta/policies/mobileDeviceManagementPolicies.""" + client = None + token = None + session = None + + if delegated_token: + token = delegated_token + session = requests.Session() + else: + client = GraphClient( + tenant_id=tenant_id, + client_ids=client_id, + client_secrets=client_secret, + concurrency=1, + retries=5, + backoff=2 + ) + client.authenticate() + token_slot = client.get_active_token() + token = token_slot['token'] + session = client.get_session() + + url = "https://graph.microsoft.com/beta/policies/mobileDeviceManagementPolicies?$filter=isValid eq true" + + headers = { + "Authorization": f"Bearer {token}", + "Accept": "application/json" + } + + rows = [] + try: + logger.info("Querying Intune Mobile Device Management Policies...") + while url and len(rows) < max_rows: + if is_cancelled_callback and is_cancelled_callback(): + logger.info("MDM policies scan cancelled. Aborting.") + break + resp = session.get(url, headers=headers) + if resp.status_code == 200: + data = resp.json() + value_list = data.get("value", []) + + for item in value_list: + if len(rows) >= max_rows: + break + # Map appliesTo to a readable Auto-Enroll value (e.g. 'all', 'selected', 'none') + applies_to = item.get("appliesTo") + if isinstance(applies_to, str): + applies_to_str = applies_to.capitalize() + else: + applies_to_str = "None" + + rows.append({ + "displayName": item.get("displayName") or "N/A", + "description": item.get("description") or "N/A", + "appliesTo": applies_to_str, + "discoveryUrl": item.get("discoveryUrl") or "N/A", + "termsOfUseUrl": item.get("termsOfUseUrl") or "N/A", + "complianceUrl": item.get("complianceUrl") or "N/A" + }) + + url = data.get("@odata.nextLink") + elif resp.status_code in [401, 403]: + logger.error("MDM policies access denied: %d %s", resp.status_code, resp.text) + raise PermissionError("Policy.Read.All or Policy.ReadWrite.ConditionalAccess permission required.") + else: + logger.error("MDM policies endpoint failed: %d %s", resp.status_code, resp.text) + raise ConnectionError(f"API request failed with status {resp.status_code}") + + if csv_path and rows: + df = pd.DataFrame(rows) + df.to_csv(csv_path, index=False, encoding='utf-8') + return rows + finally: + if client: + client.release_token(token_slot) + client.close() + elif session: + session.close() diff --git a/core/graph/intune/mobile_apps.py b/core/graph/intune/mobile_apps.py new file mode 100644 index 00000000..3777dbd2 --- /dev/null +++ b/core/graph/intune/mobile_apps.py @@ -0,0 +1,114 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Microsoft Intune Mobile Apps telemetry scanner data pipeline.""" + +import csv +import logging +import time +from core.graph.client import GraphClient + +logger = logging.getLogger(__name__) + +def run_mobile_apps_pipeline( + client_id: str, + client_secret: str, + tenant_id: str, + csv_path: str, + max_rows: int = 5000, + on_page_callback=None, + is_cancelled_callback=None +): + """Fetch mobile apps from Microsoft Graph /deviceAppManagement/mobileApps.""" + client = GraphClient( + tenant_id=tenant_id, + client_ids=client_id, + client_secrets=client_secret, + concurrency=1, + retries=5, + backoff=2 + ) + client.authenticate() + + token_slot = client.get_active_token() + session = client.get_session() + + headers = { + "Authorization": f"Bearer {token_slot['token']}", + "Accept": "application/json" + } + + base_url = "https://graph.microsoft.com/beta/deviceAppManagement/mobileApps" + next_url = f"{base_url}?$select=displayName&$top=100" + rows_written = 0 + page_number = 1 + + try: + logger.info("Fetching Intune mobile apps...") + with open(csv_path, 'a', encoding='utf-8', newline='') as f: + writer = csv.writer(f) + + while next_url and rows_written < max_rows: + if is_cancelled_callback and is_cancelled_callback(): + logger.info("Fetch cancelled in-flight for Intune mobile apps. Aborting.") + break + + retries_left = 2 + resp = None + while retries_left > 0: + try: + resp = session.get(next_url, headers=headers, timeout=40.0) + if resp.status_code == 200: + break + elif resp.status_code in [401, 403]: + break + except Exception as get_err: + logger.warning("Intune apps query attempt failed: %s", get_err) + + retries_left -= 1 + if retries_left > 0: + time.sleep(2) + + page_number += 1 + if resp and resp.status_code == 200: + data = resp.json() + value_list = data.get("value", []) + parsed_records = [] + + for item in value_list: + display_name = item.get("displayName", "") + writer.writerow([display_name]) + parsed_records.append({ + "displayName": display_name + }) + rows_written += 1 + + if on_page_callback: + on_page_callback(parsed_records) + + if rows_written >= max_rows: + break + next_url = data.get("@odata.nextLink") + else: + if resp and resp.status_code in [401, 403]: + logger.error("Intune apps endpoint access denied: %d %s", resp.status_code, resp.text) + raise PermissionError("DeviceManagementApps.Read.All permission required.") + else: + status_str = f"status {resp.status_code}" if resp else "connection/timeout error" + logger.warning("Intune apps query failed after 2 attempts (%s). Stopping pagination.", status_str) + break + logger.info("Successfully fetched %d mobile apps", rows_written) + finally: + client.release_token(token_slot) + client.close() diff --git a/core/graph/m365_apps/__init__.py b/core/graph/m365_apps/__init__.py new file mode 100644 index 00000000..399b3508 --- /dev/null +++ b/core/graph/m365_apps/__init__.py @@ -0,0 +1,19 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Facade exposing M365 Apps data pipeline functions.""" + +from core.graph.m365_apps.active_users import run_o365_pipeline, process_active_user_detail +from core.graph.m365_apps.active_users_trend import run_o365_trend_pipeline, process_active_user_counts +from core.graph.m365_apps.app_usage import run_m365_pipeline, process_m365_app_user_detail diff --git a/core/graph/m365_apps/active_users.py b/core/graph/m365_apps/active_users.py new file mode 100644 index 00000000..777b4a42 --- /dev/null +++ b/core/graph/m365_apps/active_users.py @@ -0,0 +1,107 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""O365 Active Users telemetry data pipeline and processing logic.""" + +import os +import logging +import pandas as pd +from core.graph.client import GraphClient +from core.graph.reports import ReportsService + +logger = logging.getLogger(__name__) + +def _get_reports_service(client_id: str, client_secret: str, tenant_id: str) -> tuple[GraphClient, ReportsService]: + """Helper to instantiate GraphClient/ReportsService.""" + client = GraphClient( + tenant_id=tenant_id, + client_ids=client_id, + client_secrets=client_secret, + concurrency=2, + retries=5, + backoff=2 + ) + client.authenticate() + return client, ReportsService(client) + +def process_active_user_detail(filepath: str): + """Streams the downloaded CSV and calculates usage counters over 30, 90, and 180 days.""" + logger.info(f"Processing O365 file: {os.path.basename(filepath)}") + + if not os.path.exists(filepath): + logger.error(f"Error: Could not find the file {filepath} to process.") + raise FileNotFoundError(f"Report file {os.path.basename(filepath)} not found. Download may have failed.") + + current_date = pd.Timestamp.today().normalize() + + headers = pd.read_csv(filepath, nrows=0).columns.tolist() + expected = [ + "Has Exchange License", "Exchange Last Activity Date", + "Has OneDrive License", "OneDrive Last Activity Date", + "Has SharePoint License", "SharePoint Last Activity Date", + "Has Teams License", "Teams Last Activity Date" + ] + cols = [c for c in expected if c in headers] + + exchange_online_usage = [0, 0, 0] + onedrive_usage = [0, 0, 0] + sharepoint_usage = [0, 0, 0] + teams_usage = [0, 0, 0] + + def process_chunk_col(chunk, has_license_col, date_col): + if has_license_col not in chunk.columns or date_col not in chunk.columns: + return [0, 0, 0] + mask = chunk[has_license_col].astype(str).str.strip().str.upper() == "TRUE" + dates_series = pd.to_datetime(chunk.loc[mask, date_col], errors='coerce') + days_diff = (current_date - dates_series).dt.days + d180 = int((days_diff < 180).sum()) + d90 = int((days_diff < 90).sum()) + d30 = int((days_diff < 30).sum()) + return [d30, d90, d180] + + for chunk in pd.read_csv(filepath, usecols=cols, chunksize=10000, encoding="utf-8-sig"): + e_chunk = process_chunk_col(chunk, "Has Exchange License", "Exchange Last Activity Date") + exchange_online_usage = [x + y for x, y in zip(exchange_online_usage, e_chunk)] + + od_chunk = process_chunk_col(chunk, "Has OneDrive License", "OneDrive Last Activity Date") + onedrive_usage = [x + y for x, y in zip(onedrive_usage, od_chunk)] + + sp_chunk = process_chunk_col(chunk, "Has SharePoint License", "SharePoint Last Activity Date") + sharepoint_usage = [x + y for x, y in zip(sharepoint_usage, sp_chunk)] + + t_chunk = process_chunk_col(chunk, "Has Teams License", "Teams Last Activity Date") + teams_usage = [x + y for x, y in zip(teams_usage, t_chunk)] + + logger.info("Successfully processed O365 active user data in chunks.") + return [ + ("Exchange Online", exchange_online_usage[0], exchange_online_usage[1], exchange_online_usage[2]), + ("OneDrive", onedrive_usage[0], onedrive_usage[1], onedrive_usage[2]), + ("SharePoint", sharepoint_usage[0], sharepoint_usage[1], sharepoint_usage[2]), + ("Teams", teams_usage[0], teams_usage[1], teams_usage[2]) + ] + +def run_o365_pipeline(client_id: str, client_secret: str, tenant_id: str): + """Pipeline specifically for O365 Active User Data.""" + logger.info("Starting isolated O365 Pipeline...") + client, service = _get_reports_service(client_id, client_secret, tenant_id) + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + # Go up from core/graph/m365_apps to telemetry folder reports + telemetry_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(script_dir))), "telemetry") + reports_dir = os.path.join(telemetry_dir, "reports", f"{tenant_id}_{client_id}") + + service.download_o365_active_user_detail(reports_dir) + client.close() + + return process_active_user_detail(os.path.join(reports_dir, "Office365ActiveUserDetail(180d).csv")) diff --git a/core/graph/m365_apps/active_users_trend.py b/core/graph/m365_apps/active_users_trend.py new file mode 100644 index 00000000..a81c27a6 --- /dev/null +++ b/core/graph/m365_apps/active_users_trend.py @@ -0,0 +1,99 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""O365 Active Users Trend telemetry data pipeline and processing logic.""" + +import os +import logging +import pandas as pd +from core.graph.client import GraphClient +from core.graph.reports import ReportsService + +logger = logging.getLogger(__name__) + +def _get_reports_service(client_id: str, client_secret: str, tenant_id: str) -> tuple[GraphClient, ReportsService]: + """Helper to instantiate GraphClient/ReportsService.""" + client = GraphClient( + tenant_id=tenant_id, + client_ids=client_id, + client_secrets=client_secret, + concurrency=2, + retries=5, + backoff=2 + ) + client.authenticate() + return client, ReportsService(client) + +def process_active_user_counts(filepath: str): + """Parses chronological usage data for plotting.""" + logger.info(f"Processing O365 Counts file: {os.path.basename(filepath)}") + if not os.path.exists(filepath): + logger.error(f"Error: Could not find the file {filepath} to process.") + raise FileNotFoundError(f"Report file {os.path.basename(filepath)} not found.") + + headers = pd.read_csv(filepath, nrows=0).columns.tolist() + expected = ["Report Date", "Office 365", "Exchange", "OneDrive", "SharePoint", "Teams"] + cols = [c for c in expected if c in headers] + dates = [] + office365 = [] + exchange = [] + onedrive = [] + sharepoint = [] + teams = [] + + for chunk in pd.read_csv(filepath, usecols=cols, chunksize=10000, encoding="utf-8-sig"): + if "Report Date" in chunk.columns: + chunk = chunk.sort_values(by="Report Date").fillna(0) + dates.extend(chunk["Report Date"].astype(str).tolist()) + else: + dates.extend([""] * len(chunk)) + + def extract_col(col_name): + if col_name in chunk.columns: + return pd.to_numeric(chunk[col_name], errors='coerce').fillna(0).astype(int).tolist() + return [0] * len(chunk) + + office365.extend(extract_col("Office 365")) + exchange.extend(extract_col("Exchange")) + onedrive.extend(extract_col("OneDrive")) + sharepoint.extend(extract_col("SharePoint")) + teams.extend(extract_col("Teams")) + + logger.info("Successfully processed O365 active user counts data.") + return { + "dates": dates, + "office365": office365, + "exchange": exchange, + "onedrive": onedrive, + "sharepoint": sharepoint, + "teams": teams + } + +def run_o365_trend_pipeline(client_id: str, client_secret: str, tenant_id: str): + """Pipeline specifically for O365 Trend Data.""" + try: + logger.info("Starting isolated O365 Trend Pipeline...") + client, service = _get_reports_service(client_id, client_secret, tenant_id) + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + telemetry_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(script_dir))), "telemetry") + reports_dir = os.path.join(telemetry_dir, "reports", f"{tenant_id}_{client_id}") + + service.download_o365_active_user_counts(reports_dir) + client.close() + + return process_active_user_counts(os.path.join(reports_dir, "Office365ActiveUserCounts(30d).csv")) + except Exception as e: + logger.error("O365 Trend pipeline failed.", exc_info=True) + raise diff --git a/core/graph/m365_apps/app_usage.py b/core/graph/m365_apps/app_usage.py new file mode 100644 index 00000000..f1018887 --- /dev/null +++ b/core/graph/m365_apps/app_usage.py @@ -0,0 +1,84 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""M365 App Usage telemetry data pipeline and processing logic.""" + +import os +import logging +import pandas as pd +from core.graph.client import GraphClient +from core.graph.reports import ReportsService + +logger = logging.getLogger(__name__) + +def _get_reports_service(client_id: str, client_secret: str, tenant_id: str) -> tuple[GraphClient, ReportsService]: + """Helper to instantiate GraphClient/ReportsService.""" + client = GraphClient( + tenant_id=tenant_id, + client_ids=client_id, + client_secrets=client_secret, + concurrency=2, + retries=5, + backoff=2 + ) + client.authenticate() + return client, ReportsService(client) + +def process_m365_app_user_detail(filepath: str): + """Streams the downloaded CSV and calculates usage counters.""" + logger.info(f"Processing M365 App file: {os.path.basename(filepath)}") + + if not os.path.exists(filepath): + logger.error(f"Error: Could not find the file {filepath} to process.") + raise FileNotFoundError(f"Report file {os.path.basename(filepath)} not found. Download may have failed.") + + columns_to_track = [ + "Windows", "Mac", "Mobile", "Web", "Outlook", "Word", "Excel", + "PowerPoint", "OneNote", "Teams", "Outlook (Windows)", "Word (Windows)", + "Excel (Windows)", "PowerPoint (Windows)", "OneNote (Windows)", + "Teams (Windows)", "Outlook (Mac)", "Word (Mac)", "Excel (Mac)", + "PowerPoint (Mac)", "OneNote (Mac)", "Teams (Mac)", "Outlook (Mobile)", + "Word (Mobile)", "Excel (Mobile)", "PowerPoint (Mobile)", + "OneNote (Mobile)", "Teams (Mobile)", "Outlook (Web)", "Word (Web)", + "Excel (Web)", "PowerPoint (Web)", "OneNote (Web)", "Teams (Web)" + ] + + headers = pd.read_csv(filepath, nrows=0).columns.tolist() + cols = [c for c in columns_to_track if c in headers] + + counters = {col: 0 for col in columns_to_track} + + for chunk in pd.read_csv(filepath, usecols=cols, chunksize=10000, encoding="utf-8-sig"): + for col in columns_to_track: + if col in chunk.columns: + col_series = chunk[col].astype(str).str.strip().str.lower() + count = int(col_series.isin(["yes", "true"]).sum()) + counters[col] += count + + logger.info("Successfully processed M365 App user data in chunks.") + return [(col, count) for col, count in counters.items()] + +def run_m365_pipeline(client_id: str, client_secret: str, tenant_id: str): + """Pipeline specifically for M365 Apps Data.""" + logger.info("Starting isolated M365 Apps Pipeline...") + client, service = _get_reports_service(client_id, client_secret, tenant_id) + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + telemetry_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(script_dir))), "telemetry") + reports_dir = os.path.join(telemetry_dir, "reports", f"{tenant_id}_{client_id}") + + service.download_m365_app_details(reports_dir) + client.close() + + return process_m365_app_user_detail(os.path.join(reports_dir, "M365AppUserDetail(180d).csv")) diff --git a/core/graph/network_security/__init__.py b/core/graph/network_security/__init__.py new file mode 100644 index 00000000..656544ba --- /dev/null +++ b/core/graph/network_security/__init__.py @@ -0,0 +1,59 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Facade exposing Network Security pipelines and compatibility service.""" + +from core.graph.client import GraphClient +from core.graph.network_security.filtering import run_filtering_pipeline +from core.graph.network_security.conditional_access import run_conditional_access_pipeline +from core.graph.network_security.firewall import run_firewall_pipeline + +class NetworkSecurityService: + """Service to interact with Network Security configurations (backward compatibility).""" + + def __init__(self, client: GraphClient) -> None: + self.client = client + + def fetch_filtering_policies(self, csv_path: str = None, on_page_callback=None, is_cancelled_callback=None) -> None: + """Fetches Entra Global Secure Access Filtering Policies (Beta).""" + run_filtering_pipeline( + client_id=self.client.client_ids[0] if isinstance(self.client.client_ids, list) else self.client.client_ids, + client_secret=self.client.client_secrets[0] if isinstance(self.client.client_secrets, list) else self.client.client_secrets, + tenant_id=self.client.tenant_id, + csv_path=csv_path, + on_page_callback=on_page_callback, + is_cancelled_callback=is_cancelled_callback + ) + + def fetch_conditional_access_policies(self, csv_path: str = None, on_page_callback=None, is_cancelled_callback=None) -> None: + """Fetches Entra ID Conditional Access policies.""" + run_conditional_access_pipeline( + client_id=self.client.client_ids[0] if isinstance(self.client.client_ids, list) else self.client.client_ids, + client_secret=self.client.client_secrets[0] if isinstance(self.client.client_secrets, list) else self.client.client_secrets, + tenant_id=self.client.tenant_id, + csv_path=csv_path, + on_page_callback=on_page_callback, + is_cancelled_callback=is_cancelled_callback + ) + + def fetch_firewall_policies(self, csv_path: str = None, on_page_callback=None, is_cancelled_callback=None) -> None: + """Fetches Intune device configurations and filters Firewall & Proxy configs.""" + run_firewall_pipeline( + client_id=self.client.client_ids[0] if isinstance(self.client.client_ids, list) else self.client.client_ids, + client_secret=self.client.client_secrets[0] if isinstance(self.client.client_secrets, list) else self.client.client_secrets, + tenant_id=self.client.tenant_id, + csv_path=csv_path, + on_page_callback=on_page_callback, + is_cancelled_callback=is_cancelled_callback + ) diff --git a/core/graph/network_security/conditional_access.py b/core/graph/network_security/conditional_access.py new file mode 100644 index 00000000..f50ea122 --- /dev/null +++ b/core/graph/network_security/conditional_access.py @@ -0,0 +1,96 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Microsoft Entra ID Conditional Access policies for Network Security data pipeline.""" + +import logging +import pandas as pd +from core.graph.client import GraphClient + +logger = logging.getLogger(__name__) + +def run_conditional_access_pipeline( + client_id: str, + client_secret: str, + tenant_id: str, + csv_path: str = None, + on_page_callback=None, + is_cancelled_callback=None +): + """Fetch Conditional Access policies.""" + client = GraphClient( + tenant_id=tenant_id, + client_ids=client_id, + client_secrets=client_secret, + concurrency=1, + retries=5, + backoff=2 + ) + client.authenticate() + + url = "https://graph.microsoft.com/v1.0/identity/conditionalAccess/policies" + token_slot = client.get_active_token() + session = client.get_session() + + headers = { + "Authorization": f"Bearer {token_slot['token']}", + "Accept": "application/json" + } + + rows = [] + try: + logger.info("Querying Conditional Access policies for Network Security...") + while url: + if is_cancelled_callback and is_cancelled_callback(): break + resp = session.get(url, headers=headers) + if resp.status_code == 200: + data = resp.json() + value_list = data.get("value", []) + + for p in value_list: + p["name"] = p.get("displayName") or p.get("name") or "N/A" + p["state"] = p.get("state") or "N/A" + + conds = p.get("conditions") or {} + users_cond = conds.get("users") or {} + apps_cond = conds.get("applications") or {} + + inc_users = users_cond.get("includeUsers") or [] + inc_groups = users_cond.get("includeGroups") or [] + p["target_users"] = "All Users" if "All" in inc_users else f"Specific ({len(inc_users)} users, {len(inc_groups)} groups)" + + inc_apps = apps_cond.get("includeApplications") or [] + p["target_apps"] = "All Apps" if "All" in inc_apps else f"Specific ({len(inc_apps)} apps)" + + grant_controls = p.get("grantControls") or {} + controls = grant_controls.get("builtInControls") or [] + p["controls"] = ", ".join(controls) if controls else "Block/None" + rows.append(p) + + if on_page_callback: + on_page_callback(value_list) + url = data.get("@odata.nextLink") + elif resp.status_code in [401, 403]: + logger.error("Conditional Access endpoint permission error: %d %s", resp.status_code, resp.text) + raise PermissionError("Policy.Read.All permission required for Conditional Access policies.") + else: + logger.error("Conditional Access endpoint failed: %d %s", resp.status_code, resp.text) + raise ConnectionError(f"API request failed with status {resp.status_code}") + + if csv_path: + df = pd.DataFrame(rows) if rows else pd.DataFrame() + df.to_csv(csv_path, index=False, encoding='utf-8') + finally: + client.release_token(token_slot) + client.close() diff --git a/core/graph/network_security/filtering.py b/core/graph/network_security/filtering.py new file mode 100644 index 00000000..006597da --- /dev/null +++ b/core/graph/network_security/filtering.py @@ -0,0 +1,86 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Microsoft Entra Global Secure Access filtering policies data pipeline.""" + +import logging +import pandas as pd +from core.graph.client import GraphClient + +logger = logging.getLogger(__name__) + +def run_filtering_pipeline( + client_id: str, + client_secret: str, + tenant_id: str, + csv_path: str = None, + on_page_callback=None, + is_cancelled_callback=None +): + """Fetch Entra Global Secure Access Filtering Policies (Beta).""" + client = GraphClient( + tenant_id=tenant_id, + client_ids=client_id, + client_secrets=client_secret, + concurrency=1, + retries=5, + backoff=2 + ) + client.authenticate() + + url = "https://graph.microsoft.com/beta/networkAccess/filteringPolicies" + token_slot = client.get_active_token() + session = client.get_session() + + headers = { + "Authorization": f"Bearer {token_slot['token']}", + "Accept": "application/json" + } + + rows = [] + try: + logger.info("Querying Entra Global Secure Access filtering policies...") + while url: + if is_cancelled_callback and is_cancelled_callback(): break + resp = session.get(url, headers=headers) + if resp.status_code == 200: + data = resp.json() + value_list = data.get("value", []) + + for p in value_list: + p["name"] = p.get("name") or p.get("displayName") or "N/A" + p["description"] = p.get("description") or "N/A" + p["version"] = p.get("version") or "N/A" + p["action"] = p.get("action") or "N/A" + + rules = p.get("policyRules", []) + p["rules_count"] = len(rules) if isinstance(rules, list) else 0 + rows.append(p) + + if on_page_callback: + on_page_callback(value_list) + url = data.get("@odata.nextLink") + elif resp.status_code in [401, 403]: + logger.error("Filtering policies endpoint permission error: %d %s", resp.status_code, resp.text) + raise PermissionError("NetworkAccess.Read.All permission required for beta network Access GSA.") + else: + logger.error("Filtering policies endpoint failed: %d %s", resp.status_code, resp.text) + raise ConnectionError(f"API request failed with status {resp.status_code}") + + if csv_path: + df = pd.DataFrame(rows) if rows else pd.DataFrame() + df.to_csv(csv_path, index=False, encoding='utf-8') + finally: + client.release_token(token_slot) + client.close() diff --git a/core/graph/network_security/firewall.py b/core/graph/network_security/firewall.py new file mode 100644 index 00000000..4c6869ae --- /dev/null +++ b/core/graph/network_security/firewall.py @@ -0,0 +1,104 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Intune Firewall and Proxy configurations data pipeline.""" + +import logging +import pandas as pd +from core.graph.client import GraphClient + +logger = logging.getLogger(__name__) + +def run_firewall_pipeline( + client_id: str, + client_secret: str, + tenant_id: str, + csv_path: str = None, + on_page_callback=None, + is_cancelled_callback=None +): + """Fetch Intune Firewall and Proxy configurations.""" + client = GraphClient( + tenant_id=tenant_id, + client_ids=client_id, + client_secrets=client_secret, + concurrency=1, + retries=5, + backoff=2 + ) + client.authenticate() + + url = "https://graph.microsoft.com/v1.0/deviceManagement/deviceConfigurations" + token_slot = client.get_active_token() + session = client.get_session() + + headers = { + "Authorization": f"Bearer {token_slot['token']}", + "Accept": "application/json" + } + + rows = [] + try: + logger.info("Querying Intune Device Configurations for Firewall/Proxy...") + while url: + if is_cancelled_callback and is_cancelled_callback(): break + resp = session.get(url, headers=headers) + if resp.status_code == 200: + data = resp.json() + value_list = data.get("value", []) + + for p in value_list: + p["name"] = p.get("displayName") or p.get("name") or "N/A" + p["description"] = p.get("description") or "N/A" + + odata_type = p.get("@odata.type", "") + policy_type = odata_type.replace("#microsoft.graph.", "") + p["policy_type"] = policy_type + + is_firewall = "Windows10EndpointProtection" in policy_type or "Firewall" in p["name"] or "Firewall" in p["description"] or "firewall" in policy_type.lower() + firewall_status = "Configured" if is_firewall else "Not Configured" + + if "Windows10EndpointProtectionConfiguration" in odata_type: + fw_enable = p.get("firewallEnableDomainProfile") or p.get("firewallEnablePrivateProfile") or p.get("firewallEnablePublicProfile") + if fw_enable is not None: + firewall_status = "Enabled" if fw_enable else "Disabled" + p["firewall_status"] = firewall_status + + is_proxy = "GeneralConfiguration" in policy_type or "Proxy" in p["name"] or "Proxy" in p["description"] or "proxy" in policy_type.lower() + proxy_status = "Not Configured" + if is_proxy: + proxy_status = "Configured" + p_srv = p.get("proxyServer") or p.get("proxyAutomaticConfigurationUrl") + if p_srv: + proxy_status = f"Configured ({p_srv})" + p["proxy_status"] = proxy_status + + rows.append(p) + + if on_page_callback: + on_page_callback(value_list) + url = data.get("@odata.nextLink") + elif resp.status_code in [401, 403]: + logger.error("DeviceConfigurations endpoint permission error: %d %s", resp.status_code, resp.text) + raise PermissionError("DeviceManagementConfiguration.Read.All permission required for Intune Device Configurations.") + else: + logger.error("DeviceConfigurations endpoint failed: %d %s", resp.status_code, resp.text) + raise ConnectionError(f"API request failed with status {resp.status_code}") + + if csv_path: + df = pd.DataFrame(rows) if rows else pd.DataFrame() + df.to_csv(csv_path, index=False, encoding='utf-8') + finally: + client.release_token(token_slot) + client.close() diff --git a/core/graph/reports.py b/core/graph/reports.py new file mode 100644 index 00000000..b25e7b9b --- /dev/null +++ b/core/graph/reports.py @@ -0,0 +1,574 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""ReportsService encapsulating Microsoft Graph usage report generation and downloads.""" + +import os +import csv +import time +import logging +import concurrent.futures +from typing import List, Tuple +import requests +from core.graph.client import GraphClient + +import re + +def _sanitize_string(s: str) -> str: + if not s: + return "" + # Mask JWT and access tokens in query parameters or headers + s = re.sub(r'token=[^&"\')\s]+', 'token=[MASKED]', s) + s = re.sub(r'Bearer\s+[^&"\')\s]+', 'Bearer [MASKED]', s, flags=re.IGNORECASE) + return s + +logger = logging.getLogger(__name__) + + +class ReportsService: + """Service to fetch streaming M365 telemetry reports via Microsoft Graph API.""" + + def __init__(self, client: GraphClient) -> None: + self.client = client + + def download_report(self, api_url: str, output_filename: str, output_dir: str) -> None: + """Downloads a single CSV report via a streaming connection utilizing GraphClient session.""" + token_slot = self.client.get_active_token() + session = self.client.get_session() + + headers = { + "Authorization": f"Bearer {token_slot['token']}" + } + os.makedirs(output_dir, exist_ok=True) + output_path = os.path.join(output_dir, output_filename) + + try: + logger.info("Calling Graph report endpoint for %s...", output_filename) + resp = session.get(api_url, headers=headers, allow_redirects=False, stream=True, timeout=60.0) + + # Graph APIs return status code 302 redirect to a pre-authenticated S3/Azure Blob URL + if resp.status_code == 302: + resp.close() + location_url = resp.headers.get("Location") + if not location_url: + raise ConnectionError(f"302 redirect returned for {output_filename} but Location header is missing.") + + logger.info("Pre-authenticated storage redirect retrieved. Waiting 2 seconds before download...") + time.sleep(2) + + max_retries = 5 + retry_interval = 30 + for attempt in range(1, max_retries + 1): + try: + logger.info("[Attempt %d/%d] Downloading report stream to %s...", attempt, max_retries, output_filename) + with requests.get(location_url, stream=True, timeout=120.0) as csv_response: + csv_response.raise_for_status() + with open(output_path, "wb") as f: + for chunk in csv_response.iter_content(chunk_size=8192): + if chunk: + f.write(chunk) + break + except requests.exceptions.RequestException as error: + if attempt < max_retries: + logger.warning("[Attempt %d/%d] Failed stream download. Retrying in %ds... (Error: %s)", attempt, max_retries, retry_interval, _sanitize_string(str(error))) + time.sleep(retry_interval) + else: + logger.error("Failed downloading %s after %d attempts. Error: %s", output_filename, max_retries, _sanitize_string(str(error))) + raise ConnectionError(f"Failed downloading report after {max_retries} attempts.") + + logger.info("Success! Saved report to: %s", output_path) + + elif resp.status_code == 200: + logger.info("Unexpected 200 OK status returned. Saving direct streaming stream...") + with open(output_path, "wb") as f: + for chunk in resp.iter_content(chunk_size=8192): + if chunk: + f.write(chunk) + resp.close() + logger.info("Success! Saved report to: %s", output_path) + else: + resp.close() + logger.error("Graph report request failed with status code %d: %s", resp.status_code, _sanitize_string(resp.text)) + raise ConnectionError(f"Microsoft Graph API request failed with status code {resp.status_code}") + finally: + self.client.release_token(token_slot) + + def download_reports_batch(self, reports: List[Tuple[str, str]], output_dir: str, progress_callback=None) -> None: + """Downloads a list of reports concurrently using thread executors.""" + logger.info("Starting parallel batch download for %d reports...", len(reports)) + + completed_count = 0 + total_reports = len(reports) + + with concurrent.futures.ThreadPoolExecutor(max_workers=len(reports)) as executor: + futures = [ + executor.submit(self.download_report, url, filename, output_dir) + for url, filename in reports + ] + # Gather all futures, raising exceptions if any thread failed + for future in concurrent.futures.as_completed(futures): + future.result() + completed_count += 1 + if progress_callback: + try: + progress_callback(completed_count / total_reports) + except Exception as e: + logger.warning("Failed to invoke progress callback: %s", e) + + def download_o365_active_user_detail(self, output_dir: str) -> None: + """Downloads the Office 365 active user details CSV report (180 days).""" + url = "https://graph.microsoft.com/v1.0/reports/getOffice365ActiveUserDetail(period='D180')" + self.download_report(url, "Office365ActiveUserDetail(180d).csv", output_dir) + + def download_o365_active_user_counts(self, output_dir: str) -> None: + """Downloads the Office 365 30-day active user counts CSV report.""" + url = "https://graph.microsoft.com/v1.0/reports/getOffice365ActiveUserCounts(period='D30')" + self.download_report(url, "Office365ActiveUserCounts(30d).csv", output_dir) + + def download_m365_app_details(self, output_dir: str) -> None: + """Downloads both M365 App user details and counts CSV reports concurrently.""" + reports = [ + ("https://graph.microsoft.com/v1.0/reports/getM365AppUserDetail(period='D180')", "M365AppUserDetail(180d).csv"), + ("https://graph.microsoft.com/v1.0/reports/getM365AppUserCounts(period='D180')", "getM365AppUserCounts(180d).csv") + ] + self.download_reports_batch(reports, output_dir) + + def download_sharepoint_onedrive_details(self, output_dir: str) -> None: + """Downloads SharePoint site usage, OneDrive account usage, OneDrive activity, and M365 App details CSV reports concurrently.""" + reports = [ + ("https://graph.microsoft.com/v1.0/reports/getSharePointSiteUsageDetail(period='D180')", "SharePointSiteUsageDetail(180d).csv"), + ("https://graph.microsoft.com/v1.0/reports/getOneDriveUsageAccountDetail(period='D180')", "OneDriveUsageAccountDetail(180d).csv"), + ("https://graph.microsoft.com/v1.0/reports/getOneDriveActivityUserDetail(period='D180')", "OneDriveActivityUserDetail(180d).csv"), + ("https://graph.microsoft.com/v1.0/reports/getM365AppUserDetail(period='D180')", "M365AppUserDetail_sp_od(180d).csv") + ] + self.download_reports_batch(reports, output_dir) + + def download_sharepoint_details(self, output_dir: str) -> None: + """Downloads only SharePoint site usage detail report.""" + self.download_report("https://graph.microsoft.com/v1.0/reports/getSharePointSiteUsageDetail(period='D180')", "SharePointSiteUsageDetail(180d).csv", output_dir) + + def download_onedrive_details(self, output_dir: str) -> None: + """Downloads OneDrive usage account, activity, and app user detail reports concurrently.""" + reports = [ + ("https://graph.microsoft.com/v1.0/reports/getOneDriveUsageAccountDetail(period='D180')", "OneDriveUsageAccountDetail(180d).csv"), + ("https://graph.microsoft.com/v1.0/reports/getOneDriveActivityUserDetail(period='D180')", "OneDriveActivityUserDetail(180d).csv"), + ("https://graph.microsoft.com/v1.0/reports/getM365AppUserDetail(period='D180')", "M365AppUserDetail_sp_od(180d).csv") + ] + self.download_reports_batch(reports, output_dir) + + def download_mailbox_usage_detail(self, output_dir: str) -> None: + """Downloads Exchange mailbox usage detail CSV report (180 days).""" + self.download_report("https://graph.microsoft.com/v1.0/reports/getMailboxUsageDetail(period='D180')", "MailboxUsageDetail(180d).csv", output_dir) + + def download_email_app_usage_detail(self, output_dir: str) -> None: + """Downloads Exchange email app usage detail CSV report (180 days).""" + self.download_report("https://graph.microsoft.com/v1.0/reports/getEmailAppUsageUserDetail(period='D180')", "EmailAppUsageUserDetail(180d).csv", output_dir) + + def search_cloud_pst_files(self) -> dict: + """Queries Microsoft Graph Search API to locate cloud-stored PST archive files across all active regions in a paginated fashion, up to a total of 2000 files.""" + url = "https://graph.microsoft.com/v1.0/search/query" + token_slot = self.client.get_active_token() + session = self.client.get_session() + + headers = { + "Authorization": f"Bearer {token_slot['token']}", + "Accept": "application/json", + "Content-Type": "application/json" + } + + total_hits = [] + regions = ["NAM", "EUR", "APC"] + page_size = 500 + max_total_limit = 2000 + + try: + for region in regions: + # Calculate how many hits we have already fetched in total + current_fetched = 0 + for r_resp in total_hits: + for hc in r_resp.get("hitsContainers", []): + current_fetched += len(hc.get("hits", [])) + + remaining_limit = max_total_limit - current_fetched + if remaining_limit <= 0: + logger.info(f"Reached total limit of {max_total_limit} files. Stopping search across regions.") + break + + offset = 0 + has_more = True + region_response = None + region_hits_container = None + + while has_more: + # Request only up to the remaining allowed limit + current_page_size = min(page_size, remaining_limit) + + payload = { + "requests": [ + { + "entityTypes": ["driveItem"], + "query": {"queryString": "fileextension:pst"}, + "from": offset, + "size": current_page_size, + "region": region + } + ] + } + logger.info(f"Executing Graph Search query for cloud PST archives in region: {region} (offset: {offset}, limit: {current_page_size})...") + resp = session.post(url, json=payload, headers=headers) + if resp.status_code == 200: + data = resp.json() + page_response_list = data.get("value", []) + + if not page_response_list: + has_more = False + continue + + page_response = page_response_list[0] + page_containers = page_response.get("hitsContainers", []) + + if not page_containers: + has_more = False + continue + + container = page_containers[0] + hits = container.get("hits", []) + total_count = container.get("total", 0) + more_results = container.get("moreResultsAvailable", False) + + if region_response is None: + region_response = { + "@odata.type": page_response.get("@odata.type"), + "searchTerms": page_response.get("searchTerms", []), + "hitsContainers": [ + { + "@odata.type": container.get("@odata.type"), + "total": total_count, + "moreResultsAvailable": False, + "hits": [] + } + ] + } + region_hits_container = region_response["hitsContainers"][0] + + region_hits_container["hits"].extend(hits) + + region_fetched = len(region_hits_container["hits"]) + if region_fetched >= remaining_limit: + logger.info(f"Reached remaining limit of {remaining_limit} files in region {region}. Stopping paginated fetch.") + region_hits_container["hits"] = region_hits_container["hits"][:remaining_limit] + has_more = False + elif more_results and len(hits) > 0: + offset += page_size + else: + has_more = False + + elif resp.status_code == 400 and "Only valid regions are" in resp.text: + logger.info(f"Region {region} is not active for this tenant. Skipping.") + has_more = False + else: + raise ConnectionError(f"Graph Search failed for region {region} (HTTP {resp.status_code}): {resp.text}") + + if region_response is not None: + total_hits.append(region_response) + + return {"value": total_hits} + finally: + self.client.release_token(token_slot) + + def download_email_app_usage_apps_user_counts(self, output_dir: str) -> None: + """Downloads Exchange email app usage apps user counts CSV report (180 days).""" + self.download_report("https://graph.microsoft.com/v1.0/reports/getEmailAppUsageAppsUserCounts(period='D180')", "EmailAppUsageAppsUserCounts(180d).csv", output_dir) + + def fetch_app_signin_summary(self, csv_path: str, max_rows: int = 5000, on_page_callback=None, is_cancelled_callback=None) -> None: + """Fetches Azure AD application sign-in summary for the last 7 days and dumps to a CSV file.""" + token_slot = self.client.get_active_token() + session = self.client.get_session() + + headers = { + "Authorization": f"Bearer {token_slot['token']}", + "Accept": "application/json" + } + + url = "https://graph.microsoft.com/beta/reports/getAzureADApplicationSignInSummary(period='D7')" + + try: + logger.info("Fetching Azure AD Application Sign-in Summary...") + retries_left = 4 + resp = None + value_list = [] + + while retries_left > 0: + if is_cancelled_callback and is_cancelled_callback(): + return + + try: + resp = session.get(url, headers=headers, timeout=120.0) + logger.info("App Sign-ins HTTP status: %d", resp.status_code) + if resp.status_code == 200: + data = resp.json() + value_list = data.get("value", []) + if value_list: + break + else: + logger.warning("Received empty value collection for App Sign-ins summary, retrying...") + else: + break + except Exception as get_err: + logger.warning("Query attempt failed: %s", get_err) + + retries_left -= 1 + if retries_left > 0: + time.sleep(2) + + if resp is not None and resp.status_code == 200 and value_list: + logger.info("App Sign-ins value collection length: %d", len(value_list)) + logger.info("App Sign-ins first 200 chars: %s", str(resp.json())[:200]) + + with open(csv_path, 'a', encoding='utf-8', newline='') as f: + writer = csv.writer(f) + rows_written = 0 + for item in value_list: + if rows_written >= max_rows: + break + app_name = item.get("appDisplayName") or "" + success_count = item.get("successfulSignInCount") or 0 + writer.writerow([app_name, success_count]) + rows_written += 1 + + if on_page_callback: + on_page_callback(value_list) + + logger.info("Successfully fetched %d app sign-in summary records", rows_written) + else: + if resp is not None and resp.status_code in [401, 403]: + logger.error("App Sign-ins Summary endpoint access denied: %d %s", resp.status_code, resp.text) + raise PermissionError("Reports.Read.All permission required.") + else: + status_str = f"status {resp.status_code}" if resp is not None else "connection/timeout error" + logger.warning("App Sign-ins Summary query failed (%s) or remained empty.", status_str) + finally: + self.client.release_token(token_slot) + + def fetch_auth_methods_summary(self, csv_path: str, period: str = "D7", max_rows: int = 5000, on_page_callback=None, is_cancelled_callback=None) -> None: + """Fetches User Sign-in by Authentication Method Summary for the specified period and dumps to a CSV file.""" + token_slot = self.client.get_active_token() + session = self.client.get_session() + + headers = { + "Authorization": f"Bearer {token_slot['token']}", + "Accept": "application/json" + } + + url = f"https://graph.microsoft.com/beta/reports/authenticationMethods/userSignInsByAuthMethodSummary(period='{period}')" + + try: + logger.info("Fetching User Sign-ins by Authentication Method Summary...") + if is_cancelled_callback and is_cancelled_callback(): + return + + resp = session.get(url, headers=headers, timeout=120.0) + if resp.status_code == 200: + data = resp.json() + value_list = data.get("value", []) + + with open(csv_path, 'a', encoding='utf-8', newline='') as f: + writer = csv.writer(f) + rows_written = 0 + for item in value_list: + if rows_written >= max_rows: + break + method = item.get("authenticationMethod") or "" + success_count = item.get("successActivityCount") or 0 + writer.writerow([method, success_count]) + rows_written += 1 + + if on_page_callback: + on_page_callback(value_list) + + logger.info("Successfully fetched %d authentication method summary records", rows_written) + else: + if resp.status_code in [401, 403]: + logger.error("Auth Methods Summary endpoint access denied: %d %s", resp.status_code, resp.text) + raise PermissionError("AuditLog.Read.All permission required.") + else: + logger.warning("Auth Methods Summary query failed (HTTP %d).", resp.status_code) + finally: + self.client.release_token(token_slot) + + def fetch_user_signins(self, csv_path: str, max_rows: int = 20000, on_page_callback=None, is_cancelled_callback=None) -> None: + """Fetches Microsoft Entra user sign-in logs from the v1.0 auditLogs/signIns endpoint, + filters for successful sign-ins, flattens, and appends to CSV. + """ + token_slot = self.client.get_active_token() + session = self.client.get_session() + + headers = { + "Authorization": f"Bearer {token_slot['token']}", + "Accept": "application/json" + } + + # Start URL with select query parameter to limit payload size + next_url = "https://graph.microsoft.com/v1.0/auditLogs/signIns?$select=appDisplayName,status,deviceDetail,isInteractive" + rows_written = 0 + page_number = 1 + import csv + + try: + logger.info("Starting User Sign-ins fetch...") + with open(csv_path, 'a', encoding='utf-8', newline='') as f: + writer = csv.writer(f) + + while next_url and rows_written < max_rows: + if is_cancelled_callback and is_cancelled_callback(): + logger.info("User Sign-ins fetch cancelled in-flight. Aborting pagination loop.") + break + + logger.info("Querying MSFT Graph user sign-ins (Page: %d, Successful rows so far: %d)...", + page_number, rows_written) + try: + # Timeout must be 60.0 seconds, no retries on failure, just exit loop and display what was obtained + resp = session.get(next_url, headers=headers, timeout=60.0) + except Exception as get_err: + logger.warning("Query attempt failed with exception: %s. Displaying data obtained till now.", get_err) + break + + if resp is None or resp.status_code != 200: + if resp is not None and resp.status_code in [401, 403]: + logger.error("User Sign-ins endpoint permission error: %d %s", resp.status_code, resp.text) + raise PermissionError("AuditLog.Read.All permission required.") + else: + status_str = f"status {resp.status_code}" if resp is not None else "connection/timeout error" + logger.warning("User Sign-ins query failed (%s). Displaying data obtained till now.", status_str) + break + + page_number += 1 + data = resp.json() + value_list = data.get("value", []) + + page_filtered_successful = [] + for log in value_list: + status_obj = log.get("status") or {} + # Successful sign-in records are those with status errorCode = 0 + error_code = status_obj.get("errorCode") + if error_code == 0 or error_code == "0": + app_name = log.get("appDisplayName") or "" + device = log.get("deviceDetail") or {} + os_name = device.get("operatingSystem") or "" + browser_name = device.get("browser") or "" + is_interactive = str(log.get("isInteractive", "")) + + writer.writerow([app_name, os_name, browser_name, is_interactive]) + rows_written += 1 + + page_filtered_successful.append(log) + if rows_written >= max_rows: + break + + if on_page_callback: + try: + # Invoke callback with the filtered successful records of the page + on_page_callback(page_filtered_successful) + except Exception as cb_err: + logger.warning("Error in User Sign-ins page callback: %s", cb_err) + + if rows_written >= max_rows: + logger.info("Reached maximum rows limit of %d", max_rows) + break + + next_url = data.get("@odata.nextLink") + + logger.info("Successfully fetched and appended %d successful user sign-in records.", rows_written) + finally: + self.client.release_token(token_slot) + + def fetch_app_registrations(self, csv_path: str, max_rows: int = 5000, on_page_callback=None, is_cancelled_callback=None) -> None: + """Fetches Microsoft Entra app registrations (applications) and dumps to CSV.""" + token_slot = self.client.get_active_token() + session = self.client.get_session() + + headers = { + "Authorization": f"Bearer {token_slot['token']}", + "Accept": "application/json" + } + + next_url = "https://graph.microsoft.com/v1.0/applications?$select=displayName,appId,createdDateTime,signInAudience,passwordCredentials,keyCredentials" + rows_written = 0 + page_number = 1 + import csv + + try: + logger.info("Starting App Registrations fetch...") + with open(csv_path, 'a', encoding='utf-8', newline='') as f: + writer = csv.writer(f) + + while next_url and rows_written < max_rows: + if is_cancelled_callback and is_cancelled_callback(): + logger.info("App Registrations fetch cancelled in-flight.") + break + + logger.info("Querying MSFT Graph applications (Page: %d, rows so far: %d)...", + page_number, rows_written) + try: + resp = session.get(next_url, headers=headers, timeout=60.0) + except Exception as get_err: + logger.warning("Query attempt failed with exception: %s.", get_err) + break + + if resp is None or resp.status_code != 200: + if resp is not None and resp.status_code in [401, 403]: + logger.error("Applications endpoint permission error: %d %s", resp.status_code, resp.text) + raise PermissionError("Application.Read.All permission required.") + else: + status_str = f"status {resp.status_code}" if resp is not None else "connection/timeout error" + logger.warning("Applications query failed (%s).", status_str) + break + + page_number += 1 + data = resp.json() + value_list = data.get("value", []) + + for app in value_list: + display_name = app.get("displayName") or "" + app_id = app.get("appId") or "" + created_dt = app.get("createdDateTime") or "" + audience = app.get("signInAudience") or "" + + secrets_cnt = len(app.get("passwordCredentials", [])) + certs_cnt = len(app.get("keyCredentials", [])) + credentials_str = f"{secrets_cnt} Secrets, {certs_cnt} Certs" + + writer.writerow([display_name, app_id, created_dt, audience, credentials_str]) + rows_written += 1 + + if rows_written >= max_rows: + break + + if on_page_callback: + try: + on_page_callback(value_list) + except Exception as cb_err: + logger.warning("Error in App Registrations page callback: %s", cb_err) + + if rows_written >= max_rows: + break + + next_url = data.get("@odata.nextLink") + + logger.info("Successfully fetched and appended %d app registrations.", rows_written) + finally: + self.client.release_token(token_slot) + + + diff --git a/core/graph/security/__init__.py b/core/graph/security/__init__.py new file mode 100644 index 00000000..d1caca2f --- /dev/null +++ b/core/graph/security/__init__.py @@ -0,0 +1,153 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Facade exposing security & governance pipelines and compatibility SecurityService.""" + +import logging +import csv + +from core.graph.client import GraphClient +from core.graph.security.sensitivity_labels import run_sensitivity_labels_pipeline +from core.graph.security.retention_policies import run_retention_policies_pipeline +from core.graph.security.dlp_policies import run_dlp_policies_pipeline +from core.graph.security.sensitive_info_types import run_sensitive_info_types_pipeline +from core.graph.security.authentication import run_authentication_pipeline +from core.graph.security.service_principals_sso import run_service_principals_sso_pipeline + +logger = logging.getLogger(__name__) + +class SecurityService: + """Service to interact with M365 Security and Information Protection configurations (backward compatibility).""" + + def __init__(self, client: GraphClient) -> None: + self.client = client + + def fetch_sensitivity_labels(self, csv_path: str = None, on_page_callback=None, is_cancelled_callback=None) -> list: + """Fetches sensitivity labels.""" + labels = [] + def on_page(items): + labels.extend(items) + if on_page_callback: + on_page_callback(items) + + run_sensitivity_labels_pipeline( + client_id=self.client.client_ids[0] if isinstance(self.client.client_ids, list) else self.client.client_ids, + client_secret=self.client.client_secrets[0] if isinstance(self.client.client_secrets, list) else self.client.client_secrets, + tenant_id=self.client.tenant_id, + csv_path=csv_path, + on_page_callback=on_page, + is_cancelled_callback=is_cancelled_callback + ) + return labels + + def fetch_conditional_access_policies(self, csv_path: str = None, on_page_callback=None, is_cancelled_callback=None) -> None: + """Fetches conditional access policies.""" + run_authentication_pipeline( + client_id=self.client.client_ids[0] if isinstance(self.client.client_ids, list) else self.client.client_ids, + client_secret=self.client.client_secrets[0] if isinstance(self.client.client_secrets, list) else self.client.client_secrets, + tenant_id=self.client.tenant_id, + csv_path=csv_path, + on_page_callback=on_page_callback, + is_cancelled_callback=is_cancelled_callback + ) + + def fetch_sso_service_principals(self, csv_path: str = None, on_page_callback=None, is_cancelled_callback=None) -> None: + """Fetches service principals SSO configurations.""" + run_service_principals_sso_pipeline( + client_id=self.client.client_ids[0] if isinstance(self.client.client_ids, list) else self.client.client_ids, + client_secret=self.client.client_secrets[0] if isinstance(self.client.client_secrets, list) else self.client.client_secrets, + tenant_id=self.client.tenant_id, + csv_path=csv_path, + on_page_callback=on_page_callback, + is_cancelled_callback=is_cancelled_callback + ) + + def search_cloud_pst_files(self) -> dict: + """Queries Microsoft Graph Search API to locate cloud-stored PST archive files.""" + url = "https://graph.microsoft.com/v1.0/search/query" + token_slot = self.client.get_active_token() + session = self.client.get_session() + + headers = { + "Authorization": f"Bearer {token_slot['token']}", + "Accept": "application/json", + "Content-Type": "application/json" + } + payload = { + "requests": [ + { + "entityTypes": ["driveItem"], + "query": {"queryString": "fileextension:pst"}, + "from": 0, + "size": 50 + } + ] + } + try: + logger.info("Executing Graph Search query for cloud PST archives...") + resp = session.post(url, json=payload, headers=headers) + if resp.status_code == 200: + return resp.json() + else: + return {} + except Exception as e: + logger.warning("Exception during Graph Search query: %s", e) + return {} + finally: + self.client.release_token(token_slot) + + def fetch_signin_activities(self, event_type: str, csv_path: str, max_rows: int = 10000, on_page_callback=None, is_cancelled_callback=None) -> None: + """Fetches successful sign-in logs (legacy utility).""" + base_url = "https://graph.microsoft.com/beta/auditLogs/signIns" + filter_str = f"status/errorCode eq 0 and signInEventTypes/any(t: t eq '{event_type}')" + select_str = "appDisplayName,deviceDetail" + + token_slot = self.client.get_active_token() + session = self.client.get_session() + + headers = { + "Authorization": f"Bearer {token_slot['token']}", + "Accept": "application/json" + } + next_url = f"{base_url}?$filter={filter_str}&$select={select_str}&$top=100" + rows_written = 0 + page_number = 1 + + try: + logger.info("Fetching successful sign-in activities for %s...", event_type) + with open(csv_path, 'a', encoding='utf-8', newline='') as f: + writer = csv.writer(f) + while next_url and rows_written < max_rows: + if is_cancelled_callback and is_cancelled_callback(): + break + resp = session.get(next_url, headers=headers) + if resp and resp.status_code == 200: + data = resp.json() + value_list = data.get("value", []) + for log in value_list: + if rows_written >= max_rows: + break + app_name = log.get("appDisplayName") or "" + device = log.get("deviceDetail") or {} + os_name = device.get("operatingSystem") or "" + browser_name = device.get("browser") or "" + writer.writerow([app_name, os_name, browser_name, event_type]) + rows_written += 1 + if on_page_callback: + on_page_callback(value_list) + next_url = data.get("@odata.nextLink") + else: + break + finally: + self.client.release_token(token_slot) diff --git a/core/graph/security/authentication.py b/core/graph/security/authentication.py new file mode 100644 index 00000000..4c1e07dd --- /dev/null +++ b/core/graph/security/authentication.py @@ -0,0 +1,99 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Entra ID Conditional Access policies scanner data pipeline.""" + +import logging +import csv +from core.graph.client import GraphClient + +logger = logging.getLogger(__name__) + +def run_authentication_pipeline( + client_id: str, + client_secret: str, + tenant_id: str, + csv_path: str = None, + on_page_callback=None, + is_cancelled_callback=None +): + """Fetch Conditional Access policies and stream to CSV.""" + client = GraphClient( + tenant_id=tenant_id, + client_ids=client_id, + client_secrets=client_secret, + concurrency=1, + retries=2, + backoff=1 + ) + client.authenticate() + + url = "https://graph.microsoft.com/v1.0/identity/conditionalAccess/policies" + token_slot = client.get_active_token() + session = client.get_session() + + headers = { + "Authorization": f"Bearer {token_slot['token']}", + "Accept": "application/json" + } + + f = None + writer = None + try: + logger.info("Querying Entra ID Conditional Access policies...") + if csv_path: + f = open(csv_path, 'w', encoding='utf-8', newline='') + writer = csv.writer(f) + writer.writerow(["name", "state", "target_users", "target_apps", "controls"]) + + while url: + if is_cancelled_callback and is_cancelled_callback(): break + resp = session.get(url, headers=headers) + if resp.status_code == 200: + data = resp.json() + value_list = data.get("value", []) + if writer: + for p in value_list: + name = p.get("displayName", "N/A") + state = p.get("state", "N/A") + + conds = p.get("conditions") or {} + users_cond = conds.get("users") or {} + apps_cond = conds.get("applications") or {} + + inc_users = users_cond.get("includeUsers") or [] + inc_groups = users_cond.get("includeGroups") or [] + user_target = "All Users" if "All" in inc_users else f"Specific ({len(inc_users)} users, {len(inc_groups)} groups)" + + inc_apps = apps_cond.get("includeApplications") or [] + app_target = "All Apps" if "All" in inc_apps else f"Specific ({len(inc_apps)} apps)" + + grant_controls = p.get("grantControls") or {} + controls = grant_controls.get("builtInControls") or [] + ctrl_str = ", ".join(controls) if controls else "Block/None" + + writer.writerow([name, state, user_target, app_target, ctrl_str]) + if on_page_callback: + on_page_callback(value_list) + url = data.get("@odata.nextLink") + elif resp.status_code in [401, 403]: + logger.error("Conditional Access endpoint permission error: %d %s", resp.status_code, resp.text) + raise PermissionError("Policy.Read.All or Policy.Read permission required.") + else: + logger.error("Conditional Access endpoint failed with status %d: %s", resp.status_code, resp.text) + raise ConnectionError(f"Microsoft Graph API request failed with status {resp.status_code}") + finally: + if f: f.close() + client.release_token(token_slot) + client.close() diff --git a/core/graph/security/dlp_policies.py b/core/graph/security/dlp_policies.py new file mode 100644 index 00000000..1771e00e --- /dev/null +++ b/core/graph/security/dlp_policies.py @@ -0,0 +1,55 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Data Loss Prevention (DLP) Policies PowerShell scanner data pipeline.""" + +import logging +from core.graph.client import GraphClient +from core.graph.directory.organization import OrganizationService +from core.powershell.client import PowerShellClient +from core.powershell.dlp import DLPService + +logger = logging.getLogger(__name__) + +def run_dlp_policies_pipeline( + client_id: str, + client_secret: str, + tenant_id: str +) -> list: + """Fetch DLP policies via PowerShell.""" + tenant_domain = tenant_id + client = GraphClient( + tenant_id=tenant_id, + client_ids=client_id, + client_secrets=client_secret, + concurrency=1 + ) + try: + client.authenticate() + org_svc = OrganizationService(client) + tenant_domain = org_svc.get_tenant_primary_domain() + logger.info(f"Retrieved primary tenant domain for DLP fetch: {tenant_domain}") + except Exception as e: + logger.warning(f"Could not retrieve tenant domain for DLP fetch. Falling back to Tenant ID Guid: {e}") + finally: + client.close() + + ps_client = PowerShellClient( + tenant_id=tenant_domain, + client_id=client_id, + client_secret=client_secret, + cert_tenant_id=tenant_id + ) + dlp_svc = DLPService(ps_client) + return dlp_svc.fetch_dlp_policies() diff --git a/core/graph/security/retention_policies.py b/core/graph/security/retention_policies.py new file mode 100644 index 00000000..59f052f5 --- /dev/null +++ b/core/graph/security/retention_policies.py @@ -0,0 +1,55 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Retention Policies PowerShell scanner data pipeline.""" + +import logging +from core.graph.client import GraphClient +from core.graph.directory.organization import OrganizationService +from core.powershell.client import PowerShellClient +from core.powershell.retention import RetentionService + +logger = logging.getLogger(__name__) + +def run_retention_policies_pipeline( + client_id: str, + client_secret: str, + tenant_id: str +) -> list: + """Fetch retention policies via PowerShell.""" + tenant_domain = tenant_id + client = GraphClient( + tenant_id=tenant_id, + client_ids=client_id, + client_secrets=client_secret, + concurrency=1 + ) + try: + client.authenticate() + org_svc = OrganizationService(client) + tenant_domain = org_svc.get_tenant_primary_domain() + logger.info(f"Retrieved primary tenant domain: {tenant_domain}") + except Exception as e: + logger.warning(f"Could not retrieve tenant domain. Falling back to Tenant ID Guid: {e}") + finally: + client.close() + + ps_client = PowerShellClient( + tenant_id=tenant_domain, + client_id=client_id, + client_secret=client_secret, + cert_tenant_id=tenant_id + ) + ret_service = RetentionService(ps_client) + return ret_service.fetch_retention_policies() diff --git a/core/graph/security/sensitive_info_types.py b/core/graph/security/sensitive_info_types.py new file mode 100644 index 00000000..a7ef33c8 --- /dev/null +++ b/core/graph/security/sensitive_info_types.py @@ -0,0 +1,55 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Sensitive Information Types (SIT) PowerShell scanner data pipeline.""" + +import logging +from core.graph.client import GraphClient +from core.graph.directory.organization import OrganizationService +from core.powershell.client import PowerShellClient +from core.powershell.dlp import DLPService + +logger = logging.getLogger(__name__) + +def run_sensitive_info_types_pipeline( + client_id: str, + client_secret: str, + tenant_id: str +) -> list: + """Fetch Sensitive Information Types via PowerShell.""" + tenant_domain = tenant_id + client = GraphClient( + tenant_id=tenant_id, + client_ids=client_id, + client_secrets=client_secret, + concurrency=1 + ) + try: + client.authenticate() + org_svc = OrganizationService(client) + tenant_domain = org_svc.get_tenant_primary_domain() + logger.info(f"Retrieved primary tenant domain for SIT fetch: {tenant_domain}") + except Exception as e: + logger.warning(f"Could not retrieve tenant domain for SIT fetch. Falling back to Tenant ID Guid: {e}") + finally: + client.close() + + ps_client = PowerShellClient( + tenant_id=tenant_domain, + client_id=client_id, + client_secret=client_secret, + cert_tenant_id=tenant_id + ) + dlp_svc = DLPService(ps_client) + return dlp_svc.fetch_sensitive_info_types() diff --git a/core/graph/security/sensitivity_labels.py b/core/graph/security/sensitivity_labels.py new file mode 100644 index 00000000..34115bcf --- /dev/null +++ b/core/graph/security/sensitivity_labels.py @@ -0,0 +1,101 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Sensitivity Labels query pipeline.""" + +import logging +import csv +from core.graph.client import GraphClient + +logger = logging.getLogger(__name__) + +def run_sensitivity_labels_pipeline( + client_id: str, + client_secret: str, + tenant_id: str, + csv_path: str = None, + on_page_callback=None, + is_cancelled_callback=None +): + """Fetch sensitivity labels and stream to CSV.""" + client = GraphClient( + tenant_id=tenant_id, + client_ids=client_id, + client_secrets=client_secret, + concurrency=1, + retries=3, + backoff=2 + ) + client.authenticate() + + url = "https://graph.microsoft.com/v1.0/security/dataSecurityAndGovernance/sensitivityLabels" + token_slot = client.get_active_token() + session = client.get_session() + + headers = { + "Authorization": f"Bearer {token_slot['token']}", + "Accept": "application/json" + } + + f = None + writer = None + try: + logger.info("Querying Microsoft Graph information protection sensitivity labels...") + if csv_path: + f = open(csv_path, 'w', encoding='utf-8', newline='') + writer = csv.writer(f) + writer.writerow(["name", "description", "hasProtection", "applicationMode", "priority", "applicableTo", "isEnabled", "is_sublabel"]) + + while url: + if is_cancelled_callback and is_cancelled_callback(): break + resp = session.get(url, headers=headers) + if resp.status_code == 200: + data = resp.json() + value_list = data.get("value", []) + if writer: + for parent in value_list: + writer.writerow([ + parent.get("name", "N/A"), + parent.get("description", "") or parent.get("toolTip", "") or "N/A", + 1 if parent.get("hasProtection", False) else 0, + parent.get("applicationMode", "N/A") or "N/A", + parent.get("priority", 0), + parent.get("applicableTo", ""), + 1 if parent.get("isEnabled", True) else 0, + 0 + ]) + sublabels = parent.get("sublabels", []) + if sublabels: + sublabels_sorted = sorted(sublabels, key=lambda x: x.get("priority", 0), reverse=True) + for sub in sublabels_sorted: + writer.writerow([ + f" ↳ {sub.get('name', 'N/A')}", + sub.get("description", "") or sub.get("toolTip", "") or "N/A", + 1 if sub.get("hasProtection", False) else 0, + sub.get("applicationMode", "N/A") or "N/A", + sub.get("priority", 0), + sub.get("applicableTo", ""), + 1 if sub.get("isEnabled", True) else 0, + 1 + ]) + if on_page_callback: + on_page_callback(value_list) + url = data.get("@odata.nextLink") + else: + logger.error("Graph sensitivityLabels endpoint failed with status %d: %s", resp.status_code, resp.text) + raise ConnectionError(f"Microsoft Graph API request failed with status {resp.status_code}") + finally: + if f: f.close() + client.release_token(token_slot) + client.close() diff --git a/core/graph/security/service_principals_sso.py b/core/graph/security/service_principals_sso.py new file mode 100644 index 00000000..d67fa268 --- /dev/null +++ b/core/graph/security/service_principals_sso.py @@ -0,0 +1,80 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Service Principals SSO configuration settings scanner data pipeline.""" + +import logging +import csv +from core.graph.client import GraphClient + +logger = logging.getLogger(__name__) + +def run_service_principals_sso_pipeline( + client_id: str, + client_secret: str, + tenant_id: str, + csv_path: str = None, + on_page_callback=None, + is_cancelled_callback=None +): + """Fetch Enterprise SSO service principal configurations.""" + client = GraphClient( + tenant_id=tenant_id, + client_ids=client_id, + client_secrets=client_secret, + concurrency=1, + retries=3, + backoff=2 + ) + client.authenticate() + + url = "https://graph.microsoft.com/v1.0/servicePrincipals?$select=id,appDisplayName,preferredSingleSignOnMode&$top=100" + token_slot = client.get_active_token() + session = client.get_session() + + headers = { + "Authorization": f"Bearer {token_slot['token']}", + "Accept": "application/json" + } + + f = None + writer = None + try: + logger.info("Querying Entra ID Service Principals for Single Sign-On modes...") + if csv_path: + f = open(csv_path, 'a', encoding='utf-8', newline='') + writer = csv.writer(f) + + while url: + if is_cancelled_callback and is_cancelled_callback(): break + resp = session.get(url, headers=headers) + if resp.status_code == 200: + data = resp.json() + value_list = data.get("value", []) + if writer: + for sp in value_list: + writer.writerow([sp.get("appDisplayName", ""), sp.get("preferredSingleSignOnMode", "")]) + if on_page_callback: + on_page_callback(value_list) + url = data.get("@odata.nextLink") + elif resp.status_code in [401, 403]: + logger.error("Service Principals endpoint permission error: %d %s", resp.status_code, resp.text) + raise PermissionError("Application.Read.All permission required.") + else: + logger.error("Service Principals endpoint failed with status %d: %s", resp.status_code, resp.text) + raise ConnectionError(f"Microsoft Graph API request failed with status {resp.status_code}") + finally: + if f: f.close() + client.release_token(token_slot) + client.close() diff --git a/core/powershell/__init__.py b/core/powershell/__init__.py new file mode 100644 index 00000000..41d440bb --- /dev/null +++ b/core/powershell/__init__.py @@ -0,0 +1 @@ +# Init powershell core package diff --git a/core/powershell/calendar.py b/core/powershell/calendar.py new file mode 100644 index 00000000..00da544e --- /dev/null +++ b/core/powershell/calendar.py @@ -0,0 +1,87 @@ +import json +from core.powershell.client import PowerShellClient + +class CalendarStatsService: + def __init__(self, client: PowerShellClient): + self.client = client + + def fetch_calendar_attachments_policy(self) -> dict: + """Locates certificate, triggers powershell execution, and parses Exchange calendar configurations.""" + try: + cert_path = self.client.locate_certificate() + except Exception as e: + raise RuntimeError(f"Failed to locate certificate for authentication: {str(e)}") + + args = [ + "-AppId", self.client.client_id, + "-Organization", self.client.tenant_id, + "-CertificatePath", cert_path + ] + if self.client.cert_password: + args += ["-CertificatePassword", self.client.cert_password] + + raw_output = self.client.execute_script("scripts/exchange_calendar_metadata.ps1", args) + + if not raw_output or not raw_output.strip(): + return { + "RoomsCount": 0, + "RoomsError": None, + "RoomsNaming": None, + "EquipmentCount": 0, + "EquipmentError": None, + "CanShareAttachments": True, + "OwaPolicyError": None, + "OrganizationApps": [], + "AppsError": None + } + + # Parse JSON block in case of warning/header lines outputted by powershell environment + lines = raw_output.strip().split('\n') + json_str = "" + for line in lines: + stripped = line.strip() + if stripped.startswith("[") or stripped.startswith("{") or json_str: + json_str += stripped + + if json_str: + try: + return json.loads(json_str) + except json.JSONDecodeError as e: + raise RuntimeError(f"Failed to decode JSON from script output: {str(e)}. Output: {json_str}") + + raise RuntimeError(f"PowerShell returned non-JSON format: {raw_output}") + + def fetch_room_mailboxes(self) -> list: + """Triggers scripts/exchange_room_mailboxes.ps1 to query the list of room mailboxes.""" + try: + cert_path = self.client.locate_certificate() + except Exception as e: + raise RuntimeError(f"Failed to locate certificate for authentication: {str(e)}") + + args = [ + "-AppId", self.client.client_id, + "-Organization", self.client.tenant_id, + "-CertificatePath", cert_path + ] + if self.client.cert_password: + args += ["-CertificatePassword", self.client.cert_password] + + raw_output = self.client.execute_script("scripts/exchange_room_mailboxes.ps1", args) + if not raw_output or not raw_output.strip(): + return [] + + lines = raw_output.strip().split('\n') + json_str = "" + for line in lines: + stripped = line.strip() + if stripped.startswith("[") or stripped.startswith("{") or json_str: + json_str += stripped + + if json_str: + try: + data = json.loads(json_str) + return data.get("RoomsList", []) + except json.JSONDecodeError as e: + raise RuntimeError(f"Failed to decode JSON from script output: {str(e)}. Output: {json_str}") + + raise RuntimeError(f"PowerShell returned non-JSON format: {raw_output}") diff --git a/core/powershell/client.py b/core/powershell/client.py new file mode 100644 index 00000000..79aa0942 --- /dev/null +++ b/core/powershell/client.py @@ -0,0 +1,46 @@ +import os +import subprocess +import logging + +logger = logging.getLogger("PowerShellClient") + +class PowerShellClient: + def __init__(self, tenant_id, client_id, client_secret, cert_tenant_id=None): + self.tenant_id = tenant_id + self.client_id = client_id + self.cert_password = client_secret + self.cert_tenant_id = cert_tenant_id or tenant_id + + def locate_certificate(self) -> str: + """Locates the automated hybrid auth PFX certificate path.""" + from core.cert_auth import get_cert_paths + _, _, pfx_path = get_cert_paths(tenant_id=self.cert_tenant_id, client_id=self.client_id) + if not os.path.exists(pfx_path): + raise FileNotFoundError(f"Hybrid auth PFX certificate not found at {pfx_path}. Please complete the hybrid authentication flow on the login page first.") + return pfx_path + + def execute_script(self, script_relative_path: str, args: list) -> str: + """Executes pwsh with the specified script and arguments.""" + script_path = os.path.join(os.path.dirname(__file__), script_relative_path) + + if not os.path.exists(script_path): + raise FileNotFoundError(f"PowerShell script not found at {script_path}") + + # Construct CLI command + command = ["pwsh", "-NoProfile", "-NonInteractive", "-File", script_path] + args + + try: + result = subprocess.run( + command, + capture_output=True, + text=True, + check=True + ) + if result.stderr and result.stderr.strip(): + logger.warning(f"PowerShell script '{script_path}' stderr diagnostics: {result.stderr.strip()}") + return result.stdout + except subprocess.CalledProcessError as e: + logger.error(f"PowerShell script failed: {e.stderr}", exc_info=True) + raise RuntimeError(f"PowerShell script execution failed: {e.stderr or e.stdout}") + except FileNotFoundError: + raise RuntimeError("PowerShell core ('pwsh') is not installed or not in PATH. Please install it (e.g. 'brew install powershell' on macOS).") diff --git a/core/powershell/dlp.py b/core/powershell/dlp.py new file mode 100644 index 00000000..5965b503 --- /dev/null +++ b/core/powershell/dlp.py @@ -0,0 +1,120 @@ +import os +import json +import logging + +logger = logging.getLogger(__name__) + +class DLPService: + """Service for interacting with Exchange Online / Purview PowerShell for DLP policies.""" + + def __init__(self, ps_client): + self.ps_client = ps_client + + def fetch_dlp_policies(self) -> dict: + """ + Executes the PowerShell script to retrieve DLP policies via Get-DlpCompliancePolicy. + """ + try: + cert_path = self.ps_client.locate_certificate() + except Exception as e: + raise RuntimeError(f"Failed to locate certificate for authentication: {str(e)}") + + args = [ + "-AppId", self.ps_client.client_id, + "-Organization", self.ps_client.tenant_id, + "-CertificatePath", cert_path + ] + if self.ps_client.cert_password: + args += ["-CertificatePassword", self.ps_client.cert_password] + + logger.info("Executing fetch_dlp_policies script") + + try: + raw_output = self.ps_client.execute_script("scripts/get_dlp_policies.ps1", args) + if not raw_output or not raw_output.strip(): + return {"value": []} + + try: + data = json.loads(raw_output) + if isinstance(data, dict) and "value" in data: + return data + elif isinstance(data, list): + return {"value": data} + else: + return {"value": [data]} + except json.JSONDecodeError as e: + # Parse JSON block in case of warning/header lines outputted by powershell environment + lines = raw_output.strip().split('\n') + json_str = "" + for line in lines: + if line.startswith("[") or line.startswith("{") or json_str: + json_str += line + if json_str: + data = json.loads(json_str) + if isinstance(data, dict) and "value" in data: + return data + elif isinstance(data, list): + return {"value": data} + else: + return {"value": [data]} + raise RuntimeError(f"PowerShell returned non-JSON format: {raw_output}") + + except Exception as e: + logger.error("Error executing fetch_dlp_policies", exc_info=True) + raise Exception(f"PowerShell script execution failed: {str(e)}") + + def fetch_sensitive_info_types(self) -> dict: + """ + Executes the PowerShell script to retrieve Sensitive Information Types via Get-DlpSensitiveInformationType. + """ + try: + cert_path = self.ps_client.locate_certificate() + except Exception as e: + raise RuntimeError(f"Failed to locate certificate for authentication: {str(e)}") + + args = [ + "-AppId", self.ps_client.client_id, + "-Organization", self.ps_client.tenant_id, + "-CertificatePath", cert_path + ] + if self.ps_client.cert_password: + args += ["-CertificatePassword", self.ps_client.cert_password] + + logger.info("Executing fetch_sensitive_info_types script") + + try: + raw_output = self.ps_client.execute_script("scripts/get_sensitive_info_types.ps1", args) + if not raw_output or not raw_output.strip(): + return {"value": []} + + try: + data = json.loads(raw_output) + if isinstance(data, dict) and "SensitiveInformationTypes" in data: + return {"value": data["SensitiveInformationTypes"]} + elif isinstance(data, dict) and "value" in data: + return data + elif isinstance(data, list): + return {"value": data} + else: + return {"value": [data]} + except json.JSONDecodeError as e: + lines = raw_output.strip().split('\n') + json_str = "" + for line in lines: + if line.startswith("[") or line.startswith("{") or json_str: + json_str += line + if json_str: + data = json.loads(json_str) + if isinstance(data, dict) and "SensitiveInformationTypes" in data: + return {"value": data["SensitiveInformationTypes"]} + elif isinstance(data, dict) and "value" in data: + return data + elif isinstance(data, list): + return {"value": data} + else: + return {"value": [data]} + raise RuntimeError(f"PowerShell returned non-JSON format: {raw_output}") + + except Exception as e: + logger.error("Error executing fetch_sensitive_info_types", exc_info=True) + raise Exception(f"PowerShell script execution failed: {str(e)}") diff --git a/core/powershell/encryption.py b/core/powershell/encryption.py new file mode 100644 index 00000000..7222c195 --- /dev/null +++ b/core/powershell/encryption.py @@ -0,0 +1,45 @@ +import json +import logging +from core.powershell.client import PowerShellClient +from core.cert_auth import load_certificate + +logger = logging.getLogger(__name__) + +def get_encryption_policies(client: PowerShellClient) -> dict: + """ + Executes the export_encryption_policies.ps1 script using the PowerShellClient. + Returns a dictionary with 'm365_policies' and 'exchange_deps'. + """ + logger.info("Starting M365 Data Encryption Policy fetch via PowerShell...") + try: + cert_path = client.locate_certificate() + + args = [ + "-Organization", client.tenant_id, + "-AppId", client.client_id, + "-CertificateFilePath", cert_path, + "-CertificatePassword", client.cert_password + ] + + stdout = client.execute_script("scripts/export_encryption_policies.ps1", args) + + # Parse the JSON output from PowerShell + if stdout and stdout.strip(): + # Find where the JSON starts in case of warnings + try: + json_start = stdout.find('{') + if json_start != -1: + json_str = stdout[json_start:] + return json.loads(json_str) + else: + logger.warning("No JSON object found in PowerShell output.") + return {"m365_policies": [], "exchange_deps": []} + except json.JSONDecodeError as e: + logger.error(f"Failed to decode JSON from PowerShell: {e}\nOutput: {stdout}") + raise RuntimeError("Failed to parse encryption policies from Exchange Online.") + else: + return {"m365_policies": [], "exchange_deps": []} + + except Exception as e: + logger.error(f"Error fetching encryption policies: {e}", exc_info=True) + raise diff --git a/core/powershell/exchange_connectors.py b/core/powershell/exchange_connectors.py new file mode 100644 index 00000000..9748c903 --- /dev/null +++ b/core/powershell/exchange_connectors.py @@ -0,0 +1,46 @@ +import json +import logging +from core.powershell.client import PowerShellClient + +logger = logging.getLogger("ExchangeConnectorsService") + +class ExchangeConnectorsService: + def __init__(self, client: PowerShellClient): + self.client = client + + def fetch_exchange_connectors(self) -> dict: + """Executes get_connectors.ps1 and parses the JSON response.""" + logger.info("Executing Exchange Connectors PowerShell script...") + try: + cert_path = self.client.locate_certificate() + except FileNotFoundError as e: + logger.error(str(e)) + raise RuntimeError(str(e)) + + args = [ + "-AppId", self.client.client_id, + "-Organization", self.client.tenant_id, + "-CertificatePath", cert_path + ] + + if self.client.cert_password: + args.extend(["-CertificatePassword", self.client.cert_password]) + + try: + output = self.client.execute_script("scripts/get_connectors.ps1", args) + + # Extract JSON from output + json_str = output + if "{" in output: + json_str = output[output.find("{"):] + + if not json_str.strip(): + return {"InboundConnectors": [], "OutboundConnectors": [], "Errors": {}} + + return json.loads(json_str) + except json.JSONDecodeError as e: + logger.error(f"Failed to parse Exchange Connectors JSON: {e}") + raise RuntimeError(f"Failed to parse Exchange Connectors data: {e}") + except Exception as e: + logger.error(f"Exchange Connectors retrieval failed: {e}") + raise RuntimeError(f"Exchange Connectors retrieval failed: {e}") diff --git a/core/powershell/mailbox.py b/core/powershell/mailbox.py new file mode 100644 index 00000000..e2e543b0 --- /dev/null +++ b/core/powershell/mailbox.py @@ -0,0 +1,39 @@ +import json +from core.powershell.client import PowerShellClient + +class MailboxStatsService: + def __init__(self, client: PowerShellClient): + self.client = client + + def fetch_mailbox_and_folder_stats(self) -> dict: + """Locates certificate, triggers powershell execution, and parses shared mailbox and public folder stats.""" + try: + cert_path = self.client.locate_certificate() + except Exception as e: + raise RuntimeError(f"Failed to locate certificate for authentication: {str(e)}") + + args = [ + "-AppId", self.client.client_id, + "-Organization", self.client.tenant_id, + "-CertificatePath", cert_path + ] + if self.client.cert_password: + args += ["-CertificatePassword", self.client.cert_password] + + raw_output = self.client.execute_script("scripts/get_mailbox_and_folder_stats.ps1", args) + + if not raw_output or not raw_output.strip(): + return {} + + try: + return json.loads(raw_output) + except json.JSONDecodeError: + # Parse JSON block in case of warning/header lines outputted by powershell environment + lines = raw_output.strip().split('\n') + json_str = "" + for line in lines: + if line.startswith("[") or line.startswith("{") or json_str: + json_str += line + if json_str: + return json.loads(json_str) + raise RuntimeError(f"PowerShell returned non-JSON format: {raw_output}") diff --git a/core/powershell/retention.py b/core/powershell/retention.py new file mode 100644 index 00000000..214a6ae2 --- /dev/null +++ b/core/powershell/retention.py @@ -0,0 +1,39 @@ +import json +from core.powershell.client import PowerShellClient + +class RetentionService: + def __init__(self, client: PowerShellClient): + self.client = client + + def fetch_retention_policies(self) -> list: + """Locates certificate, triggers powershell execution, and parses retention policies.""" + try: + cert_path = self.client.locate_certificate() + except Exception as e: + raise RuntimeError(f"Failed to locate certificate for authentication: {str(e)}") + + args = [ + "-AppId", self.client.client_id, + "-Organization", self.client.tenant_id, + "-CertificatePath", cert_path + ] + if self.client.cert_password: + args += ["-CertificatePassword", self.client.cert_password] + + raw_output = self.client.execute_script("scripts/get_retention_policies.ps1", args) + + if not raw_output or not raw_output.strip(): + return [] + + try: + return json.loads(raw_output) + except json.JSONDecodeError: + # Parse JSON block in case of warning/header lines outputted by powershell environment + lines = raw_output.strip().split('\n') + json_str = "" + for line in lines: + if line.startswith("[") or line.startswith("{") or json_str: + json_str += line + if json_str: + return json.loads(json_str) + raise RuntimeError(f"PowerShell returned non-JSON format: {raw_output}") diff --git a/core/powershell/scripts/exchange_calendar_metadata.ps1 b/core/powershell/scripts/exchange_calendar_metadata.ps1 new file mode 100644 index 00000000..4b5faaf8 --- /dev/null +++ b/core/powershell/scripts/exchange_calendar_metadata.ps1 @@ -0,0 +1,99 @@ +param( + [Parameter(Mandatory=$true)] + [string]$AppId, + [Parameter(Mandatory=$true)] + [string]$Organization, + [Parameter(Mandatory=$true)] + [string]$CertificatePath, + [Parameter(Mandatory=$false)] + [string]$CertificatePassword +) + +$ErrorActionPreference = "Stop" + +if (-not (Get-Module -ListAvailable -Name ExchangeOnlineManagement)) { + throw "ExchangeOnlineManagement PowerShell module is not installed." +} + +Import-Module ExchangeOnlineManagement + +# Connect to Exchange Online using App-Only Cert Auth +if ($CertificatePassword) { + $secPassword = ConvertTo-SecureString -String $CertificatePassword -AsPlainText -Force + Connect-ExchangeOnline -CertificateFilePath $CertificatePath -CertificatePassword $secPassword -AppId $AppId -Organization $Organization -ShowBanner:$false -WarningAction SilentlyContinue +} else { + Connect-ExchangeOnline -CertificateFilePath $CertificatePath -AppId $AppId -Organization $Organization -ShowBanner:$false -WarningAction SilentlyContinue +} + +try { + $roomsCount = 0 + $roomsNaming = $null + $roomsError = $null + try { + $rooms = @(Get-EXOMailbox -RecipientTypeDetails RoomMailbox -ResultSize Unlimited) + $roomsCount = $rooms.Count + $roomsNaming = if ($rooms) { (($rooms | Select-Object -First 5 -ExpandProperty Name) -join ", ") } else { $null } + } catch { + $roomsError = $_.Exception.Message + } + + $equipmentCount = 0 + $equipmentError = $null + try { + $equipment = @(Get-EXOMailbox -RecipientTypeDetails EquipmentMailbox -ResultSize Unlimited) + $equipmentCount = $equipment.Count + } catch { + $equipmentError = $_.Exception.Message + } + + $owaPolicyError = $null + $canShareAttachments = $true + try { + $owaPolicy = Get-OwaMailboxPolicy | Where-Object { $_.IsDefault -eq $true } + if (-not $owaPolicy) { + $owaPolicy = Get-OwaMailboxPolicy | Select-Object -First 1 + } + $canShareAttachments = if ($owaPolicy) { $owaPolicy.ClassicAttachmentsEnabled } else { $true } + } catch { + $owaPolicyError = $_.Exception.Message + } + + $orgAppsData = @() + $appsError = $null + try { + $orgApps = @(Get-App -OrganizationApp -ErrorAction Stop) + if ($orgApps) { + foreach ($app in $orgApps) { + $orgAppsData += [PSCustomObject]@{ + DisplayName = $app.DisplayName + AppId = $app.AppId + Enabled = $app.Enabled + } + } + } + } catch { + $appsError = $_.Exception.Message + } + + $roomsList = @() + if ($rooms) { + $roomsList = @($rooms | Select-Object -ExpandProperty PrimarySmtpAddress) + } + + $result = [PSCustomObject]@{ + RoomsCount = $roomsCount + RoomsError = $roomsError + RoomsNaming = $roomsNaming + RoomsList = $roomsList + EquipmentCount = $equipmentCount + EquipmentError = $equipmentError + CanShareAttachments = $canShareAttachments + OwaPolicyError = $owaPolicyError + OrganizationApps = $orgAppsData + AppsError = $appsError + } + $result | ConvertTo-Json +} +finally { + Disconnect-ExchangeOnline -Confirm:$false -ErrorAction SilentlyContinue +} diff --git a/core/powershell/scripts/exchange_room_mailboxes.ps1 b/core/powershell/scripts/exchange_room_mailboxes.ps1 new file mode 100644 index 00000000..9e3683c6 --- /dev/null +++ b/core/powershell/scripts/exchange_room_mailboxes.ps1 @@ -0,0 +1,55 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +param( + [Parameter(Mandatory=$true)] + [string]$AppId, + [Parameter(Mandatory=$true)] + [string]$Organization, + [Parameter(Mandatory=$true)] + [string]$CertificatePath, + [Parameter(Mandatory=$false)] + [string]$CertificatePassword +) + +$ErrorActionPreference = "Stop" + +if (-not (Get-Module -ListAvailable -Name ExchangeOnlineManagement)) { + throw "ExchangeOnlineManagement PowerShell module is not installed." +} + +Import-Module ExchangeOnlineManagement + +# Connect to Exchange Online using App-Only Cert Auth +if ($CertificatePassword) { + $secPassword = ConvertTo-SecureString -String $CertificatePassword -AsPlainText -Force + Connect-ExchangeOnline -CertificateFilePath $CertificatePath -CertificatePassword $secPassword -AppId $AppId -Organization $Organization -ShowBanner:$false -WarningAction SilentlyContinue +} else { + Connect-ExchangeOnline -CertificateFilePath $CertificatePath -AppId $AppId -Organization $Organization -ShowBanner:$false -WarningAction SilentlyContinue +} + +try { + $rooms = @(Get-EXOMailbox -RecipientTypeDetails RoomMailbox -ResultSize Unlimited) + $roomsList = @() + if ($rooms) { + $roomsList = @($rooms | Select-Object -ExpandProperty PrimarySmtpAddress) + } + $result = [PSCustomObject]@{ + RoomsList = $roomsList + } + $result | ConvertTo-Json +} +finally { + Disconnect-ExchangeOnline -Confirm:$false -ErrorAction SilentlyContinue +} diff --git a/core/powershell/scripts/export_encryption_policies.ps1 b/core/powershell/scripts/export_encryption_policies.ps1 new file mode 100644 index 00000000..17eed01d --- /dev/null +++ b/core/powershell/scripts/export_encryption_policies.ps1 @@ -0,0 +1,67 @@ +param ( + [Parameter(Mandatory=$true)][string]$Organization, + [Parameter(Mandatory=$true)][string]$AppId, + [Parameter(Mandatory=$true)][string]$CertificateFilePath, + [Parameter(Mandatory=$true)][string]$CertificatePassword +) + +# Force TLS 1.2 +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + +$ErrorActionPreference = "Stop" + +if (-not (Get-Module -ListAvailable -Name ExchangeOnlineManagement)) { + throw "ExchangeOnlineManagement PowerShell module is not installed." +} + +Import-Module ExchangeOnlineManagement + +try { + # Convert password to SecureString for macOS/Linux compatibility + $secPass = ConvertTo-SecureString $CertificatePassword -AsPlainText -Force + # Connect using Certificate-based App-Only Authentication + Connect-ExchangeOnline -AppID $AppId -Organization $Organization -CertificateFilePath $CertificateFilePath -CertificatePassword $secPass -ShowBanner:$false -ErrorAction Stop + + $result = @{ + "m365_policies" = @() + "exchange_deps" = @() + } + + # Fetch M365 Data at Rest Encryption Policies (Customer Key multi-workload) + try { + $m365Policies = Get-M365DataAtRestEncryptionPolicy -ErrorAction SilentlyContinue + if ($m365Policies) { + foreach ($pol in $m365Policies) { + $result["m365_policies"] += @{ + "Name" = $pol.Name + "Description" = $pol.Description + } + } + } + } catch { + # Catch unsupported or missing permissions specifically for this cmdlet + } + + # Fetch legacy Exchange Data Encryption Policies (DEPs) + try { + $deps = Get-DataEncryptionPolicy -ErrorAction SilentlyContinue + if ($deps) { + foreach ($dep in $deps) { + $result["exchange_deps"] += @{ + "Name" = $dep.Name + "Description" = $dep.Description + } + } + } + } catch { + # Catch unsupported or missing permissions specifically for this cmdlet + } + + $result | ConvertTo-Json -Depth 5 -Compress + +} catch { + Write-Error $_.Exception.Message + exit 1 +} finally { + Disconnect-ExchangeOnline -Confirm:$false -ErrorAction SilentlyContinue +} diff --git a/core/powershell/scripts/get_connectors.ps1 b/core/powershell/scripts/get_connectors.ps1 new file mode 100644 index 00000000..e61ff473 --- /dev/null +++ b/core/powershell/scripts/get_connectors.ps1 @@ -0,0 +1,76 @@ +param( + [Parameter(Mandatory=$true)] + [string]$AppId, + [Parameter(Mandatory=$true)] + [string]$Organization, + [Parameter(Mandatory=$true)] + [string]$CertificatePath, + [Parameter(Mandatory=$false)] + [string]$CertificatePassword +) + +$ErrorActionPreference = "Stop" + +if (-not (Get-Module -ListAvailable -Name ExchangeOnlineManagement)) { + throw "ExchangeOnlineManagement PowerShell module is not installed. Please install it beforehand by running: Install-Module -Name ExchangeOnlineManagement -Scope CurrentUser" +} + +Import-Module ExchangeOnlineManagement + +# Connect to Exchange Online using App-Only Cert Auth +if ($CertificatePassword) { + $secPassword = ConvertTo-SecureString -String $CertificatePassword -AsPlainText -Force + Connect-ExchangeOnline -CertificateFilePath $CertificatePath -CertificatePassword $secPassword -AppId $AppId -Organization $Organization -ShowBanner:$false -WarningAction SilentlyContinue +} else { + Connect-ExchangeOnline -CertificateFilePath $CertificatePath -AppId $AppId -Organization $Organization -ShowBanner:$false -WarningAction SilentlyContinue +} + +try { + $errors = @{} + $inbound = @() + $outbound = @() + + try { + $inboundRaw = Get-InboundConnector -ResultSize Unlimited -ErrorAction Stop + if ($inboundRaw) { + foreach ($conn in @($inboundRaw)) { + $inbound += @{ + Name = $conn.Identity + Enabled = $conn.Enabled + ConnectorType = $conn.ConnectorType + SenderDomains = ($conn.SenderDomains -join ", ") + RequireTls = $conn.RequireTls + } + } + } + } catch { + $errors["InboundConnectors"] = $_.Exception.Message + } + + try { + $outboundRaw = Get-OutboundConnector -ResultSize Unlimited -ErrorAction Stop + if ($outboundRaw) { + foreach ($conn in @($outboundRaw)) { + $outbound += @{ + Name = $conn.Identity + Enabled = $conn.Enabled + RecipientDomains = ($conn.RecipientDomains -join ", ") + SmartHosts = ($conn.SmartHosts -join ", ") + UseMxRecord = $conn.UseMxRecord + } + } + } + } catch { + $errors["OutboundConnectors"] = $_.Exception.Message + } + + $result = [PSCustomObject]@{ + InboundConnectors = $inbound + OutboundConnectors = $outbound + Errors = $errors + } + $result | ConvertTo-Json -Depth 5 +} +finally { + Disconnect-ExchangeOnline -Confirm:$false -ErrorAction SilentlyContinue +} diff --git a/core/powershell/scripts/get_dlp_policies.ps1 b/core/powershell/scripts/get_dlp_policies.ps1 new file mode 100644 index 00000000..def4672a --- /dev/null +++ b/core/powershell/scripts/get_dlp_policies.ps1 @@ -0,0 +1,107 @@ +param ( + [Parameter(Mandatory=$true)] + [string]$AppId, + + [Parameter(Mandatory=$true)] + [string]$Organization, + + [Parameter(Mandatory=$true)] + [string]$CertificatePath, + + [Parameter(Mandatory=$true)] + [string]$CertificatePassword +) + +$ErrorActionPreference = "Stop" + +try { + # Convert password to secure string + $secPassword = ConvertTo-SecureString -String $CertificatePassword -AsPlainText -Force + + # Import module and connect + Import-Module ExchangeOnlineManagement -ErrorAction Stop + Connect-IPPSSession -CertificateFilePath $CertificatePath -CertificatePassword $secPassword -AppId $AppId -Organization $Organization -ShowBanner:$false -WarningAction SilentlyContinue + + # Retrieve policies + $policies = Get-DlpCompliancePolicy + $output = @() + + if ($policies) { + # Retrieve all compliance rules at once to map them + $allRules = Get-DlpComplianceRule + $ruleMap = @{} + if ($allRules) { + foreach ($rule in @($allRules)) { + if ($rule.Policy) { + $rawKey = $rule.Policy.ToString() + + if (-not $ruleMap.ContainsKey($rawKey)) { + $ruleMap[$rawKey] = @() + } + $ruleMap[$rawKey] += $rule + + # Extract the policy name if the reference is a DistinguishedName + if ($rawKey -match "CN=([^,]+)") { + $cnKey = $Matches[1] + if (-not $ruleMap.ContainsKey($cnKey)) { + $ruleMap[$cnKey] = @() + } + $ruleMap[$cnKey] += $rule + } + } + } + } + + # Handle case where $policies is not an array + $policies_list = @($policies) + + foreach ($policy in $policies_list) { + $actions = "None" + + $rules = $null + if ($ruleMap.ContainsKey($policy.Name)) { + $rules = $ruleMap[$policy.Name] + } elseif ($ruleMap.ContainsKey($policy.Identity.ToString())) { + $rules = $ruleMap[$policy.Identity.ToString()] + } elseif ($policy.Guid -and $ruleMap.ContainsKey($policy.Guid.ToString())) { + $rules = $ruleMap[$policy.Guid.ToString()] + } + + if ($rules) { + $actionList = @() + foreach ($rule in $rules) { + if ($rule.BlockAccess -eq $true) { $actionList += "BlockAccess" } + if ($rule.GenerateIncidentReport -ne $null) { $actionList += "IncidentReport" } + if ($rule.NotifyUser -ne $null) { $actionList += "NotifyUser" } + } + if ($actionList.Count -gt 0) { + # Dedup actions + $actions = ($actionList | Select-Object -Unique) -join ", " + } + } + + # Construct a combined custom object + $policyObj = [PSCustomObject]@{ + Name = $policy.Name + Comment = $policy.Comment + Workload = $policy.Workload + Mode = $policy.Mode + DistributionStatus = $policy.DistributionStatus + Enabled = $policy.Enabled + Actions = $actions + Identity = $policy.Identity.ToString() + WhenCreated = $policy.WhenCreated + WhenChanged = $policy.WhenChanged + CreatedBy = $policy.CreatedBy + LastModifiedBy = $policy.LastModifiedBy + } + $output += $policyObj + } + $output | ConvertTo-Json -Depth 5 + } else { + "[]" + } +} +finally { + Disconnect-ExchangeOnline -Confirm:$false -WarningAction SilentlyContinue +} diff --git a/core/powershell/scripts/get_mailbox_and_folder_stats.ps1 b/core/powershell/scripts/get_mailbox_and_folder_stats.ps1 new file mode 100644 index 00000000..dad48032 --- /dev/null +++ b/core/powershell/scripts/get_mailbox_and_folder_stats.ps1 @@ -0,0 +1,131 @@ +param( + [Parameter(Mandatory=$true)] + [string]$AppId, + [Parameter(Mandatory=$true)] + [string]$Organization, + [Parameter(Mandatory=$true)] + [string]$CertificatePath, + [Parameter(Mandatory=$false)] + [string]$CertificatePassword +) + +$ErrorActionPreference = "Stop" + +# Check if ExchangeOnlineManagement module is installed beforehand +if (-not (Get-Module -ListAvailable -Name ExchangeOnlineManagement)) { + throw "ExchangeOnlineManagement PowerShell module is not installed. Please install it beforehand by running: Install-Module -Name ExchangeOnlineManagement -Scope CurrentUser" +} + +Import-Module ExchangeOnlineManagement + +# Connect to Exchange Online using App-Only Cert Auth +if ($CertificatePassword) { + $secPassword = ConvertTo-SecureString -String $CertificatePassword -AsPlainText -Force + Connect-ExchangeOnline -CertificateFilePath $CertificatePath -CertificatePassword $secPassword -AppId $AppId -Organization $Organization -ShowBanner:$false -WarningAction SilentlyContinue +} else { + Connect-ExchangeOnline -CertificateFilePath $CertificatePath -AppId $AppId -Organization $Organization -ShowBanner:$false -WarningAction SilentlyContinue +} + +try { + $errors = @{} + + # 1. Query Shared Mailboxes + $sharedCount = $null + $sharedTotalBytes = $null + try { + $sharedMailboxes = Get-EXOMailbox -RecipientTypeDetails SharedMailbox -ResultSize Unlimited -ErrorAction Stop + if ($sharedMailboxes) { + $sharedStats = @($sharedMailboxes | Get-EXOMailboxStatistics -ErrorAction Stop) + $sharedCount = @($sharedMailboxes).Count + $sharedTotalBytes = [long]0 + foreach ($stat in $sharedStats) { + if ($stat.TotalItemSize -and $stat.TotalItemSize.Value) { + try { + $bytes = $stat.TotalItemSize.Value.ToBytes() + } catch { + $bytesStr = $stat.TotalItemSize.ToString() + if ($bytesStr -match '\(([\d,]+) bytes\)') { + $bytes = [long]($Matches[1] -replace ',', '') + } else { + $bytes = [long]0 + } + } + $sharedTotalBytes += $bytes + } + } + } else { + $sharedCount = 0 + $sharedTotalBytes = 0 + } + } catch { + $errors["SharedMailboxes"] = $_.Exception.Message + } + + # 2. Query Public Folders + $pfCount = $null + try { + $pfs = Get-PublicFolder -Recurse -ResultSize Unlimited -ErrorAction Stop + if ($pfs) { + $pfCount = @($pfs).Count + } else { + $pfCount = 0 + } + } catch { + $errors["PublicFolders"] = $_.Exception.Message + } + + # 2b. Query Mail-Enabled Public Folders + $mailPfCount = $null + try { + $mailPfs = Get-MailPublicFolder -ResultSize Unlimited -ErrorAction Stop + if ($mailPfs) { + $mailPfCount = @($mailPfs).Count + } else { + $mailPfCount = 0 + } + } catch { + $errors["MailPublicFolders"] = $_.Exception.Message + } + + # 2c. Query Public Folder Stats (Size) + $pfTotalBytes = $null + try { + $pfStats = Get-PublicFolderStatistics -ResultSize Unlimited -ErrorAction Stop + if ($pfStats) { + $pfTotalBytes = [long]0 + foreach ($stat in @($pfStats)) { + if ($stat.TotalItemSize -and $stat.TotalItemSize.Value) { + try { + $bytes = $stat.TotalItemSize.Value.ToBytes() + } catch { + $bytesStr = $stat.TotalItemSize.ToString() + if ($bytesStr -match '\(([\d,]+) bytes\)') { + $bytes = [long]($Matches[1] -replace ',', '') + } else { + $bytes = [long]0 + } + } + $pfTotalBytes += $bytes + } + } + } else { + $pfTotalBytes = 0 + } + } catch { + $errors["PublicFolderStats"] = $_.Exception.Message + } + + # Output results as JSON + $result = [PSCustomObject]@{ + SharedMailboxesCount = $sharedCount + SharedMailboxesTotalBytes = $sharedTotalBytes + PublicFoldersCount = $pfCount + PublicFoldersTotalBytes = $pfTotalBytes + MailPublicFoldersCount = $mailPfCount + Errors = $errors + } + $result | ConvertTo-Json +} +finally { + Disconnect-ExchangeOnline -Confirm:$false -ErrorAction SilentlyContinue +} diff --git a/core/powershell/scripts/get_retention_policies.ps1 b/core/powershell/scripts/get_retention_policies.ps1 new file mode 100644 index 00000000..b151b147 --- /dev/null +++ b/core/powershell/scripts/get_retention_policies.ps1 @@ -0,0 +1,102 @@ +param( + [Parameter(Mandatory=$true)] + [string]$AppId, + [Parameter(Mandatory=$true)] + [string]$Organization, + [Parameter(Mandatory=$true)] + [string]$CertificatePath, + [Parameter(Mandatory=$false)] + [string]$CertificatePassword +) + +$ErrorActionPreference = "Stop" + +# Check if ExchangeOnlineManagement module is installed beforehand +if (-not (Get-Module -ListAvailable -Name ExchangeOnlineManagement)) { + throw "ExchangeOnlineManagement PowerShell module is not installed. Please install it beforehand by running: Install-Module -Name ExchangeOnlineManagement -Scope CurrentUser" +} + +Import-Module ExchangeOnlineManagement + +# Connect to Security & Compliance PowerShell using App-Only Cert Auth +if ($CertificatePassword) { + $secPassword = ConvertTo-SecureString -String $CertificatePassword -AsPlainText -Force + Connect-IPPSSession -CertificateFilePath $CertificatePath -CertificatePassword $secPassword -AppId $AppId -Organization $Organization -ShowBanner:$false -WarningAction SilentlyContinue +} else { + Connect-IPPSSession -CertificateFilePath $CertificatePath -AppId $AppId -Organization $Organization -ShowBanner:$false -WarningAction SilentlyContinue +} + +try { + # Retrieve policies + $policies = Get-RetentionCompliancePolicy + $output = @() + + if ($policies) { + # Retrieve all compliance rules at once + $allRules = Get-RetentionComplianceRule + $ruleMap = @{} + if ($allRules) { + foreach ($rule in @($allRules)) { + if ($rule.Policy) { + $rawKey = $rule.Policy.ToString() + $ruleMap[$rawKey] = $rule + + # Extract the policy name if the reference is a DistinguishedName + if ($rawKey -match "CN=([^,]+)") { + $cnKey = $Matches[1] + $ruleMap[$cnKey] = $rule + } + } + } + } + + # Handle case where $policies is not an array + $policies_list = @($policies) + + foreach ($policy in $policies_list) { + $duration = "N/A" + $action = "N/A" + $trigger = "N/A" + + $rule = $null + if ($ruleMap.ContainsKey($policy.Name)) { + $rule = $ruleMap[$policy.Name] + } elseif ($ruleMap.ContainsKey($policy.Identity.ToString())) { + $rule = $ruleMap[$policy.Identity.ToString()] + } elseif ($policy.Guid -and $ruleMap.ContainsKey($policy.Guid.ToString())) { + $rule = $ruleMap[$policy.Guid.ToString()] + } + + if ($rule) { + $duration = $rule.RetentionDuration + $action = $rule.RetentionAction + $trigger = $rule.RetentionTrigger + } + + # Construct a combined custom object + $policyObj = [PSCustomObject]@{ + Name = $policy.Name + Comment = $policy.Comment + Workload = $policy.Workload + Mode = $policy.Mode + DistributionStatus = $policy.DistributionStatus + Enabled = $policy.Enabled + Duration = $duration + RetentionAction = $action + RetentionTrigger = $trigger + Identity = $policy.Identity.ToString() + WhenCreated = $policy.WhenCreated + WhenChanged = $policy.WhenChanged + CreatedBy = $policy.CreatedBy + LastModifiedBy = $policy.LastModifiedBy + } + $output += $policyObj + } + $output | ConvertTo-Json -Depth 5 + } else { + "[]" + } +} +finally { + Disconnect-ExchangeOnline -Confirm:$false +} diff --git a/core/powershell/scripts/get_sensitive_info_types.ps1 b/core/powershell/scripts/get_sensitive_info_types.ps1 new file mode 100644 index 00000000..57cb50ca --- /dev/null +++ b/core/powershell/scripts/get_sensitive_info_types.ps1 @@ -0,0 +1,40 @@ +param( + [Parameter(Mandatory=$true)] + [string]$AppId, + + [Parameter(Mandatory=$true)] + [string]$Organization, + + [Parameter(Mandatory=$true)] + [string]$CertificatePath, + + [Parameter(Mandatory=$false)] + [string]$CertificatePassword +) + +try { + # Import the module + Import-Module ExchangeOnlineManagement -ErrorAction Stop + + # Authenticate + if ([string]::IsNullOrEmpty($CertificatePassword)) { + Connect-IPPSSession -AppId $AppId -Organization $Organization -CertificateFilePath $CertificatePath -ShowBanner:$false + } else { + $secPassword = ConvertTo-SecureString -String $CertificatePassword -AsPlainText -Force + Connect-IPPSSession -AppId $AppId -Organization $Organization -CertificateFilePath $CertificatePath -CertificatePassword $secPassword -ShowBanner:$false + } + + # Retrieve sensitive info types + $sitData = Get-DlpSensitiveInformationType | Select-Object Name, Description, PublisherName, RecommendedConfidence, Type, IsExactMatch, ContainsData + + # Return as JSON + $result = @{ + "SensitiveInformationTypes" = $sitData + } + $result | ConvertTo-Json -Depth 10 +} catch { + Write-Error "Failed to fetch Sensitive Information Types: $_" + exit 1 +} finally { + Disconnect-ExchangeOnline -Confirm:$false -ErrorAction SilentlyContinue +} diff --git a/core/powershell/scripts/get_transport_rules.ps1 b/core/powershell/scripts/get_transport_rules.ps1 new file mode 100644 index 00000000..2b272372 --- /dev/null +++ b/core/powershell/scripts/get_transport_rules.ps1 @@ -0,0 +1,58 @@ +param( + [Parameter(Mandatory=$true)] + [string]$AppId, + [Parameter(Mandatory=$true)] + [string]$Organization, + [Parameter(Mandatory=$true)] + [string]$ClientSecret, + [Parameter(Mandatory=$true)] + [string]$CsvPath +) + +$ErrorActionPreference = "Stop" + +if (-not (Get-Module -ListAvailable -Name ExchangeOnlineManagement)) { + throw "ExchangeOnlineManagement PowerShell module is not installed." +} + +Import-Module ExchangeOnlineManagement + +$body = @{ + grant_type = "client_credentials" + client_id = $AppId + client_secret = $ClientSecret + scope = "https://outlook.office365.com/.default" +} + +try { + $tokenResponse = Invoke-RestMethod -Method Post -Uri "https://login.microsoftonline.com/$Organization/oauth2/v2.0/token" -ContentType "application/x-www-form-urlencoded" -Body $body + $token = $tokenResponse.access_token + + Connect-ExchangeOnline -AccessToken $token -Organization $Organization -ShowBanner:$false -WarningAction SilentlyContinue + + $rules = Get-TransportRule -ErrorAction Stop + if ($rules) { + $rules | Select-Object Name, State, Priority, Mode, Description, + @{Name="Conditions"; Expression={if ($_.Conditions) { $_.Conditions -join ", " } else { $null }}}, + @{Name="Actions"; Expression={if ($_.Actions) { $_.Actions -join ", " } else { $null }}}, + @{Name="Exceptions"; Expression={if ($_.Exceptions) { $_.Exceptions -join ", " } else { $null }}}, + Comments | Export-Csv -Path $CsvPath -NoTypeInformation -Encoding UTF8 -Force + } else { + # Create an empty CSV with headers + "" | Select-Object Name, State, Priority, Mode, Description, Conditions, Actions, Exceptions, Comments | ConvertTo-Csv -NoTypeInformation | Select-Object -Skip 1 | Out-File -FilePath $CsvPath -Encoding UTF8 -Force + } + + $result = [PSCustomObject]@{ + Success = $true + Errors = @{} + } + $result | ConvertTo-Json -Depth 4 +} catch { + $result = [PSCustomObject]@{ + Success = $false + Errors = @{ "TransportRules" = $_.Exception.Message } + } + $result | ConvertTo-Json -Depth 4 +} finally { + Disconnect-ExchangeOnline -Confirm:$false -ErrorAction SilentlyContinue +} diff --git a/core/powershell/transport_rules.py b/core/powershell/transport_rules.py new file mode 100644 index 00000000..4b790cde --- /dev/null +++ b/core/powershell/transport_rules.py @@ -0,0 +1,43 @@ +import os +import json +import logging +from core.powershell.client import PowerShellClient + +logger = logging.getLogger("PowerShell.TransportRules") + +class TransportRulesFetcher: + def __init__(self, tenant_id: str, client_id: str, client_secret: str): + self.tenant_id = tenant_id + self.client_id = client_id + self.client_secret = client_secret + self.runner = PowerShellClient(tenant_id, client_id, client_secret) + self.script_path = "scripts/get_transport_rules.ps1" + + def fetch_rules(self, csv_path: str) -> dict: + """ + Executes the get_transport_rules.ps1 script, exporting data to csv_path. + Returns a dict: {"success": bool, "errors": dict} + """ + args = [ + "-AppId", self.client_id, + "-Organization", self.tenant_id, + "-ClientSecret", self.client_secret, + "-CsvPath", csv_path + ] + + try: + logger.info("Executing Exchange Transport Rules PowerShell script...") + stdout = self.runner.execute_script(self.script_path, args) + + if not stdout.strip(): + return {"success": False, "errors": {"ParseError": "Empty response from script"}} + + data = json.loads(stdout) + return { + "success": data.get("Success", False), + "errors": data.get("Errors", {}) + } + + except Exception as e: + logger.error(f"Failed to fetch transport rules: {e}", exc_info=True) + return {"success": False, "errors": {"ExecutionError": str(e)}} diff --git a/deal_assistant.py b/deal_assistant.py new file mode 100644 index 00000000..574f74a9 --- /dev/null +++ b/deal_assistant.py @@ -0,0 +1,1493 @@ +# Copyright 2026 Google LLC +"""Standalone application for the License Usage and Telemetry view.""" + +import os +import pandas as pd +import customtkinter as ctk + +# Performance optimizations for CustomTkinter across OS +ctk.set_window_scaling(1.0) +ctk.set_widget_scaling(1.0) + +from telemetry.m365_telemetry import M365TelemetryTab, async_logger +import logging +from telemetry.power_automate import PowerAutomateScanner +from telemetry.styles import * + +import queue +import threading +import tkinter as tk +from tkinter import filedialog, messagebox +import ui.exchange_online_ui +import ui.chats_ui +import ui.files_ui + +# Create custom helper base class to embed CTk windows as CTkFrames +class EmbeddedCTkFrameHelper(ctk.CTkFrame): + _current_master = None + + def __init__(self, master=None, *args, **kwargs): + actual_master = getattr(EmbeddedCTkFrameHelper, "_current_master", master) + super().__init__(actual_master, *args, **kwargs) + + def title(self, *args, **kwargs): + pass + + def geometry(self, *args, **kwargs): + pass + + def protocol(self, *args, **kwargs): + pass + + def attributes(self, *args, **kwargs): + pass + + +# Patch estimator tool base classes in-memory +ui.exchange_online_ui.MigrationEstimatorTool.__bases__ = (EmbeddedCTkFrameHelper,) +ui.chats_ui.ChatMigrationEstimatorTool.__bases__ = (EmbeddedCTkFrameHelper,) + + +class EmbeddedExchangeOnlineTool(ui.exchange_online_ui.MigrationEstimatorTool): + def __init__(self, master, controller, *args, **kwargs): + EmbeddedCTkFrameHelper._current_master = master + self.controller = controller + super().__init__() + + # Populate credentials + self.tenant_id.set(self.controller.stored_tenant) + self.client_ids.set(self.controller.stored_client) + self.client_secrets.set(self.controller.stored_secret) + + def create_entry(self, parent, label, var, show=None): + if label in ["Tenant ID", "Client ID", "Client Secret"]: + if parent.winfo_exists(): + parent.pack_forget() + if parent.master and parent.master.winfo_exists(): + parent.master.pack_forget() + return + super().create_entry(parent, label, var, show) + + def go_back_to_selector(self): + if hasattr(self, "_back_callback") and self._back_callback: + self._back_callback() + + +class EmbeddedChatTool(ui.chats_ui.ChatMigrationEstimatorTool): + def __init__(self, master, controller, *args, **kwargs): + EmbeddedCTkFrameHelper._current_master = master + self.controller = controller + super().__init__() + + # Populate credentials + self.tenant_id.set(self.controller.stored_tenant) + self.client_ids.set(self.controller.stored_client) + self.client_secrets.set(self.controller.stored_secret) + + def create_entry(self, parent, label, var, show=None): + if label in ["Tenant ID", "Client ID", "Client Secret"]: + if parent.winfo_exists(): + parent.pack_forget() + if parent.master and parent.master.winfo_exists(): + parent.master.pack_forget() + return + super().create_entry(parent, label, var, show) + + def go_back_to_selector(self): + if hasattr(self, "_back_callback") and self._back_callback: + self._back_callback() + + +class EmbeddedFilesTool(ui.files_ui.FileMigrationEstimatorTool): + def __init__(self, master, controller, *args, **kwargs): + EmbeddedCTkFrameHelper._current_master = master + self.controller = controller + super().__init__() + + # Populate credentials + self.tenant_id.set(self.controller.stored_tenant) + self.client_ids.set(self.controller.stored_client) + self.client_secrets.set(self.controller.stored_secret) + + def create_entry(self, parent, label, var, show=None): + if label in ["Tenant ID", "Client ID", "Client Secret"]: + if parent.winfo_exists(): + parent.pack_forget() + if parent.master and parent.master.winfo_exists(): + parent.master.pack_forget() + return + super().create_entry(parent, label, var, show) + + def go_back_to_selector(self): + if hasattr(self, "_back_callback") and self._back_callback: + self._back_callback() + + +class MigrationPlannerView(ctk.CTkFrame): + """Container for the Migration Planner workload selector and tool views.""" + + def __init__(self, master, controller, **kwargs): + super().__init__(master, fg_color="transparent", **kwargs) + self.controller = controller + + # Workload selector frame + self.selector_frame = ctk.CTkFrame(self, fg_color="transparent") + self.selector_frame.pack(fill="both", expand=True) + + self.setup_selector_ui() + + # Active tool frame container + self.active_tool_frame = None + + def setup_selector_ui(self): + # Title of selector + ctk.CTkLabel( + self.selector_frame, + text="Select Workload to Estimate", + font=FONT_HEADER_MEDIUM, + text_color=COLOR_TEXT_MAIN, + ).pack(pady=(40, 10), anchor="w", padx=40) + + ctk.CTkLabel( + self.selector_frame, + text="Plan your migration by estimating data size and timelines for your workloads.", + font=FONT_BODY_MEDIUM, + text_color=COLOR_TEXT_SUB, + ).pack(pady=(0, 30), anchor="w", padx=40) + + # Card container grid + self.cards_container = ctk.CTkFrame(self.selector_frame, fg_color="transparent") + self.cards_container.pack(fill="x", padx=40) + + self.cards_container.grid_columnconfigure((0, 1, 2), weight=1, uniform="equal") + + # Card 1: Exchange + self.create_workload_card( + parent=self.cards_container, + column=0, + icon="📩", + title="Exchange Online", + desc="Estimate migration time and resource sizes for mailboxes, calendars, and contacts.", + callback=lambda: self.launch_tool("Exchange") + ) + + # Card 2: Chat + self.create_workload_card( + parent=self.cards_container, + column=1, + icon="💬", + title="Chat (Teams)", + desc="Plan Teams private chats, group channels, and message history migration.", + callback=lambda: self.launch_tool("Chat") + ) + + # Card 3: Files + self.create_workload_card( + parent=self.cards_container, + column=2, + icon="📁", + title="Files (SharePoint/OneDrive)", + desc="Analyze OneDrive personal sites and SharePoint team site collections.", + callback=lambda: self.launch_tool("Files") + ) + + def create_workload_card(self, parent, column, icon, title, desc, callback): + card = ctk.CTkFrame( + parent, + fg_color=COLOR_SURFACE, + corner_radius=12, + border_width=1, + border_color=COLOR_OUTLINE_LIGHT + ) + card.grid(row=0, column=column, padx=10, pady=10, sticky="nsew") + + # Icon + ctk.CTkLabel( + card, + text=icon, + font=ctk.CTkFont(size=36), + text_color=COLOR_PRIMARY + ).pack(pady=(25, 10)) + + # Title + ctk.CTkLabel( + card, + text=title, + font=FONT_BODY_BOLD, + text_color=COLOR_TEXT_MAIN + ).pack(pady=(0, 10)) + + # Description + ctk.CTkLabel( + card, + text=desc, + font=FONT_BODY_MEDIUM, + text_color=COLOR_TEXT_SUB, + wraplength=220, + justify="center" + ).pack(fill="both", expand=True, padx=20, pady=(0, 20)) + + # Launch Button + btn = ctk.CTkButton( + card, + text="Open Planner", + command=callback, + height=36, + corner_radius=8, + fg_color=COLOR_PRIMARY, + hover_color=COLOR_PRIMARY_HOVER, + font=FONT_BODY_BOLD + ) + btn.pack(pady=(0, 25), padx=20, fill="x") + + def launch_tool(self, workload): + # Hide selector + self.selector_frame.pack_forget() + + # Clean old tool if any + if self.active_tool_frame: + self.active_tool_frame.destroy() + self.active_tool_frame = None + + # Instantiate the embedded tool + if workload == "Exchange": + self.active_tool_frame = EmbeddedExchangeOnlineTool(self, self.controller) + elif workload == "Chat": + self.active_tool_frame = EmbeddedChatTool(self, self.controller) + elif workload == "Files": + self.active_tool_frame = EmbeddedFilesTool(self, self.controller) + + if self.active_tool_frame: + self.active_tool_frame._back_callback = self.show_selector + self.active_tool_frame.pack(fill="both", expand=True) + + def show_selector(self): + if self.active_tool_frame: + self.active_tool_frame.pack_forget() + self.active_tool_frame.destroy() + self.active_tool_frame = None + self.selector_frame.pack(fill="both", expand=True) + + + + + + +# Orchestrator logging +logger = logging.getLogger("M365TelemetryAsyncLogger.TelemetryOrchestrator") + + +class CertDecryptionErrorDialog(ctk.CTkToplevel): + """Custom Modal Dialog giving the user choices when local certificate decryption fails.""" + + def __init__(self, parent, error_message): + super().__init__(parent) + self.title("Certificate Decryption Error") + self.geometry("500x260") + self.resizable(False, False) + self.transient(parent) + self.grab_set() + + # Center relative to parent window + parent_x = parent.winfo_rootx() + parent_y = parent.winfo_rooty() + parent_w = parent.winfo_width() + parent_h = parent.winfo_height() + x = parent_x + (parent_w - 500) // 2 + y = parent_y + (parent_h - 260) // 2 + self.geometry(f"+{x}+{y}") + + self.result = None # "retry", "generate", or None + + self.configure(fg_color=COLOR_SURFACE) + + pad_frame = ctk.CTkFrame(self, fg_color="transparent") + pad_frame.pack(fill="both", expand=True, padx=24, pady=24) + + lbl_msg = ctk.CTkLabel( + pad_frame, + text="Unable to decrypt existing certificate passkey using the provided Client Secret. How would you like to proceed?", + font=FONT_BODY_MEDIUM, + text_color=COLOR_TEXT_MAIN, + wraplength=450, + justify="left", + anchor="w" + ) + lbl_msg.pack(anchor="w", pady=(0, 10)) + + lbl_detail = ctk.CTkLabel( + pad_frame, + text=f"Error details: {error_message}", + font=FONT_BODY_SMALL, + text_color=COLOR_ERROR, + wraplength=450, + justify="left", + anchor="w" + ) + lbl_detail.pack(anchor="w", pady=(0, 24)) + + btn_frame = ctk.CTkFrame(pad_frame, fg_color="transparent") + btn_frame.pack(fill="x", side="bottom") + + self.btn_retry = ctk.CTkButton( + btn_frame, + text="Retry with existing secret", + font=FONT_BODY_BOLD, + width=180, + height=36, + fg_color="transparent", + border_width=1, + border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, + hover_color=COLOR_SECONDARY_HOVER, + command=self._on_retry + ) + self.btn_retry.pack(side="left") + + self.btn_generate = ctk.CTkButton( + btn_frame, + text="Generate new certificate", + font=FONT_BODY_BOLD, + width=200, + height=36, + fg_color=COLOR_PRIMARY, + text_color="white", + hover_color=COLOR_PRIMARY_HOVER, + command=self._on_generate + ) + self.btn_generate.pack(side="right") + + self.protocol("WM_DELETE_WINDOW", self._on_close) + + def _on_retry(self): + self.result = "retry" + self.grab_release() + self.destroy() + + def _on_generate(self): + self.result = "generate" + self.grab_release() + self.destroy() + + def _on_close(self): + self.result = None + self.grab_release() + self.destroy() + + + +class TelemetryApp(ctk.CTk): + """Standalone application for the License Usage and Telemetry view.""" + + def __init__(self): + super().__init__() + logger.info("Initializing TelemetryApp application...") + self.title("Deal Assistant") # CITATION: self.title("Deal Assistant") + self.geometry("1230x950") # Expanded window width to support increased sidebar dimensions + + # Maximize window cross-platform + try: + self.state('zoomed') + except Exception: + try: + self.attributes('-zoomed', True) + except Exception: + pass + + # FIX: Bind the window close button to a custom exit handler + # to prevent CustomTkinter 'after script' errors when closing. + self.protocol("WM_DELETE_WINDOW", self.on_closing) # CITATION: self.protocol("WM_DELETE_WINDOW", self.on_closing) + + # Initialize variables required by the M365TelemetryTab + self.retries = ctk.IntVar(value=30) # CITATION: self.retries = ctk.IntVar(value=30) + self.backoff = ctk.IntVar(value=2) # CITATION: self.backoff = ctk.IntVar(value=2) + + # Stage 1: In-memory variables to store connection credentials + self.stored_tenant = "" + self.stored_client = "" + self.stored_secret = "" + self.stored_use_delegated = False + + # Page containers + self.auth_frame = ctk.CTkFrame(self, fg_color="transparent") + self.report_frame = ctk.CTkFrame(self, fg_color="transparent") + + # Render Page 1 (Authentication screen) + self.setup_auth_ui() + + # Render Page 2 (Reports Dashboard with Sidebar) + self.setup_report_ui() + + # Initial view + self.show_auth_page() + logger.info("TelemetryApp UI initialized successfully.") + + def setup_auth_ui(self): + """Builds a modern, polished Connection interface for Page 1.""" + # Welcome Branding Card + self.brand_card = ctk.CTkFrame( + self.auth_frame, + fg_color=COLOR_SURFACE, + corner_radius=12, + border_width=1, + border_color=COLOR_OUTLINE_LIGHT + ) + self.brand_card.pack(fill="x", padx=40, pady=(60, 20)) + + self.brand_title = ctk.CTkLabel( + self.brand_card, + text="Deal Assistant", + font=FONT_HEADER_MEDIUM, + text_color=COLOR_PRIMARY + ) + self.brand_title.pack(pady=(25, 5)) + + self.brand_subtitle = ctk.CTkLabel( + self.brand_card, + text="Connect your Azure App Credentials to begin auditing your tenant.", + font=FONT_BODY_MEDIUM, + text_color=COLOR_TEXT_SUB + ) + self.brand_subtitle.pack(pady=(0, 25)) + + # Credentials Form Box + self.credentials_card = ctk.CTkFrame( + self.auth_frame, + fg_color=COLOR_SURFACE, + corner_radius=12, + border_width=1, + border_color=COLOR_OUTLINE_LIGHT + ) + self.credentials_card.pack(fill="both", expand=True, padx=40, pady=(0, 40)) + + self.form_container = ctk.CTkFrame(self.credentials_card, fg_color="transparent") + self.form_container.pack(pady=40, anchor="center") + + # Tenant ID Input + self.tenant_lbl = ctk.CTkLabel(self.form_container, text="Tenant ID", font=FONT_BODY_BOLD, text_color=COLOR_TEXT_MAIN) + self.tenant_lbl.pack(anchor="w", pady=(10, 5)) + import tkinter as tk + + # Tenant ID Input Wrapper Frame (using native tk.Frame to ensure full X11 border rendering) + self.tenant_border = tk.Frame( + self.form_container, width=850, height=42, + highlightbackground=COLOR_OUTLINE, highlightcolor=COLOR_PRIMARY, highlightthickness=1, + bd=0, background=COLOR_SURFACE + ) + self.tenant_border.pack(pady=(0, 15)) + self.tenant_border.pack_propagate(False) + + self.tenant_entry = ctk.CTkEntry( + self.tenant_border, border_width=0, fg_color="transparent", text_color=COLOR_TEXT_MAIN, + placeholder_text="Enter Tenant ID" + ) + self.tenant_entry.pack(fill="both", expand=True, padx=10, pady=2) + + # Client ID Input + self.client_lbl = ctk.CTkLabel(self.form_container, text="Client ID", font=FONT_BODY_BOLD, text_color=COLOR_TEXT_MAIN) + self.client_lbl.pack(anchor="w", pady=(10, 5)) + + self.client_border = tk.Frame( + self.form_container, width=850, height=42, + highlightbackground=COLOR_OUTLINE, highlightcolor=COLOR_PRIMARY, highlightthickness=1, + bd=0, background=COLOR_SURFACE + ) + self.client_border.pack(pady=(0, 15)) + self.client_border.pack_propagate(False) + + self.client_entry = ctk.CTkEntry( + self.client_border, border_width=0, fg_color="transparent", text_color=COLOR_TEXT_MAIN, + placeholder_text="Enter Client ID" + ) + self.client_entry.pack(fill="both", expand=True, padx=10, pady=2) + + # Client Secret Input + self.secret_lbl = ctk.CTkLabel(self.form_container, text="Client Secret", font=FONT_BODY_BOLD, text_color=COLOR_TEXT_MAIN) + self.secret_lbl.pack(anchor="w", pady=(10, 5)) + + self.secret_border = tk.Frame( + self.form_container, width=850, height=42, + highlightbackground=COLOR_OUTLINE, highlightcolor=COLOR_PRIMARY, highlightthickness=1, + bd=0, background=COLOR_SURFACE + ) + self.secret_border.pack(pady=(0, 30)) + self.secret_border.pack_propagate(False) + + self.secret_entry = ctk.CTkEntry( + self.secret_border, show="*", border_width=0, fg_color="transparent", text_color=COLOR_TEXT_MAIN, + placeholder_text="Enter Client Secret" + ) + self.secret_entry.pack(fill="both", expand=True, padx=10, pady=2) + + # Delegated Auth Checkbox + self.use_delegated_var = ctk.BooleanVar(value=False) + self.delegated_checkbox = ctk.CTkCheckBox( + self.form_container, + text="Enable Delegated Authentication (Required for eDiscovery & MDM Policies)", + variable=self.use_delegated_var, + font=FONT_BODY_MEDIUM, + text_color=COLOR_TEXT_MAIN, + fg_color=COLOR_PRIMARY, + hover_color=COLOR_PRIMARY_HOVER, + command=self._on_delegated_toggled + ) + self.delegated_checkbox.pack(anchor="w", pady=(0, 5)) + + self.delegated_warning_lbl = ctk.CTkLabel( + self.form_container, + text="⚠️ Requires 'Allow public client flows' to be Yes and 'http://localhost' Redirect URI in Azure App Reg.", + font=FONT_BODY_SMALL, + text_color=COLOR_TEXT_SUB, + wraplength=800, + justify="left" + ) + # Initially hidden until checked + # self.delegated_warning_lbl.pack(anchor="w", pady=(0, 20)) + + # Status & Feedback Display + self.auth_status_lbl = ctk.CTkLabel( + self.form_container, + text="", + font=FONT_BODY_MEDIUM, + text_color=COLOR_ERROR, + wraplength=450, + justify="center" + ) + self.auth_status_lbl.pack(pady=(0, 15)) + + # Action Submission Trigger + self.connect_btn = ctk.CTkButton( + self.form_container, + text="Connect & Continue", + command=self.on_connect_clicked, + height=44, + corner_radius=8, + fg_color=COLOR_PRIMARY, + hover_color=COLOR_PRIMARY_HOVER, + font=FONT_BODY_BOLD + ) + self.connect_btn.pack(fill="x", pady=(15, 10)) + + def _on_delegated_toggled(self): + if self.use_delegated_var.get(): + self.delegated_warning_lbl.pack(anchor="w", pady=(0, 20)) + else: + self.delegated_warning_lbl.pack_forget() + + def setup_report_ui(self): + """Instantiates the ReportsPage frame which contains the collapsible navigation structure.""" + self.reports_page = ReportsPage( + master=self.report_frame, + controller=self, + retries_var=self.retries, + backoff_var=self.backoff + ) + self.reports_page.pack(fill="both", expand=True) + + def on_connect_clicked(self): + """Validates inputs, caches credentials in-memory, checks certificate status, and transitions/generates cert.""" + tenant = self.tenant_entry.get().strip() + client = self.client_entry.get().strip() + secret = self.secret_entry.get().strip() + + logger.info("Connect & Continue clicked. Verifying connection credentials...") + + use_delegated = getattr(self, 'use_delegated_var', None) and self.use_delegated_var.get() + + if not tenant or not client or not secret: + logger.warning("Connection failed: Missing one or more required credential fields.") + self.auth_status_lbl.configure(text="Error: Tenant ID, Client ID, and Client Secret are required.", text_color="red") + return + + self.auth_status_lbl.configure(text="⏳ Authenticating in browser...", text_color=COLOR_TEXT_MAIN) + self._cancel_auth_flag = False + + # Turn the connect button into a Cancel button + original_command = self.connect_btn.cget("command") + self.connect_btn.configure( + text="Cancel Authentication", + fg_color=COLOR_ERROR, + hover_color="#b91c1c", + command=lambda: self._cancel_auth(original_command) + ) + self.update_idletasks() + + def _reset_btn(): + self.connect_btn.configure( + text="Connect & Continue", + fg_color=COLOR_PRIMARY, + hover_color=COLOR_PRIMARY_HOVER, + command=original_command + ) + + def _on_success(): + if getattr(self, '_cancel_auth_flag', False): + return + _reset_btn() + self._complete_connection(tenant, client, secret, use_delegated) + + if use_delegated: + import threading + def _auth_worker(): + try: + from core.graph.delegated_auth import DelegatedAuthClient + auth_client = DelegatedAuthClient(tenant, client, secret) + # Use a shorter timeout to prevent hanging forever + token = auth_client.get_token(scopes=["https://graph.microsoft.com/.default"], force_interactive=True) + + if getattr(self, '_cancel_auth_flag', False): + return + + if not token: + self.after(0, lambda: self.auth_status_lbl.configure(text="Error: Failed to authenticate via browser popup.", text_color="red")) + self.after(0, _reset_btn) + return + self.after(0, _on_success) + except Exception as e: + if getattr(self, '_cancel_auth_flag', False): + return + self.after(0, lambda err=str(e): self.auth_status_lbl.configure(text=f"Error during delegated auth: {err}", text_color="red")) + self.after(0, _reset_btn) + + threading.Thread(target=_auth_worker, daemon=True).start() + return + + _on_success() + + def _cancel_auth(self, original_command): + """Allows the user to abort if the browser flow is stuck or throws a Microsoft error.""" + self._cancel_auth_flag = True + self.auth_status_lbl.configure(text="Authentication cancelled.", text_color=COLOR_TEXT_SUB) + self.connect_btn.configure( + text="Connect & Continue", + fg_color=COLOR_PRIMARY, + hover_color=COLOR_PRIMARY_HOVER, + command=original_command + ) + + def _complete_connection(self, tenant, client, secret, use_delegated): + self.auth_status_lbl.configure(text="") + + # Cache connection details safely in memory + self.stored_tenant = tenant + self.stored_client = client + self.stored_secret = secret + self.stored_use_delegated = use_delegated + + logger.info("Credentials validated and cached in memory. Updating log directories...") + + # Update log directory to use sub-folder based on tenant and client + from telemetry.m365_telemetry import update_log_directory as update_license_log_dir + from core.cert_auth import update_log_directory as update_cert_log_dir + + update_license_log_dir(tenant, client) + update_cert_log_dir(tenant, client) + + from core.cert_auth import check_certificate_exists, generate_certificate, load_certificate + + if check_certificate_exists(tenant_id=tenant, client_id=client): + try: + # Decrypt the PFX certificate using the client secret + load_certificate(secret, tenant_id=tenant, client_id=client) + self.show_reports_page() + except Exception as e: + logger.error(f"Certificate decryption/load failed: {e}", exc_info=True) + # Invoke the custom modal selection dialog + dialog = CertDecryptionErrorDialog(self, str(e)) + self.wait_window(dialog) + + if dialog.result == "retry": + logger.info("Option 1 chosen: Retry connection with correct client secret.") + return + elif dialog.result == "generate": + logger.info("Option 2 chosen: Overwrite and generate a new certificate.") + try: + pem_path, _ = generate_certificate(secret, tenant_id=tenant, client_id=client) + self.setup_cert_instructions_ui(pem_path) + except Exception as gen_err: + logger.error(f"Certificate generation failed: {gen_err}", exc_info=True) + from tkinter import messagebox + messagebox.showerror( + "Certificate Generation Error", + f"Unable to generate certificate: {gen_err}", + parent=self + ) + else: + logger.info("Decryption modal closed without option selection.") + return + else: + try: + # Generate new certificate and pfx encrypted with the client secret + pem_path, _ = generate_certificate(secret, tenant_id=tenant, client_id=client) + # Setup instructions UI requesting the user to upload it to Entra + self.setup_cert_instructions_ui(pem_path) + except Exception as e: + logger.error(f"Certificate generation failed: {e}", exc_info=True) + from tkinter import messagebox + messagebox.showerror( + "Certificate Generation Error", + f"Unable to generate certificate. Proceeding with standard Client Secret authentication fallback.\n\nError: {e}", + parent=self + ) + self.show_reports_page() + + def setup_cert_instructions_ui(self, pem_path): + """Displays certificate upload instructions screen when a new certificate is generated.""" + self.form_container.pack_forget() + + if hasattr(self, "cert_container") and self.cert_container: + self.cert_container.destroy() + + self.cert_container = ctk.CTkFrame(self.credentials_card, fg_color="transparent") + self.cert_container.pack(pady=30, padx=50, fill="both", expand=True) + + ctk.CTkLabel( + self.cert_container, + text="Certificate Upload", + font=FONT_HEADER_MEDIUM, + text_color=COLOR_PRIMARY + ).pack(anchor="w", pady=(0, 15)) + + intro_text = ( + "A new security certificate has been generated for hybrid authentication.\n\n" + "Uploading this certificate is highly recommended, but optional:" + ) + self.cert_intro_lbl = ctk.CTkLabel( + self.cert_container, + text=intro_text, + font=FONT_BODY_MEDIUM, + text_color=COLOR_TEXT_MAIN, + justify="left", + wraplength=1000 + ) + self.cert_intro_lbl.pack(anchor="w", pady=(0, 10)) + + upload_statement = ( + "• If you UPLOAD the certificate:\n" + " All report sections will be fully functional." + ) + self.cert_upload_lbl = ctk.CTkLabel( + self.cert_container, + text=upload_statement, + font=FONT_BODY_BOLD, + text_color=COLOR_SUCCESS, + justify="left", + wraplength=1000 + ) + self.cert_upload_lbl.pack(anchor="w", pady=(0, 10)) + + skip_statement = ( + "• If you SKIP uploading the certificate:\n" + " You can still run the reports. However, sections relying on certificate-based authentication (such as detailed Calendar settings, " + "Shared/Public mailbox statistics, Retention Policies etc.) will be skipped and show as unavailable." + ) + self.cert_skip_lbl = ctk.CTkLabel( + self.cert_container, + text=skip_statement, + font=FONT_BODY_BOLD, + text_color=COLOR_ERROR, + justify="left", + wraplength=1000 + ) + self.cert_skip_lbl.pack(anchor="w", pady=(0, 10)) + + # Prominent Configuration Callout Card + self.cert_info_card = ctk.CTkFrame( + self.cert_container, + fg_color=COLOR_TONAL_BG, + border_width=1, + border_color=COLOR_OUTLINE_LIGHT, + corner_radius=8 + ) + self.cert_info_card.pack(fill="x", pady=(15, 25)) + + # Title of info card + self.cert_card_title = ctk.CTkLabel( + self.cert_info_card, + text="Upload Instructions", + font=FONT_BODY_BOLD, + text_color=COLOR_TONAL_TEXT, + justify="left" + ) + self.cert_card_title.pack(anchor="w", padx=15, pady=(15, 5)) + + # Instructions body + self.cert_footer_lbl = ctk.CTkLabel( + self.cert_info_card, + text=f"1. Locate the certificate file generated at:\n {pem_path}\n\n2. Log in to the Microsoft Azure portal and navigate to the App Registration with Client ID:\n {self.stored_client}\n\n3. Upload the certificate under:\n Certificates & secrets -> Certificates -> Upload certificate", + font=FONT_BODY_MEDIUM, + text_color=COLOR_TEXT_MAIN, + justify="left", + wraplength=970 + ) + self.cert_footer_lbl.pack(anchor="w", padx=15, pady=(0, 15)) + + self.cert_continue_btn = ctk.CTkButton( + self.cert_container, + text="Continue", + command=self.on_cert_continue_clicked, + height=40, + corner_radius=20, + font=FONT_BODY_BOLD, + fg_color=COLOR_PRIMARY, + hover_color=COLOR_PRIMARY_HOVER + ) + self.cert_continue_btn.pack(fill="x") + + # Bind resize event to dynamically adjust text wrapping based on container width + self.cert_container.bind( + "", + lambda event: self.adjust_cert_wraplengths( + event.width - 20 + ) + ) + + def adjust_cert_wraplengths(self, width): + w = max(200, width) + if hasattr(self, "cert_intro_lbl") and self.cert_intro_lbl.winfo_exists(): + self.cert_intro_lbl.configure(wraplength=w) + if hasattr(self, "cert_upload_lbl") and self.cert_upload_lbl.winfo_exists(): + self.cert_upload_lbl.configure(wraplength=w) + if hasattr(self, "cert_skip_lbl") and self.cert_skip_lbl.winfo_exists(): + self.cert_skip_lbl.configure(wraplength=w) + if hasattr(self, "cert_card_title") and self.cert_card_title.winfo_exists(): + self.cert_card_title.configure(wraplength=w - 30) + if hasattr(self, "cert_footer_lbl") and self.cert_footer_lbl.winfo_exists(): + self.cert_footer_lbl.configure(wraplength=w - 30) + + def on_cert_continue_clicked(self): + """Validates certificate after user claims to have uploaded it and transitions to reports.""" + from core.cert_auth import load_certificate + try: + # Verify we can unlock/read the cert successfully + load_certificate(self.stored_secret, tenant_id=self.stored_tenant, client_id=self.stored_client) + except Exception as e: + logger.error(f"Certificate validation failed: {e}", exc_info=True) + from tkinter import messagebox + messagebox.showerror( + "Certificate Verification Error", + f"Unable to verify certificate. Proceeding with standard Client Secret authentication fallback.\n\nError: {e}", + parent=self + ) + self.show_reports_page() + + # Reset UI form in case of subsequent logins + if hasattr(self, "cert_container") and self.cert_container: + self.cert_container.pack_forget() + self.form_container.pack(pady=40, anchor="center") + + def on_disconnect_clicked(self): + """Clears all session properties and safely resets screens back to Page 1.""" + logger.info("Disconnect clicked. Clearing cached session and credentials...") + # Wipes local entry buffers + self.tenant_entry.delete(0, "end") + self.client_entry.delete(0, "end") + self.secret_entry.delete(0, "end") + self.auth_status_lbl.configure(text="") + + # Wipes stored in-memory configurations + self.stored_tenant = "" + self.stored_client = "" + self.stored_secret = "" + + # Clears variables inside nested dashboards safely + try: + self.reports_page.clear_session_data() + except Exception as e: + logger.error(f"Error during clear_session_data: {e}", exc_info=True) + + # Revert log directories to default safely + from telemetry.m365_telemetry import update_log_directory as update_license_log_dir + from core.cert_auth import update_log_directory as update_cert_log_dir + + try: + update_license_log_dir() + update_cert_log_dir() + except Exception as e: + logger.error(f"Error resetting log directories: {e}", exc_info=True) + + # Clean up cert screen UI and restore normal entry layout + if hasattr(self, "cert_container") and self.cert_container: + try: + self.cert_container.pack_forget() + except Exception: + pass + self.form_container.pack(pady=40, anchor="center") + + # Shifts screen orientation + self.show_auth_page() + logger.info("Session successfully disconnected. Returned to Auth page.") + + def show_auth_page(self): + """Transitions view port to Page 1 (Authentication screen).""" + logger.info("Showing Authentication Page.") + self.report_frame.pack_forget() + self.auth_frame.pack(fill="both", expand=True) + + def show_reports_page(self): + """Transitions view port to Page 2 (Reports Dashboard).""" + logger.info("Showing Reports Dashboard Page.") + self.auth_frame.pack_forget() + self.report_frame.pack(fill="both", expand=True) + + # Force Tkinter to finish all geometry calculations and draw operations synchronously + # This prevents the macOS window manager from capturing a "half-drawn" frame with only the sidebar. + self.update_idletasks() + + def log_msg(self, text): # CITATION: def log_msg(self, text): + """Simple callback handler for telemetry UI logs. Pipes to log file instead of stdout.""" + async_logger.info(text) # CITATION: async_logger.info(text) + + def on_closing(self): # CITATION: def on_closing(self): + """Trigger an OS-level exit to cleanly bypass Tkinter background tasks.""" + self.destroy() # CITATION: self.destroy() + os._exit(0) # CITATION: os._exit(0) + + +class ReportsPage(ctk.CTkFrame): + """Page 2 Content Host. Organizes the Left Collapsible Panel and the Main Data Panel side-by-side.""" + + def __init__(self, master, controller, retries_var, backoff_var): + super().__init__(master, fg_color="transparent") + self.controller = controller + + # 1. Left Collapsible Navigation Sidebar + self.sidebar = SidebarFrame( + self, + disconnect_callback=self.controller.on_disconnect_clicked, + selection_callback=self.on_sidebar_selection_changed + ) + self.sidebar.pack(side="left", fill="y", padx=(0, 10)) + + # 2. Right-hand Main Dashboard Container + self.dashboard_container = ctk.CTkFrame(self, fg_color="transparent") + self.dashboard_container.pack(side="right", fill="both", expand=True) + + # Top Header Bar mimicking Google Workspace Deal Assistant details + self.nav_header = ctk.CTkFrame( + self.dashboard_container, + fg_color=COLOR_SURFACE, + corner_radius=12, + border_width=1, + border_color=COLOR_OUTLINE_LIGHT + ) + self.nav_header.pack(fill="x", pady=(0, 15)) + + # Text container frame to keep alignment clean next to the Action Button + self.header_text_frame = ctk.CTkFrame(self.nav_header, fg_color="transparent") + self.header_text_frame.pack(side="left", padx=20, pady=17) + + self.nav_title = ctk.CTkLabel( + self.header_text_frame, + text="Usage Report", + font=ctk.CTkFont(family="Segoe UI", size=20, weight="bold"), + text_color=COLOR_TEXT_MAIN + ) + self.nav_title.pack(anchor="w") + + # 3. Fetch Report button (Far Right) + self.fetch_btn = ctk.CTkButton( + self.nav_header, + text="Fetch Report", + command=self.on_fetch_report_clicked, + width=150, + height=36, + corner_radius=8, + fg_color=COLOR_PRIMARY, + hover_color=COLOR_PRIMARY_HOVER, + font=FONT_BODY_BOLD + ) + self.fetch_btn.pack(side="right", padx=(10, 20), pady=17) + + # 4. Download PDF button (Middle) + self.pdf_btn = ctk.CTkButton( + self.nav_header, + text="Download PDF", + command=self.on_download_pdf_clicked, + width=150, + height=36, + corner_radius=8, + fg_color="transparent", + border_width=1, + border_color=COLOR_PRIMARY, + text_color=COLOR_PRIMARY, + hover_color=COLOR_SECONDARY_HOVER, + font=FONT_BODY_BOLD, + state="disabled" + ) + self.pdf_btn.pack(side="right", padx=(10, 0), pady=17) + + self.pdf_btn.pack(side="right", padx=(10, 0), pady=17) + + + # Initialize the telemetry view (No TabView layout) + self.m365_telemetry_view = M365TelemetryTab( + master=self.dashboard_container, + log_callback=controller.log_msg, + retries_var=retries_var, + backoff_var=backoff_var + ) + self.m365_telemetry_view.pack(fill="both", expand=True) + self.m365_telemetry_view.on_all_done_callback = self.on_telemetry_fetch_completed + + # Initialize the migration planner view (initially hidden) + self.migration_planner_view = MigrationPlannerView( + master=self.dashboard_container, + controller=self.controller + ) + + + + # Adapt layout recursively to hide original inputs from view + self.adapt_embedded_view() + + def on_sidebar_selection_changed(self, label): + import gc + if label == "Usage and adoption": + # Show Telemetry, Hide Migration Planner + self.migration_planner_view.pack_forget() + self.nav_title.configure(text="Usage Report") + self.fetch_btn.pack(side="right", padx=(10, 20), pady=17) + self.pdf_btn.pack(side="right", padx=(10, 0), pady=17) + self.m365_telemetry_view.pack(fill="both", expand=True) + self.after(500, gc.collect) + elif label == "Migration planner": + # Show Migration Planner, Hide Telemetry + self.m365_telemetry_view.pack_forget() + self.fetch_btn.pack_forget() + self.pdf_btn.pack_forget() + self.nav_title.configure(text="Migration Planner") + self.migration_planner_view.pack(fill="both", expand=True) + self.after(500, gc.collect) + + def adapt_embedded_view(self): + """Traverses M365TelemetryTab to identify and hide native login components.""" + self.embedded_entries = [] + self.embedded_submit_btn = None + self.embedded_labels = [] + + def find_widgets_recursive(widget): + if isinstance(widget, ctk.CTkEntry): + self.embedded_entries.append(widget) + elif isinstance(widget, ctk.CTkButton): + btn_txt = str(widget.cget("text")).lower() + if "submit" in btn_txt or btn_txt == "": + self.embedded_submit_btn = widget + elif isinstance(widget, ctk.CTkLabel): + lbl_txt = str(widget.cget("text")).lower() + if any(kw in lbl_txt for kw in ["tenant id", "client id", "client secret", "connect your", "authenticate and audit"]): + self.embedded_labels.append(widget) + + for child in widget.winfo_children(): + find_widgets_recursive(child) + + find_widgets_recursive(self.m365_telemetry_view) + + # Remove the target widgets from layout grids/packs programmatically + for entry in self.embedded_entries: + entry.pack_forget() + entry.grid_forget() + + for label in self.embedded_labels: + label.pack_forget() + label.grid_forget() + + if self.embedded_submit_btn: + self.embedded_submit_btn.pack_forget() + self.embedded_submit_btn.grid_forget() + + if hasattr(self.m365_telemetry_view, "inputs_frame"): + self.m365_telemetry_view.inputs_frame.pack_forget() + self.m365_telemetry_view.inputs_frame.grid_forget() + + def on_fetch_report_clicked(self): + """Stage 2: Migrates stored variables into the telemetry coordinator and triggers fetch directly, or cancels if in progress.""" + if getattr(self.m365_telemetry_view, "is_fetching", False): + self.m365_telemetry_view.cancel_fetching() + return + + tenant = self.controller.stored_tenant + client = self.controller.stored_client + secret = self.controller.stored_secret + + if not tenant or not client or not secret: + logger.warning("Fetch Report triggered, but connection credentials are empty.") + return + + logger.info("Fetch Report triggered. Invoking background parallel audits...") + # Toggle button to Cancel and keep it enabled and active + self.fetch_btn.configure(state="normal", text="Cancel", fg_color="#DC2626") # Red color for cancel + self.pdf_btn.configure(state="disabled") + + # Set variables of m365_telemetry_view directly + self.m365_telemetry_view.lic_tenant_id.set(tenant) + self.m365_telemetry_view.lic_client_ids.set(client) + self.m365_telemetry_view.lic_client_secrets.set(secret) + self.m365_telemetry_view.use_delegated_auth.set(self.controller.stored_use_delegated) + + # Directly call the fetch command + self.m365_telemetry_view.authenticate_licenses_tab() + + def on_telemetry_fetch_completed(self, success: bool): + """Callback from M365TelemetryTab when all parallel reports complete or are cancelled.""" + self.fetch_btn.configure(state="normal", text="Fetch Report", fg_color=COLOR_PRIMARY) + + # Check if any data has been successfully fetched (enables partial downloads post-cancellation or partial failures) + data = self.m365_telemetry_view.get_all_telemetry_data() + directory = data.get("directory") or {} + has_any_data = any([ + data.get("skus"), + directory.get("domains"), + directory.get("group_counts"), + directory.get("user_counts"), + data.get("o365_usage"), + data.get("o365_trend"), + data.get("m365_apps"), + data.get("mailbox"), + data.get("calendar"), + data.get("mail_security"), + data.get("connectors"), + data.get("email_clients"), + data.get("pst_files"), + data.get("sharepoint"), + data.get("onedrive"), + data.get("devices_apps"), + data.get("intune"), + data.get("security_labels"), + data.get("retention_policies"), + data.get("power_automate") + ]) + + if has_any_data: + self.pdf_btn.configure(state="normal") + else: + self.pdf_btn.configure(state="disabled") + + def on_download_pdf_clicked(self): + """Prompts the user to save the M365 usage report as a detailed PDF file.""" + from tkinter import filedialog, messagebox + import datetime + import threading + from telemetry.pdf_report import generate_pdf_report + + ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + f = filedialog.asksaveasfilename( + initialfile=f"m365_usage_report_{ts}.pdf", + defaultextension=".pdf", + filetypes=[("PDF Documents", "*.pdf"), ("All Files", "*.*")], + parent=self + ) + if not f: + return + + data = self.m365_telemetry_view.get_all_telemetry_data() + self.pdf_btn.configure(state="disabled", text="Generating...") + + def _generate_pdf_worker(): + try: + generate_pdf_report(data, f) + self.after(0, lambda: messagebox.showinfo("Export Successful", f"PDF report successfully saved to:\n{f}", parent=self)) + except Exception as e: + logger.error("Failed to generate PDF report", exc_info=True) + self.after(0, lambda err=e: messagebox.showerror("Export Failed", f"Failed to generate PDF report: {err}", parent=self)) + finally: + self.after(0, lambda: self.pdf_btn.configure(state="normal", text="Download PDF")) + + threading.Thread(target=_generate_pdf_worker, daemon=True).start() + + + + def clear_session_data(self): + """Wipes the cached parameters from telemetry objects and resets the Fetch button.""" + logger.info("Clearing session data in ReportsPage.") + for entry in self.embedded_entries: + entry.delete(0, "end") + + # Reset Fetch Report button state + self.fetch_btn.configure(state="normal", text="Fetch Report", fg_color="#1E3A8A") + self.pdf_btn.configure(state="disabled") + + # Reset the telemetry coordinator tab and hide all grids + self.m365_telemetry_view.reset_tab() + + # Reset migration planner view back to selector screen + self.migration_planner_view.show_selector() + + # Reset the sidebar selection state + self.sidebar.reset_selection() + # Switch back UI elements to default Usage report view + self.on_sidebar_selection_changed("Usage and adoption") + + +class SidebarFrame(ctk.CTkFrame): + """Collapsible Left Navigation Sidebar, matching Workspace Deal Assistant styling.""" + + def __init__(self, master, disconnect_callback, selection_callback, **kwargs): + # Increased initial width to 380px to avoid text truncation of longer menu items + super().__init__(master, width=380, fg_color=COLOR_SURFACE, corner_radius=12, border_width=1, border_color=COLOR_OUTLINE_LIGHT, **kwargs) + self.pack_propagate(False) # Lock sidebar panel dimensions + self.disconnect_callback = disconnect_callback + self.selection_callback = selection_callback + self.is_expanded = True + + # Header Branding Section + self.header_frame = ctk.CTkFrame(self, fg_color="transparent") + self.header_frame.pack(fill="x", padx=20, pady=(25, 30)) + + # Brand Icon representing a deal helper (Handshake 🤝) + self.logo_label = ctk.CTkLabel( + self.header_frame, + text="🤝", + font=ctk.CTkFont(family="Segoe UI", size=24), + text_color=COLOR_PRIMARY + ) + self.logo_label.pack(side="left", padx=(5, 5)) + + self.brand_text_area = ctk.CTkFrame(self.header_frame, fg_color="transparent") + self.brand_text_area.pack(side="left", fill="both", expand=True) + + self.brand_title = ctk.CTkLabel( + self.brand_text_area, + text="Deal Assistant", + font=FONT_HEADER_SMALL, + text_color=COLOR_TEXT_MAIN, + anchor="w" + ) + self.brand_title.pack(fill="x", pady=2) + + # Collapse / Expand control button + self.toggle_btn = ctk.CTkButton( + self, + text="⏴", + command=self.toggle_sidebar, + width=26, + height=28, + corner_radius=13, + fg_color=COLOR_SURFACE_VARIANT, + hover_color=COLOR_SECONDARY_HOVER, + text_color=COLOR_TEXT_SUB, + font=FONT_BODY_BOLD + ) + self.toggle_btn.place(relx=1.0, x=-22, y=26, anchor="center") + + # Vertical menu container (increased padding for larger sidebar) + self.menu_items_frame = ctk.CTkFrame(self, fg_color="transparent") + self.menu_items_frame.pack(fill="both", expand=True, padx=20) + + # Definitions for active/inactive routes + self.menu_buttons = [] + self.menu_data = [ + ("Usage and adoption", "📊", True), + ("Migration planner", "🚀", False) + ] + + self.render_navigation_menu() + + # Session closure button pinned safely at bottom + self.disconnect_row = ctk.CTkFrame(self, fg_color="transparent") + self.disconnect_row.pack(fill="x", side="bottom", padx=20, pady=25) + + self.disconnect_symbol = ctk.CTkLabel( + self.disconnect_row, + text="🚪", + font=ctk.CTkFont(family="Segoe UI", size=14), + text_color=COLOR_ERROR + ) + self.disconnect_symbol.pack(side="left", padx=(12, 6)) + + self.disconnect_btn = ctk.CTkButton( + self.disconnect_row, + text="Disconnect", + command=self.disconnect_callback, + anchor="w", + height=38, + corner_radius=8, + fg_color="transparent", + text_color=COLOR_ERROR, + hover_color="#FCE8E6", + font=FONT_BODY_MEDIUM + ) + self.disconnect_btn.pack(side="left", fill="x", expand=True) + + # RAM Usage Display Row (packed second with side="bottom", placing it directly above disconnect_row) + self.ram_row = ctk.CTkFrame(self, fg_color="transparent") + self.ram_row.pack(fill="x", side="bottom", padx=20, pady=(0, 10)) + + self.ram_symbol = ctk.CTkLabel( + self.ram_row, + text="💾", + font=ctk.CTkFont(family="Segoe UI", size=14), + text_color=COLOR_TEXT_SUB + ) + self.ram_symbol.pack(side="left", padx=(12, 6)) + + self.ram_lbl = ctk.CTkLabel( + self.ram_row, + text="RAM: Checking...", + anchor="w", + height=38, + text_color=COLOR_TEXT_SUB, + font=FONT_BODY_MEDIUM + ) + self.ram_lbl.pack(side="left", fill="x", expand=True) + + self._last_ram_log_time = 0 + # Start periodic RAM usage updates + self.update_ram_usage() + + def reset_selection(self): + self.menu_data = [ + ("Usage and adoption", "📊", True), + ("Migration planner", "🚀", False) + ] + self.render_navigation_menu() + + def on_menu_item_clicked(self, label): + for i, (m_label, m_icon, m_active) in enumerate(self.menu_data): + self.menu_data[i] = (m_label, m_icon, m_label == label) + self.render_navigation_menu() + self.selection_callback(label) + + + + def render_navigation_menu(self): + """Builds clean selection widgets mapping Workspace items.""" + for item in self.menu_buttons: + item[0].destroy() + item[1].destroy() + self.menu_buttons.clear() + + for label, icon, is_active in self.menu_data: + row_frame = ctk.CTkFrame(self.menu_items_frame, fg_color="transparent") + row_frame.pack(fill="x", pady=4) + + # Highlighting indicators mapping screenshot + if is_active: + btn_fg = COLOR_TONAL_BG + text_color = COLOR_PRIMARY + hover_color = "#D2E3FC" + weight = "bold" + else: + btn_fg = "transparent" + text_color = COLOR_TEXT_SUB + hover_color = COLOR_SURFACE_VARIANT + weight = "normal" + + icon_lbl = ctk.CTkLabel( + row_frame, + text=icon, + font=ctk.CTkFont(family="Segoe UI", size=15), + text_color=text_color + ) + icon_lbl.pack(side="left", padx=(12, 8)) + + btn = ctk.CTkButton( + row_frame, + text=label if self.is_expanded else "", + anchor="w", + height=38, + corner_radius=8, + fg_color=btn_fg, + text_color=text_color, + hover_color=hover_color, + state="normal", + font=ctk.CTkFont(family="Segoe UI", size=13, weight=weight), + command=lambda l=label: self.on_menu_item_clicked(l) + ) + btn.pack(side="left", fill="x", expand=True) + self.menu_buttons.append((row_frame, btn, icon_lbl)) + + + def toggle_sidebar(self): + """Performs layout adjustments to expand/collapse panel width dynamically.""" + if self.is_expanded: + # Shift width configuration to compact state (72px) + self.configure(width=72) + self.brand_text_area.pack_forget() + self.logo_label.pack(side="top", pady=10) + self.is_expanded = False + self.toggle_btn.configure(text="⏵") + + # Wipe text arrays inside panel items + for row, btn, icon in self.menu_buttons: + btn.configure(text="") + self.disconnect_btn.configure(text="") + self.ram_lbl.configure(text="") + logger.info("Sidebar collapsed.") + else: + # Return layout to expanded parameters (380px) + self.configure(width=380) + self.logo_label.pack(side="left", padx=(5, 5)) + self.brand_text_area.pack(side="left", fill="both", expand=True) + self.is_expanded = True + self.toggle_btn.configure(text="⏴") + + # Restore original strings dynamically + for idx, (row, btn, icon) in enumerate(self.menu_buttons): + btn.configure(text=self.menu_data[idx][0]) + self.disconnect_btn.configure(text="Disconnect") + self.update_ram_label_immediate() + logger.info("Sidebar expanded.") + + def update_ram_label_immediate(self): + """Updates the RAM label text immediately without waiting for the timer.""" + try: + import psutil + process = psutil.Process(os.getpid()) + ram_mb = process.memory_info().rss / (1024 * 1024) + if self.is_expanded: + self.ram_lbl.configure(text=f"RAM: {ram_mb:.1f} MB") + else: + self.ram_lbl.configure(text="") + + # Log RAM usage to log file every 10 seconds + import time + now = time.time() + if now - self._last_ram_log_time >= 10: + logger.info(f"App memory consumption: {ram_mb:.1f} MB") + self._last_ram_log_time = now + except Exception as e: + logger.error(f"Error checking RAM usage: {e}") + if self.is_expanded: + self.ram_lbl.configure(text="RAM: N/A") + else: + self.ram_lbl.configure(text="") + + + def update_ram_usage(self): + """Periodically updates the displayed RAM usage of the current process.""" + self.update_ram_label_immediate() + self._ram_timer_id = self.after(2000, self.update_ram_usage) + + + +def collect_power_automate_telemetry(tenant_id, client_id, client_secret, env_url): # CITATION: def collect_power_automate_telemetry(tenant_id, client_id, client_secret, env_url): + """Integrates the Power Automate scan into the telemetry execution flow.""" + logger.info("--- Power Automate Telemetry Phase Initiated ---") # CITATION: logger.info("--- Power Automate Telemetry Phase Initiated ---") + + if not env_url: # CITATION: if not env_url: + logger.warning("Skipping Power Automate: Environment URL not provided.") # CITATION: logger.warning("Skipping Power Automate: Environment URL not provided.") + return {} + + try: + scanner = PowerAutomateScanner(tenant_id, client_id, client_secret, env_url) # CITATION: scanner = PowerAutomateScanner(tenant_id, client_id, client_secret, env_url) + results = scanner.scan_flows() # CITATION: results = scanner.scan_flows() + + if results: # CITATION: if results: + logger.info(f"Telemetry Success: Aggregated data for {results['total_active_flows']} flows.") # CITATION: logger.info(f"Telemetry Success: Aggregated data for {results['total_active_flows']} flows.") + return results + else: + logger.error("Telemetry Warning: No flow data was returned from the scanner.") # CITATION: logger.error("Telemetry Warning: No flow data was returned from the scanner.") + return {} + + except Exception as e: # CITATION: except Exception as e: + logger.error(f"Critical Error during Power Automate scan: {str(e)}") # CITATION: logger.error(f"Critical Error during Power Automate scan: {str(e)}") + return {} + finally: + logger.info("--- Power Automate Telemetry Phase Concluded ---") # CITATION: logger.info("--- Power Automate Telemetry Phase Concluded ---") + + +if __name__ == "__main__": + ctk.set_appearance_mode("Light") # CITATION: ctk.set_appearance_mode("Light") + app = TelemetryApp() # CITATION: app = TelemetryApp() + app.mainloop() # CITATION: app.mainloop() diff --git a/docs/m365_telemetry_scaling_skill.md b/docs/m365_telemetry_scaling_skill.md new file mode 100644 index 00000000..b313dc42 --- /dev/null +++ b/docs/m365_telemetry_scaling_skill.md @@ -0,0 +1,326 @@ +# M365 Telemetry Module Design & Scaling Guide + +This document defines the architectural patterns, coding standards, and design principles for implementing new sections or sub-sections in the **Deal Assistant** application. Follow these guidelines to ensure consistency, safety, and scalability when handling 100K+ user tenant sizes. + +--- + +## 1. Architectural Strategy + +Every telemetry module must follow a decoupled **Model-View-Controller (MVC)**-style division: +1. **Core Service (Backend)**: Located in `core/graph/` (or `core/powershell/`). Independent of GUI components. Interacts with the API, returns raw objects, or writes raw CSV files. +2. **Database Cache (SQLite)**: Located in `core/graph/db.py`. Stores raw CSV outputs into a local SQLite cache. Used for fast paginated queries on the UI thread. +3. **UI Component (Frontend)**: Located in `telemetry/`. Builds CustomTkinter layouts, triggers the backend pipeline in a background thread, and queries the local SQLite cache to render paginated data. + +```mermaid +graph TD + UI[CustomTkinter UI Frame] -->|1. trigger_fetch| Thread[Background Thread] + Thread -->|2. HTTP Request| Graph[Microsoft Graph API] + Graph -->|3. Streaming Chunks| CSV[Raw CSV on Disk] + Thread -->|4. Async Import| SQLite[SQLite Cache db] + UI -->|5. Paginated Read| SQLite +``` + +--- + +## 2. Backend Service Guidelines + +1. **Decouple API Logic**: Always put API request and data-wrangling code in `core/graph/` or `core/powershell/`. Do not import `customtkinter` or layout modules here. +2. **Streaming Chunk Downloads**: For large CSV reports, stream the download using chunk sizes of `8192` bytes to prevent high memory consumption: + ```python + with requests.get(download_url, stream=True) as response: + response.raise_for_status() + with open(output_path, "wb") as f: + for chunk in response.iter_content(chunk_size=8192): + if chunk: f.write(chunk) + ``` +3. **EXO V3 PowerShell Cmdlets**: If writing PowerShell scripts for Exchange Online, always use REST-based **V3 cmdlets** (`Get-EXOMailbox` / `Get-EXOMailboxStatistics`) rather than legacy cmdlets. Specify only the minimum required properties to prevent WinRM throttling: + ```powershell + Get-EXOMailbox -ResultSize Unlimited -PropertySets Minimum -Properties RecipientTypeDetails + ``` + +--- + +## 3. Database Caching & File Storage Guidelines + +1. **Raw CSV Preservation**: All sections must save raw CSV telemetry data to the designated reports folder. Raw compliance files must follow the path template: + `telemetry/reports/{tenant_id}_{client_id}/{report_name}.csv` +2. **Temporary File Handling**: Write downloads to a `.tmp` file (e.g. `report_name.csv.tmp`) first, and swap it with the production CSV only upon successful completion. Ensure temporary files are deleted in a `finally` block if the execution fails or is cancelled. +3. **Bulk SQLite Imports**: Use `aiosqlite` inside a single transaction to drop and recreate the table, then batch-insert data using `executemany` (e.g., in chunks of 5,000): + ```python + from core.graph.db import import_csv_to_sqlite + + # Run asynchronously inside a background worker + asyncio.run(import_csv_to_sqlite(csv_path, db_path, "table_name")) + ``` +4. **SQLite Column Escaping**: Column headers dynamically parsed from Microsoft Graph may contain reserved SQL keywords (e.g. `Default`, `Group`, `Order`). + - **Rule**: Always wrap column names in square brackets `[column_name]` in SQL query builders to prevent syntax errors: + ```python + cols_def = ", ".join(f"[{h}] TEXT" for h in sanitized_headers) + await db.execute(f"CREATE TABLE {table_name} ({cols_def})") + ``` +5. **Case-Insensitive Database Lookups**: + - The database wrapper (`core/graph/db.py`) automatically intercepts all database rows and connections to implement case, spacing, and punctuation insensitivity. + - When retrieving values from database rows or query results, you can use any casing (camelCase, snake_case, PascalCase, or with spaces) to query fields. For example, all of these are valid and return the same value: + ```python + sku_name = row.get("skuPartNumber") + sku_name = row.get("sku_part_number") + sku_name = row.get("SKU Part Number") + sku_name = row.get("SKU_Part_Number") + ``` + - This eliminates coding discrepancies between API payload casing, database schema casing, and frontend layout property lookups. + +--- + +## 4. Error Handling, Logging, and Retry Policy + +1. **Graceful Separation of Warnings and Exceptions**: + - **Log Files**: Always log the detailed raw traceback (`exc_info=True`) to log files (e.g. `telemetry_log.txt` via `logger.error()`) for system audits and developers. + - **UI Views**: Catch exceptions gracefully and show user-friendly instructions. Avoid displaying raw tracebacks on the UI frame. For example, if a `403 Forbidden` error is returned, tell the user: + `"Directory read permission required. Please grant 'Directory.Read.All' to your App Registration in Microsoft Entra ID."` +2. **Individual Tab Retryability**: + - Every telemetry frame/sub-frame must remain individually reloadable. Provide a `↻ Reload` / `Try Again` button on every section. + - Clicking reload must reset the local states, clear the grid, and spawn a fresh thread to fetch data from Graph. +3. **Configurable Retries & Backoff**: + - Expose retry counts and backoff delays in the central dashboard configuration. + - Pass these settings to all Graph Clients to automatically handle intermittent connection drops and HTTP 429 rate limit exceptions: + ```python + client = GraphClient( + tenant_id=tenant, + client_ids=client_id, + client_secrets=client_secret, + retries=retries_val, + backoff=backoff_val + ) + ``` + +--- + +## 5. UI Thread-Safety & Layout Guidelines + +1. **Never Block the Main Thread**: Never make API requests, run shell scripts, or parse entire files directly inside UI button commands or layout initializers. +2. **Use Background Workers & Semaphores**: Spawn fetches in a background `threading.Thread`. If a semaphore is provided, acquire and release it to keep concurrently running queries limited (e.g., max 3 active queries globally): + ```python + threading.Thread(target=self._execute_worker, args=(tenant, client_id), daemon=True).start() + ``` +3. **Request IDs and Stale Threads**: Users can toggle dashboard components or re-submit fetches. To prevent stale thread callbacks from corrupting UI state, track thread requests with `current_request_id`: + ```python + # Inside worker thread execution + if self.is_cancelled or request_id != self.current_request_id: + return + ``` +4. **Paginated Data Retrieval**: Do not read full SQLite tables or CSVs in memory. Instead, retrieve data using SQL pagination (LIMIT and OFFSET): + ```sql + SELECT * FROM table_name WHERE [column] != '' LIMIT ? OFFSET ? + ``` +5. **Layout & Color Consistency**: + - Match colors using predefined tokens in `telemetry/styles.py` (`COLOR_PRIMARY`, `COLOR_SURFACE`, `COLOR_OUTLINE_LIGHT`, etc.). Do not define ad-hoc hex values in frames. + - Every telemetry panel must display floating execution timers (`⏱ 4.25s`) on the top-right upon successful completion. + +--- + +## 6. Standard Component Template + +### Backend Pipeline Component (`core/graph/telemetry_feature.py`) +```python +import logging +from core.graph.client import GraphClient + +logger = logging.getLogger(__name__) + +class FeatureService: + def __init__(self, client: GraphClient) -> None: + self.client = client + + def get_feature_data(self) -> dict: + token_slot = self.client.get_active_token() + session = self.client.get_session() + headers = {"Authorization": f"Bearer {token_slot['token']}"} + + try: + url = "https://graph.microsoft.com/v1.0/reports/myTelemetryEndpoint" + resp = session.get(url, headers=headers) + resp.raise_for_status() + return resp.json() + finally: + self.client.release_token(token_slot) +``` + +### UI Frame Component (`telemetry/telemetry_feature.py`) +```python +import os +import csv +import logging +import threading +import sqlite3 +import asyncio +import customtkinter as ctk +from core.graph.client import GraphClient +from core.graph.db import import_csv_to_sqlite +from telemetry.styles import * + +logger = logging.getLogger("M365TelemetryAsyncLogger.FeatureUI") + +class FeatureTelemetryFrame(ctk.CTkFrame): + def __init__(self, master, log_callback, credentials_callback, status_change_callback, semaphore=None, **kwargs): + super().__init__(master, fg_color="transparent", **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.semaphore = semaphore + + self.status = None + self.is_cancelled = False + self.current_request_id = 0 + self.current_page = 0 + self.ITEMS_PER_PAGE = 10 + self.csv_path = None + + self.build_ui() + + def build_ui(self): + # Build CTk Labels, Grids, and Page controls here... + pass + + def trigger_fetch(self, tenant, client_id, client_secret): + self.status = "loading" + self.is_cancelled = False + self.current_page = 0 + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + reports_dir = os.path.join(script_dir, "reports", f"{tenant}_{client_id}") + self.csv_path = os.path.join(reports_dir, "feature_telemetry.csv") + + self._set_state_loading("Scanning feature details...") + self.on_status_change() + + threading.Thread( + target=self._execute_worker, + args=(tenant, client_id, client_secret, self.current_request_id), + daemon=True + ).start() + + def _execute_worker(self, tenant, client_id, client_secret, request_id): + if self.semaphore: self.semaphore.acquire() + temp_csv_path = self.csv_path + ".tmp" + try: + if self.is_cancelled or request_id != self.current_request_id: return + + # 1. Fetch from Microsoft Graph + client = GraphClient(tenant_id=tenant, client_ids=client_id, client_secrets=client_secret) + client.authenticate(required_scopes=["Directory.Read.All"]) + # (Execute your pipeline/service calls...) + + # 2. Write CSV to Disk (Compliance audit layer) + # 3. Import to SQLite Cache (Fast UI query layer) + db_path = os.path.join(os.path.dirname(self.csv_path), "telemetry_cache.db") + asyncio.run(import_csv_to_sqlite(self.csv_path, db_path, "feature_table")) + + if self.is_cancelled or request_id != self.current_request_id: return + self.status = "success" + self.after(0, self._render_success, request_id) + except Exception as e: + logger.error(f"Error fetching feature telemetry: {e}", exc_info=True) + self.status = "error" + self.after(0, self._render_error, str(e), request_id) + finally: + if os.path.exists(temp_csv_path): + try: os.remove(temp_csv_path) + except Exception: pass + if self.semaphore: self.semaphore.release() + self.after(0, self.on_status_change) + + def _load_page_from_sqlite(self, page): + if not self.csv_path or not os.path.exists(self.csv_path): return [], 0 + db_path = os.path.join(os.path.dirname(self.csv_path), "telemetry_cache.db") + if not os.path.exists(db_path): return [], 0 + + conn = sqlite3.connect(db_path) + try: + conn.row_factory = sqlite3.Row + cursor = conn.cursor() + + # Retrieve counts + cursor.execute("SELECT COUNT(*) FROM feature_table WHERE [Feature_ID] IS NOT NULL AND [Feature_ID] != ''") + row = cursor.fetchone() + total_count = row[0] if row else 0 + + # Retrieve page + offset = page * self.ITEMS_PER_PAGE + cursor.execute( + "SELECT [Feature_ID], [Status] FROM feature_table WHERE [Feature_ID] IS NOT NULL AND [Feature_ID] != '' LIMIT ? OFFSET ?", + (self.ITEMS_PER_PAGE, offset) + ) + rows = cursor.fetchall() + + page_data = [] + for r in rows: + page_data.append((r["Feature_ID"] or "", r["Status"] or "")) + return page_data, total_count + except Exception as e: + logger.error(f"Error reading SQLite: {e}") + return [], 0 + finally: + conn.close() + + def _render_success(self, request_id): + if self.is_cancelled or request_id != self.current_request_id: return + self._update_grid() + + def _render_error(self, err, request_id): + if self.is_cancelled or request_id != self.current_request_id: return + # Display error state in UI... + pass + + def _update_grid(self): + page_data, total_count = self._load_page_from_sqlite(self.current_page) + # Populate CustomTkinter cells and draw page labels/buttons... + pass +``` + +--- + +## 7. PDF Report Integration Guidelines + +When adding a new telemetry section or sub-section, you must ensure it is integrated correctly with the PDF generation feature (`telemetry/pdf_report.py`). The main integration point is the `get_all_telemetry_data()` method inside `telemetry/m365_telemetry.py`. + +Follow these rules to prevent missing data in the exported PDF: + +1. **Verify State Properties**: + - Do not assume UI property names. Check the target frame class to find the exact property where it stores its parsed data (usually `last_data`). For example, check whether it uses `o365_data` or `last_data` and map it appropriately. + - When retrieving data via sub-elements, use `getattr(self.some_view.sub_view, "last_data", [])`. + +2. **Query Subframe Components Directly**: + - If a feature is hosted within a container frame (e.g., `DataSecurityGovernanceFrame`), do not call batch methods like `load_all_from_csv` on the container unless it specifically implements them. + - Instead, access the sub-elements' properties directly: + ```python + "security_labels": getattr(self.security_gov_view.sensitivity_frame, "last_data", []) + ``` + +3. **Fallback to Direct CSV Loading**: + - If a subframe component cleans up its memory footprint (e.g., setting `self.last_data = []` to save space, like `EDiscoveryFrame`), load the data directly from the local CSV reports directory using the internal `load_csv("filename.csv")` helper: + ```python + "ediscovery_cases": load_csv("ediscovery_cases.csv") + ``` + +4. **Coordinate Data Schemas**: + - Ensure the structure of data retrieved (e.g., list of dictionaries, flat rows, or tuples) matches exactly what the PDF builder (`telemetry/pdf_report.py`) expects. + - If `pdf_report.py` expects nested objects (e.g. `closedBy` containing `user`), ensure the data returned by `.last_data` or `load_csv()` matches this format or parse it accordingly. + +--- + +## 8. Documentation Update Guidelines + +Whenever a new telemetry section or sub-section is added, you must update the project [README.md](file:///usr/local/google/home/projjalkundu/.gemini/jetski/scratch/splash_one/README.md) to keep it in sync with the codebase: + +1. **Telemetry Modules & Technical Mechanisms Section**: + - Add the new module or sub-section to the functional list. + - Explicitly document its **Functional Scope** (what metrics/configurations it audits). + - Document its **Mechanism & Endpoints** (what Graph API URLs it hits, what PowerShell scripts/cmdlets it calls, and what CSV/SQLite database table caches it creates). + +2. **Deal Assistant Telemetry Permissions Section**: + - Verify if the new API endpoints or PowerShell commands require new Microsoft Graph permissions (Application or Delegated) or directory roles (e.g. Compliance Administrator). + - List any new permissions in the bulleted scopes in `README.md` with a brief description of what they are used for. + - Clearly state if the permission is optional (i.e. skipped gracefully if missing) or mandatory. + +3. **Tab 1 Outputs Section**: + - If the new section writes to a custom log file or introduces a new CSV report pattern that users might want to audit directly, document its exact file path template and purpose under the "Usage and Adoption" outputs. diff --git a/estimators/estimator.py b/estimators/estimator.py index 096a254f..db16ca10 100644 --- a/estimators/estimator.py +++ b/estimators/estimator.py @@ -26,7 +26,6 @@ def calculate_migration_eta(self, data: Dict[str, Any]) -> float: global_limit = data.get("global_limit", 100) batch_time = data.get("batch_time", 1) user_limit = data.get("user_limit", 1) - multiplier = data.get("multiplier", 1) active_counts = [c for c in item_counts if c > 0] if not active_counts: @@ -51,7 +50,7 @@ def calculate_migration_eta(self, data: Dict[str, Any]) -> float: total_seconds += seconds_for_layer previous_level = current_level - return multiplier * (total_seconds / 3600.0) + return total_seconds / 3600.0 def get_resource_type(self) -> str: raise NotImplementedError("Subclasses must implement the get_resource_type method") diff --git a/estimators/file_estimator.py b/estimators/file_estimator.py index 30ebeb76..99da825e 100644 --- a/estimators/file_estimator.py +++ b/estimators/file_estimator.py @@ -159,8 +159,6 @@ def calculate_resource_metrics( url_to_site_id = self._get_sites_from_urls(data["siteUrls"], site_discovery_progress_metrics, failures) for url, site_id in url_to_site_id.items(): top_level_sites.append(site_id) - self.site_to_metadata[site_id] = {"isPersonalSite": False} - metrics["teamSiteCount"] += 1 site_id_to_url = {site_id: url for url, site_id in url_to_site_id.items()} @@ -168,7 +166,7 @@ def calculate_resource_metrics( metrics["siteCount"] = len(top_level_sites) all_sites = [{"siteId": site_id, "siteLevel": 0} for site_id in top_level_sites] - self._get_subsites_in_site(top_level_sites, all_sites, subsite_to_top_level_site, site_discovery_progress_metrics, failures, metrics, 1) + self._get_subsites_in_site(top_level_sites, all_sites, subsite_to_top_level_site, site_discovery_progress_metrics, failures, 1) if not has_emails and not has_urls: metrics["personalSiteCount"] = site_discovery_progress_metrics.get("personalSiteCount", 0) @@ -360,7 +358,8 @@ def _update_tenant_metrics_from_drive_metrics( metrics["maxSubsiteDepth"] = max(metrics["maxSubsiteDepth"], metrics["siteMetrics"][subsite_id]["siteLevel"]) top_level_site = subsite_to_top_level_site.get(subsite_id, subsite_id) - subsite_item_count = 0 # Used to track if this subsite is a Large Resource + if self.id_to_display.get(subsite_id, "") == "https://smh3v.sharepoint.com/subsiteofrootsite": + print(f"FOUND the URL: {subsite_to_top_level_site.get(subsite_id, "")}") if top_level_site != subsite_id: metrics["siteMetrics"][top_level_site]["subsiteCount"] = metrics["siteMetrics"][top_level_site].get("subsiteCount", 0) + 1 @@ -369,20 +368,7 @@ def _update_tenant_metrics_from_drive_metrics( if drive_id in metrics["driveMetrics"]: drive_metric = metrics["driveMetrics"][drive_id] - subsite_item_count += drive_metric.get("fileCount", 0) + drive_metric.get("folderCount", 0) metrics["siteMetrics"][top_level_site]["largeResourceCount"] = metrics["siteMetrics"].get(top_level_site, {}).get("largeResourceCount", 0) + len(drive_metric.get("largeResources", [])) - if drive_metric.get("fileCount", 0) + drive_metric.get("folderCount", 0) > self.config.large_resource_count_limit: - metrics["siteMetrics"][top_level_site]["largeResourceCount"] += 1 - metrics["tenantLevelLargeResources"].append( - { - "type": ResourceType.DL.value, - "id": drive_id, - "subTreeCount": drive_metric.get("fileCount", 0) + drive_metric.get("folderCount", 0), - "parent": subsite_id, # Explicitly showing subsite id here as users can use it to determine site collection easily (webUrl will be displayed in final report). - "Limit": self.config.large_resource_count_limit - } - ) - metrics["siteMetrics"][top_level_site]["folderCount"] = metrics["siteMetrics"].get(top_level_site, {}).get("folderCount", 0) + drive_metric.get("folderCount", 0) metrics["siteMetrics"][top_level_site]["fileCount"] = metrics["siteMetrics"].get(top_level_site, {}).get("fileCount", 0) + drive_metric.get("fileCount", 0) metrics["siteMetrics"][top_level_site]["shortcutCount"] = metrics["siteMetrics"].get(top_level_site, {}).get("shortcutCount", 0) + drive_metric.get("shortcutCount", 0) @@ -390,19 +376,7 @@ def _update_tenant_metrics_from_drive_metrics( metrics["siteMetrics"][top_level_site]["folderCountExceedingDepthLimit"] = metrics["siteMetrics"].get(top_level_site, {}).get("folderCountExceedingDepthLimit", 0) + drive_metric.get("folderCountExceedingDepthLimit", 0) metrics["siteMetrics"][top_level_site]["fileCountExceedingDepthLimit"] = metrics["siteMetrics"].get(top_level_site, {}).get("fileCountExceedingDepthLimit", 0) + drive_metric.get("fileCountExceedingDepthLimit", 0) - # Check if this subsite is a Large Resource - if subsite_item_count > self.config.large_resource_count_limit and subsite_id != top_level_site: - metrics["siteMetrics"][top_level_site]["largeResourceCount"] = metrics["siteMetrics"][top_level_site].get("largeResourceCount", 0) + 1 - metrics["tenantLevelLargeResources"].append( - { - "type": ResourceType.SUBSITE.value, - "id": subsite_id, - "subTreeCount": subsite_item_count, - "parent": top_level_site, - "Limit": self.config.large_resource_count_limit - } - ) - + metrics["siteMetrics"][top_level_site]["dlCount"] = metrics["siteMetrics"].get(top_level_site, {}).get("dlCount", 0) + len(drive_ids) if self._is_subsite_personal(subsite_id): @@ -412,18 +386,6 @@ def _update_tenant_metrics_from_drive_metrics( if top_level_site != subsite_id: metrics["subsiteCount"] += 1 - - for site_id, metric in metrics["siteMetrics"].items(): - if metric.get("folderCount", 0) + metric.get("fileCount", 0) > self.config.large_resource_count_limit: - metrics["tenantLevelLargeResources"].append( - { - "type": ResourceType.SITE.value, - "id": site_id, - "subTreeCount": metric.get("fileCount", 0) + metric.get("folderCount", 0), - "parent": "N/A (Top level site)", - "Limit": self.config.large_resource_count_limit - } - ) for siteId in subsite_to_drives.keys(): top_level_site = subsite_to_top_level_site.get(siteId, siteId) @@ -457,7 +419,7 @@ def _update_tenant_metrics_from_drive_metrics( for drive_id, metric in metrics["driveMetrics"].items(): for large_resource in metric["largeResources"]: curr_dict = large_resource - curr_dict["parent"] = drive_id + curr_dict["drive"] = drive_id metrics["tenantLevelLargeResources"].append(curr_dict) metrics["tenantLevelLargeResourceCount"] = len(metrics["tenantLevelLargeResources"]) @@ -476,11 +438,10 @@ def _get_subsites_in_site( subsite_to_top_level_site: Dict[str, str], site_discovery_progress_metrics: Dict[str, Any], failures: List[Dict[str, str]], - tenant_metrics: Dict[str, Any], level: int = 1 ): try: - site_url = "/sites/{siteId}/sites?$select=id,webUrl,isPersonalSite&$top=999" + site_url = "/sites/{siteId}/sites?$select=id,weburl,isPersonalSite&$top=999" batches = create_batches(site_url, [{"siteId": site_id} for site_id in site_ids], self.config.parallel_batches, True) futures_map: Dict[int, Future[List[Dict[str, Any]]]] = {} @@ -581,15 +542,9 @@ def local_progress_callback(responses: List, has_next=False): self.site_to_metadata[site["id"]] = { "isPersonalSite": site.get("isPersonalSite", False) } - self.id_to_display[site["id"]] = site.get("webUrl", site["id"]) - - if site.get("isPersonalSite", False): - tenant_metrics["personalSiteCount"] += 1 - else: - tenant_metrics["teamSiteCount"] += 1 if new_sub_site_ids: - self._get_subsites_in_site(new_sub_site_ids, all_sites, subsite_to_top_level_site, site_discovery_progress_metrics, failures, tenant_metrics, level + 1) + self._get_subsites_in_site(new_sub_site_ids, all_sites, subsite_to_top_level_site, site_discovery_progress_metrics, failures, level + 1) except Exception as e: self._log_and_fail("Error in _get_subsites_in_site", e, failures) @@ -1075,7 +1030,7 @@ def _create_in_memory_tree( parent_references[drive_id] = {} try: # use delta api to fetch the folders - delta_api = "/drives/{driveId}/root/delta?$select=id,parentReference,name,webUrl,folder,file,remoteItem,size" + delta_api = "/drives/{driveId}/root/delta?$select=id,parentReference,name,folder,file,remoteItem,size" batches = create_batches(delta_api, [{"driveId": drive_id} for drive_id in drive_ids], self.config.parallel_batches, True) futures_map: Dict[int, Future[List[Dict[str, Any]]]] = {} @@ -1191,7 +1146,7 @@ def local_progress_callback(responses: List, has_next=False): if "body" in resp and "value" in resp["body"]: for file in resp["body"]["value"]: resource_id_to_details[file["id"]] = file - self.id_to_display[file["id"]] = file.get("webUrl", file["name"]) + if "parentReference" in file and "id" in file["parentReference"]: parent_references[drive_id][file["id"]] = file["parentReference"]["id"] @@ -1475,7 +1430,7 @@ def _update_drive_metrics_from_resource( if resource_metric["subTreeCount"] >= self.config.large_resource_count_limit: drive_metric["largeResources"].append({ "type": ResourceType.FOLDER.value if "folder" in resource else ResourceType.FILE.value, - "id": resource["id"], + "id": resource["name"], "subTreeCount": resource_metric["subTreeCount"], "Limit": self.config.large_resource_count_limit }) diff --git a/flet_app/README.md b/flet_app/README.md new file mode 100644 index 00000000..73ea2366 --- /dev/null +++ b/flet_app/README.md @@ -0,0 +1,91 @@ +# Deal Assistant - Flet Dashboard UI + +This folder contains the Flet-based modern desktop dashboard application for the **Migration Planner Tool / Deal Assistant**. It provides an interactive, beautiful, web-like graphical user interface to collect and analyze Microsoft 365 tenant license adoption, compliance policies, active usage trends, and Power Automate flow telemetry. + +--- + +## Key Features + +- **Unified Credentials Connection (`AuthView`):** Enter your Tenant ID, Client ID, and Client Secret. +- **Certificate-Based Auth Flow (`CertInstructionsView`):** Automatically detects if a security certificate is missing. Generates a new `certificate.pem` file locally and guides you on uploading it to the Microsoft Entra ID portal to enable secure Exchange Online and Retention Policy scans. +- **Interactive Reports Dashboard (`DashboardView`):** + 1. **Subscribed SKUs Inventory Summary:** Lists active license plans, pre-paid quantities, consumed units, and status. Includes a **CSV Export** button. + 2. **O365 Active Users Usage:** Shows active user metrics (30-day, 90-day, 180-day) for Exchange, OneDrive, SharePoint, and Teams. + 3. **O365 30-Day Active User Trend:** Renders a gorgeous visual line chart (using Matplotlib backend rendering) showing historical trends. + 4. **M365 App Usage (180 Days):** Platform/App distribution details for user endpoints. + 5. **Exchange Online Mailbox Usage Telemetry:** Details total mailboxes, collective size, average sizes, and email volumes. + 6. **SharePoint Site Usage Telemetry:** Details total sites, storage consumed, files stored, and percent active files. + 7. **OneDrive Usage Telemetry:** Highlights OneDrive accounts, usage levels, file synchronisation percentages, and active OneNote users. + 8. **Sensitivity Labels:** Displays configured sensitivity labels (with child hierarchies), protection details, priority, and application targets. Supports **Pagination** (Page 1 of N). + 9. **Retention Compliance Policies:** Displays tenant compliance rules, workloads, and duration metrics. Features a quick link to open Microsoft Purview. + 10. **Power Automate Flows:** Lists environment counts, flow types, and premium/custom connector usage. Includes a **CSV Export** button to download complex logic flows. +- **Granular Individual Card Retry/Refresh:** Each card features a Refresh (`ft.Icons.REFRESH`) button on the top-right. You can re-fetch telemetry for an individual section (e.g. just SharePoint or just SKUs) without having to trigger a full master scan of the entire tenant again. + +--- + +## Prerequisites & Installation + +To run the Flet application, you need to set up Python and install the required UI and backend libraries. + +### 1. Python Environment +Make sure you have **Python 3.10** or newer installed. We highly recommend using a virtual environment: + +```bash +# Create a virtual environment +python -m venv venv + +# Activate the environment +# On macOS/Linux: +source venv/bin/activate +# On Windows (cmd): +.\venv\Scripts\activate +# On Windows (PowerShell): +.\venv\Scripts\Activate.ps1 +``` + +### 2. Install Required Python Packages +With your virtual environment active, run: + +```bash +pip install flet matplotlib pandas requests urllib3 aiohttp certifi psutil Pillow customtkinter +``` + +*Note: Flet does not require any additional web server setup. Matplotlib is used in headless mode (`matplotlib.use("Agg")`) to render the trend chart into the Flet UI natively.* + +### 3. PowerShell Prerequisites (For Retention Policy Scan) +The Retention Compliance Policy scanner uses PowerShell Core and the Exchange Online module. + +#### Install PowerShell Core (`pwsh`): +- **macOS (via Homebrew):** + ```bash + brew install powershell + ``` +- **Windows (via winget):** + ```cmd + winget install --id Microsoft.Powershell --source winget + ``` + +#### Install the Exchange Online Module: +Open PowerShell (`pwsh`) and install the module: +```powershell +Install-Module -Name ExchangeOnlineManagement -Scope CurrentUser -Force +``` + +--- + +## How to Run + +1. Open your terminal or command prompt and navigate to the project root directory (the parent of `flet_app/`): + ```bash + cd /path/to/project_root + ``` +2. Activate your virtual environment: + ```bash + source venv/bin/activate + ``` +3. Launch the Flet application: + ```bash + python flet_app/main.py + ``` +4. Enter your Azure Client/Tenant credentials to connect. +5. If requested, locate the auto-generated certificate (`certificate/certificate.pem`), upload it to the Azure portal under **App Registrations > Certificates & secrets**, and click **Continue** to load the dashboard. diff --git a/flet_app/auth_view.py b/flet_app/auth_view.py new file mode 100644 index 00000000..3507e9e8 --- /dev/null +++ b/flet_app/auth_view.py @@ -0,0 +1,86 @@ +import flet as ft +from flet_app.styles import * + +class AuthView(ft.Container): + def __init__(self, on_connect_clicked): + super().__init__() + self.on_connect_clicked = on_connect_clicked + + self.expand = True + + self.tenant_input = ft.TextField( + label="Tenant ID", + border_color=COLOR_OUTLINE, + focused_border_color=COLOR_PRIMARY, + text_size=14, + ) + self.client_input = ft.TextField( + label="Client ID", + border_color=COLOR_OUTLINE, + focused_border_color=COLOR_PRIMARY, + text_size=14, + ) + self.secret_input = ft.TextField( + label="Client Secret", + password=True, + can_reveal_password=True, + border_color=COLOR_OUTLINE, + focused_border_color=COLOR_PRIMARY, + text_size=14, + ) + + self.status_text = ft.Text(value="", color=COLOR_ERROR, size=14) + + self.content = ft.Column( + alignment=ft.MainAxisAlignment.CENTER, + horizontal_alignment=ft.CrossAxisAlignment.CENTER, + controls=[ + ft.Container( + width=600, + bgcolor=COLOR_SURFACE, + border_radius=12, + border=ft.Border.all(1, COLOR_OUTLINE_LIGHT), + padding=40, + content=ft.Column( + controls=[ + ft.Text("Deal Assistant", size=28, weight=ft.FontWeight.BOLD, color=COLOR_PRIMARY), + ft.Text("Connect your Azure App Credentials to begin auditing your tenant.", size=16, color=COLOR_TEXT_SUB), + ft.Divider(height=40, color="transparent"), + self.tenant_input, + ft.Divider(height=10, color="transparent"), + self.client_input, + ft.Divider(height=10, color="transparent"), + self.secret_input, + ft.Divider(height=10, color="transparent"), + self.status_text, + ft.Divider(height=20, color="transparent"), + ft.ElevatedButton( + content="Connect & Continue", + bgcolor=COLOR_PRIMARY, + color=ft.Colors.WHITE, + height=45, + style=ft.ButtonStyle(shape=ft.RoundedRectangleBorder(radius=8)), + on_click=self.handle_connect, + width=float('inf') + ) + ] + ) + ) + ] + ) + + def handle_connect(self, e): + tenant = self.tenant_input.value.strip() if self.tenant_input.value else "" + client = self.client_input.value.strip() if self.client_input.value else "" + secret = self.secret_input.value.strip() if self.secret_input.value else "" + + if not tenant or not client or not secret: + self.status_text.value = "Error: Tenant ID, Client ID, and Client Secret are required." + self.update() + return + + self.status_text.value = "" + self.update() + + # Pass the credentials to the parent callback + self.on_connect_clicked(tenant, client, secret) diff --git a/flet_app/cert_instructions_view.py b/flet_app/cert_instructions_view.py new file mode 100644 index 00000000..2308dd43 --- /dev/null +++ b/flet_app/cert_instructions_view.py @@ -0,0 +1,75 @@ +import flet as ft +from flet_app.styles import * + +class CertInstructionsView(ft.Container): + def __init__(self, pem_path, client_id, on_continue): + super().__init__() + self.pem_path = pem_path + self.client_id = client_id + self.on_continue = on_continue + + self.expand = True + + self.content = ft.Column( + alignment=ft.MainAxisAlignment.CENTER, + horizontal_alignment=ft.CrossAxisAlignment.STRETCH, + controls=[ + ft.Container( + bgcolor=COLOR_SURFACE, + border_radius=12, + border=ft.Border.all(1, COLOR_OUTLINE_LIGHT), + padding=40, + margin=ft.margin.symmetric(horizontal=40), + content=ft.Column( + controls=[ + ft.Text("Certificate Upload", size=24, weight=ft.FontWeight.BOLD, color=COLOR_PRIMARY), + ft.Text("A new security certificate has been generated for hybrid authentication.\n\nUploading this certificate is highly recommended, but optional:", size=14, color=COLOR_TEXT_MAIN), + ft.Text("• If you UPLOAD the certificate:\n All report sections will be fully functional.", size=14, color=COLOR_SUCCESS, weight=ft.FontWeight.BOLD), + ft.Text("• If you SKIP uploading the certificate:\n You can still run the reports. However, sections relying on certificate-based authentication (such as detailed Calendar settings, Shared/Public mailbox statistics, Retention Policies etc.) will be skipped and show as unavailable.", size=14, color=COLOR_ERROR, weight=ft.FontWeight.BOLD), + ft.Divider(height=20, color="transparent"), + + ft.Container( + content=ft.Column([ + ft.Text("Upload Instructions", size=14, weight=ft.FontWeight.BOLD, color=COLOR_TONAL_TEXT), + ft.Text("1. Locate the certificate file generated at:", size=13, color=COLOR_TEXT_MAIN), + ft.Container( + content=ft.Text(self.pem_path, size=12, color=COLOR_TEXT_MAIN, selectable=True, font_family="Courier New"), + bgcolor=COLOR_SURFACE, + padding=10, + border_radius=6, + width=float('inf') + ), + ft.Text("2. Log in to the Microsoft Azure portal and navigate to the App Registration with Client ID:", size=13, color=COLOR_TEXT_MAIN), + ft.Container( + content=ft.Text(self.client_id, size=12, color=COLOR_TEXT_MAIN, selectable=True, font_family="Courier New"), + bgcolor=COLOR_SURFACE, + padding=10, + border_radius=6, + width=float('inf') + ), + ft.Text("3. Upload the certificate under:", size=13, color=COLOR_TEXT_MAIN), + ft.Text(" Certificates & secrets -> Certificates -> Upload certificate", size=13, weight=ft.FontWeight.BOLD, color=COLOR_TEXT_MAIN), + ]), + border=ft.Border.all(1, COLOR_OUTLINE), + border_radius=8, + padding=20, + bgcolor=COLOR_TONAL_BG + ), + + ft.Divider(height=30, color="transparent"), + ft.ElevatedButton( + content=ft.Text("Continue", color=ft.Colors.WHITE, weight=ft.FontWeight.BOLD), + bgcolor=COLOR_PRIMARY, + height=40, + style=ft.ButtonStyle(shape=ft.RoundedRectangleBorder(radius=20)), + on_click=self.handle_continue, + width=float('inf') + ) + ] + ) + ) + ] + ) + + def handle_continue(self, e): + self.on_continue() diff --git a/flet_app/custom_chart.py b/flet_app/custom_chart.py new file mode 100644 index 00000000..8133c2ca --- /dev/null +++ b/flet_app/custom_chart.py @@ -0,0 +1,59 @@ +import flet as ft +import io +import base64 +import matplotlib +matplotlib.use('Agg') +import matplotlib.pyplot as plt + +class CustomLineChart(ft.Container): + def __init__(self, dates, datasets, colors, height=300): + super().__init__() + self.dates = dates + self.datasets = datasets + self.colors = colors + self.chart_height = height + + self.content = self.create_chart_image() + self.height = self.chart_height + self.expand = True + + def create_chart_image(self): + if not self.dates or not self.datasets: + return ft.Container() + + fig, ax = plt.subplots(figsize=(10, 4)) + + for name, data in self.datasets.items(): + color = self.colors.get(name, "#000000") + ax.plot(self.dates, data, label=name, color=color, marker='o', markersize=5, linewidth=2.5) + + # Formatting + fig.patch.set_facecolor('none') + ax.set_facecolor('none') + ax.spines['top'].set_visible(False) + ax.spines['right'].set_visible(False) + ax.spines['left'].set_color('#E2E8F0') + ax.spines['bottom'].set_color('#E2E8F0') + ax.grid(axis='y', color='#E2E8F0', linestyle='-', linewidth=1) + ax.tick_params(axis='both', colors='#64748B', labelsize=10) + + # Adjust x ticks to not overlap + num_dates = len(self.dates) + label_interval = max(1, num_dates // 6) + ax.set_xticks(range(0, num_dates, label_interval)) + ax.set_xticklabels([self.dates[i] for i in range(0, num_dates, label_interval)]) + + # Apply tight layout to minimize whitespace + fig.tight_layout() + + # Save to buffer + buf = io.BytesIO() + fig.savefig(buf, format='png', bbox_inches='tight', transparent=True, dpi=120) + buf.seek(0) + + # Close fig + plt.close(fig) + + b64_string = base64.b64encode(buf.read()).decode('utf-8') + + return ft.Image(src=f"data:image/png;base64,{b64_string}", fit="contain", expand=True) diff --git a/flet_app/dashboard.py b/flet_app/dashboard.py new file mode 100644 index 00000000..f36c4b15 --- /dev/null +++ b/flet_app/dashboard.py @@ -0,0 +1,1184 @@ +import flet as ft +from flet_app.styles import * +from flet_app.sidebar import Sidebar +from flet_app.custom_chart import CustomLineChart +import threading +import sys +import os +import datetime +import csv + +# Import existing backend modules +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from core.graph.client import GraphClient +from core.graph.directory import DirectoryService +from telemetry import active_users_usage as usage +from telemetry.power_automate import PowerAutomateScanner +from telemetry.mailbox_usage import run_mailbox_usage_pipeline +from telemetry.calendar_telemetry import run_calendar_telemetry_pipeline +from telemetry.sharepoint_onedrive_usage import run_sharepoint_pipeline, run_onedrive_pipeline +from telemetry.data_security_governance import run_security_governance_pipeline + +class DashboardView(ft.Container): + def __init__(self, tenant, client, secret, on_disconnect): + super().__init__() + self.tenant = tenant + self.client = client + self.secret = secret + self.on_disconnect = on_disconnect + + self.expand = True + + self.sidebar = Sidebar(on_disconnect=self.on_disconnect) + + # Saved data for CSV exports + self.last_licenses_items = [] + self.last_complex_flows = [] + + # Saved data for Sensitivity Labels pagination + self.flattened_labels = [] + self.current_labels_page = 0 + self.labels_per_page = 8 + + # Track states of parallel fetches + self.fetch_statuses = {} + + # Header + self.fetch_btn = ft.ElevatedButton( + content=ft.Text("Fetch Report", weight=ft.FontWeight.BOLD), + bgcolor=COLOR_PRIMARY, + color=ft.Colors.WHITE, + height=40, + style=ft.ButtonStyle(shape=ft.RoundedRectangleBorder(radius=8)), + on_click=self.handle_fetch + ) + + self.header = ft.Container( + content=ft.Row( + controls=[ + ft.Text("Usage Report", size=20, weight=ft.FontWeight.BOLD, color=COLOR_TEXT_MAIN), + self.fetch_btn + ], + alignment=ft.MainAxisAlignment.SPACE_BETWEEN + ), + bgcolor=COLOR_SURFACE, + border_radius=12, + border=ft.Border.all(1, COLOR_OUTLINE_LIGHT), + padding=ft.Padding.symmetric(horizontal=20, vertical=15), + margin=ft.Margin.only(bottom=20) + ) + + # 1. SKUs Card (with Export Button) + self.export_sku_btn = ft.IconButton( + icon=ft.Icons.DOWNLOAD, + icon_color=COLOR_PRIMARY, + tooltip="Export Spreadsheet", + disabled=True, + on_click=self.handle_export_skus + ) + self.sku_link = ft.TextButton( + content=ft.Text("Service Plan Reference ↗", color=COLOR_PRIMARY, weight=ft.FontWeight.BOLD), + on_click=lambda e: e.page.launch_url("https://learn.microsoft.com/en-us/entra/identity/users/licensing-service-plan-reference") + ) + sku_actions = ft.Row( + controls=[ + self.sku_link, + self.export_sku_btn + ], + spacing=5 + ) + self.sku_section = self.create_card("Subscribed SKUs", action_control=sku_actions, on_retry=self.handle_retry_skus) + + # 2. O365 Usage Card + self.o365_section = self.create_card("O365 Active Users Usage", on_retry=self.handle_retry_o365) + + # 3. O365 Trend Chart Card + self.trend_section = self.create_card("O365 30-Day Active User Trend", on_retry=self.handle_retry_trend) + + # 4. M365 App Usage Card + self.m365_section = self.create_card("M365 App Usage (180 Days)", on_retry=self.handle_retry_m365) + + # 5. Exchange Online Card (Combined Emails and Calendar Sections) + self.emails_container = ft.Container(content=ft.Text("No data yet.", color=COLOR_TEXT_SUB)) + self.calendar_container = ft.Container(content=ft.Text("No data yet.", color=COLOR_TEXT_SUB)) + self.exchange_layout = ft.Column( + controls=[ + ft.Text("Exchange Online Email", size=16, weight=ft.FontWeight.BOLD, color=COLOR_PRIMARY), + self.emails_container, + ft.Divider(height=10, color="transparent"), + ft.Text("Exchange Online Calendar", size=16, weight=ft.FontWeight.BOLD, color=COLOR_PRIMARY), + self.calendar_container + ], + spacing=5 + ) + self.exchange_section = self.create_card("Exchange Online", on_retry=self.handle_retry_exchange) + self.exchange_section.content_container.content = self.exchange_layout + + # 6. Files Card (Combined SharePoint and OneDrive Sections) + self.sharepoint_container = ft.Container(content=ft.Text("No data yet.", color=COLOR_TEXT_SUB)) + self.onedrive_container = ft.Container(content=ft.Text("No data yet.", color=COLOR_TEXT_SUB)) + self.files_layout = ft.Column( + controls=[ + ft.Text("SharePoint Site Usage (180 Days)", size=16, weight=ft.FontWeight.BOLD, color=COLOR_PRIMARY), + self.sharepoint_container, + ft.Divider(height=10, color="transparent"), + ft.Text("OneDrive Usage (180 Days)", size=16, weight=ft.FontWeight.BOLD, color=COLOR_PRIMARY), + self.onedrive_container + ], + spacing=5 + ) + self.files_section = self.create_card("Files", on_retry=self.handle_retry_files) + self.files_section.content_container.content = self.files_layout + + # 8. Sensitivity Labels Card (with Pagination Controls) + self.labels_pagination_info = ft.Text("Page 1 of 1", size=13, color=COLOR_TEXT_MAIN) + self.labels_prev_btn = ft.IconButton(ft.Icons.ARROW_BACK, on_click=self.handle_labels_prev, disabled=True) + self.labels_next_btn = ft.IconButton(ft.Icons.ARROW_FORWARD, on_click=self.handle_labels_next, disabled=True) + self.labels_pagination_row = ft.Row( + controls=[self.labels_prev_btn, self.labels_pagination_info, self.labels_next_btn], + alignment=ft.MainAxisAlignment.CENTER, + visible=False + ) + self.purview_labels_btn = ft.TextButton( + content=ft.Text("Open Purview Sensitivity Label Portal ↗", color=COLOR_PRIMARY, weight=ft.FontWeight.BOLD), + on_click=lambda e: e.page.launch_url("https://purview.microsoft.com/informationprotection/informationprotectionlabels/sensitivitylabels") + ) + self.labels_section = self.create_card( + "Sensitivity Labels", + action_control=self.purview_labels_btn, + bottom_control=self.labels_pagination_row, + on_retry=self.handle_retry_labels + ) + + # 9. Retention Policies Card (with Purview Link) + self.purview_btn = ft.TextButton( + content=ft.Text("Open Purview Retention Policy Portal ↗", color=COLOR_PRIMARY, weight=ft.FontWeight.BOLD), + on_click=lambda e: e.page.launch_url("https://purview.microsoft.com/datalifecyclemanagement/retention") + ) + self.retention_section = self.create_card("Retention Compliance Policies", action_control=self.purview_btn, on_retry=self.handle_retry_retention) + + # 9c. eDiscovery Cases Card (Instructional) + self.ediscovery_portal_btn = ft.TextButton( + content=ft.Text("Open Purview eDiscovery Portal ↗", color=COLOR_PRIMARY, weight=ft.FontWeight.BOLD), + on_click=lambda e: e.page.launch_url("https://purview.microsoft.com/ediscovery/casespage") + ) + self.ediscovery_permissions_btn = ft.TextButton( + content=ft.Text("Open Purview Permissions Settings ↗", color=COLOR_PRIMARY, weight=ft.FontWeight.BOLD), + on_click=lambda e: e.page.launch_url("https://purview.microsoft.com/settings/purviewpermissions") + ) + + self.ediscovery_section = self.create_card( + "eDiscovery Cases", + action_control=self.ediscovery_portal_btn + ) + + self.ediscovery_section.content_container.content = ft.Column([ + ft.Text( + "eDiscovery cases cannot be scanned directly under standard Application permissions. " + "To view your active cases, please navigate to Microsoft Purview on behalf of a user who has the eDiscovery Manager role.", + color=COLOR_TEXT_MAIN, + size=14 + ), + ft.Divider(height=10, color="transparent"), + ft.Row([ + ft.Text("To assign the eDiscovery Manager role, go to:", color=COLOR_TEXT_SUB, size=13), + self.ediscovery_permissions_btn + ], alignment=ft.MainAxisAlignment.START, spacing=5) + ], spacing=10) + + # 10. Power Automate Card (with Export Button) + self.export_pa_btn = ft.IconButton( + icon=ft.Icons.DOWNLOAD, + icon_color=COLOR_PRIMARY, + tooltip="Export Complex Flows", + disabled=True, + on_click=self.handle_export_pa + ) + self.pa_section = self.create_card("Power Automate", action_control=self.export_pa_btn, on_retry=self.handle_retry_pa) + + self.content_area = ft.Column( + controls=[ + self.sku_section, + self.o365_section, + self.trend_section, + self.m365_section, + self.exchange_section, + self.files_section, + self.labels_section, + self.retention_section, + self.ediscovery_section, + self.pa_section + ], + scroll=ft.ScrollMode.AUTO, + expand=True, + spacing=20 + ) + + self.content = ft.Row( + controls=[ + self.sidebar, + ft.Container( + content=ft.Column( + controls=[self.header, self.content_area], + expand=True + ), + expand=True, + padding=ft.Padding.only(left=20) + ) + ], + expand=True + ) + + def create_card(self, title, action_control=None, bottom_control=None, on_retry=None): + content_container = ft.Container(content=ft.Text("No data yet.", color=COLOR_TEXT_SUB)) + header_controls = [ft.Text(title, size=16, weight=ft.FontWeight.BOLD, color=COLOR_TEXT_MAIN)] + + actions = [] + if action_control: + actions.append(action_control) + + retry_btn = None + if on_retry: + retry_btn = ft.IconButton( + icon=ft.Icons.REFRESH, + icon_color=COLOR_PRIMARY, + icon_size=20, + tooltip="Refresh this section", + on_click=on_retry + ) + actions.append(retry_btn) + + if actions: + header_controls.append(ft.Row(controls=actions, spacing=10)) + + header_row = ft.Row( + controls=header_controls, + alignment=ft.MainAxisAlignment.SPACE_BETWEEN + ) + + column_controls = [ + header_row, + ft.Divider(height=20, color="transparent"), + content_container + ] + if bottom_control: + column_controls.append(bottom_control) + + card = ft.Container( + content=ft.Column(column_controls), + bgcolor=COLOR_SURFACE, + border_radius=12, + border=ft.Border.all(1, COLOR_OUTLINE_LIGHT), + padding=20, + width=float('inf') + ) + card.content_container = content_container + card.retry_btn = retry_btn + return card + + def set_loading(self, card, message): + card.content_container.content = ft.Column([ + ft.ProgressRing(), + ft.Text(message, color=COLOR_TEXT_SUB) + ], alignment=ft.MainAxisAlignment.CENTER, horizontal_alignment=ft.CrossAxisAlignment.CENTER) + if hasattr(card, "retry_btn") and card.retry_btn: + card.retry_btn.disabled = True + try: + card.retry_btn.update() + except Exception: + pass + + def set_tab_loading(self, tab_container, message): + tab_container.content = ft.Column([ + ft.ProgressRing(width=30, height=30), + ft.Text(message, color=COLOR_TEXT_SUB, size=13) + ], alignment=ft.MainAxisAlignment.CENTER, horizontal_alignment=ft.CrossAxisAlignment.CENTER) + + def set_error(self, card, message): + card.content_container.content = ft.Text(f"Error: {message}", color=COLOR_ERROR) + if hasattr(card, "retry_btn") and card.retry_btn: + card.retry_btn.disabled = False + try: + card.retry_btn.update() + except Exception: + pass + + def clear_loading(self, card): + if hasattr(card, "retry_btn") and card.retry_btn: + card.retry_btn.disabled = False + try: + card.retry_btn.update() + except Exception: + pass + + def start_individual_fetch(self, key, section, message, target): + self.fetch_btn.disabled = True + self.fetch_btn.update() + self.fetch_statuses[key] = "pending" + self.set_loading(section, message) + section.update() + threading.Thread(target=target, daemon=True).start() + + def handle_retry_skus(self, e): + self.start_individual_fetch("sku", self.sku_section, "Fetching SKU inventories...", self.fetch_skus) + + def handle_retry_o365(self, e): + self.start_individual_fetch("o365", self.o365_section, "Downloading O365 Active User reports...", self.fetch_o365) + + def handle_retry_trend(self, e): + self.start_individual_fetch("trend", self.trend_section, "Downloading O365 Trend report...", self.fetch_trend) + + def handle_retry_m365(self, e): + self.start_individual_fetch("m365", self.m365_section, "Downloading M365 App reports...", self.fetch_m365) + + def handle_retry_exchange(self, e): + self.fetch_btn.disabled = True + self.fetch_btn.update() + self.fetch_statuses["mailbox"] = "pending" + self.fetch_statuses["calendar"] = "pending" + self.set_tab_loading(self.emails_container, "Downloading Mailbox reports...") + self.set_tab_loading(self.calendar_container, "Querying Calendar settings...") + self.exchange_section.update() + threading.Thread(target=self.fetch_mailbox, daemon=True).start() + threading.Thread(target=self.fetch_calendar, daemon=True).start() + + def handle_retry_files(self, e): + self.fetch_btn.disabled = True + self.fetch_btn.update() + self.fetch_statuses["sharepoint"] = "pending" + self.fetch_statuses["onedrive"] = "pending" + self.set_tab_loading(self.sharepoint_container, "Downloading SharePoint reports...") + self.set_tab_loading(self.onedrive_container, "Downloading OneDrive reports...") + self.files_section.update() + threading.Thread(target=self.fetch_sharepoint, daemon=True).start() + threading.Thread(target=self.fetch_onedrive, daemon=True).start() + + def handle_retry_labels(self, e): + self.start_individual_fetch("labels", self.labels_section, "Retrieving Sensitivity labels...", self.fetch_labels) + + def handle_retry_retention(self, e): + self.start_individual_fetch("retention", self.retention_section, "Retrieving Retention policies...", self.fetch_retention) + + def handle_retry_pa(self, e): + self.start_individual_fetch("pa", self.pa_section, "Scanning Power Automate flows...", self.fetch_pa) + + def handle_fetch(self, e): + self.fetch_btn.disabled = True + self.fetch_btn.content = ft.Text("Fetching...", color=ft.Colors.WHITE) + self.fetch_btn.update() + + # Initialize fetch statuses + self.fetch_statuses = { + "sku": "pending", + "o365": "pending", + "trend": "pending", + "m365": "pending", + "mailbox": "pending", + "calendar": "pending", + "sharepoint": "pending", + "onedrive": "pending", + "labels": "pending", + "retention": "pending", + "pa": "pending" + } + + # 1. SKUs + self.set_loading(self.sku_section, "Fetching SKU inventories...") + self.sku_section.update() + threading.Thread(target=self.fetch_skus, daemon=True).start() + + # 2. O365 Usage + self.set_loading(self.o365_section, "Downloading O365 Active User reports...") + self.o365_section.update() + threading.Thread(target=self.fetch_o365, daemon=True).start() + + # 3. O365 Trend + self.set_loading(self.trend_section, "Downloading O365 Trend report...") + self.trend_section.update() + threading.Thread(target=self.fetch_trend, daemon=True).start() + + # 4. M365 App Usage + self.set_loading(self.m365_section, "Downloading M365 App reports...") + self.m365_section.update() + threading.Thread(target=self.fetch_m365, daemon=True).start() + + # 5. Exchange Online (Mailbox & Calendar) + self.set_tab_loading(self.emails_container, "Downloading Mailbox reports...") + self.set_tab_loading(self.calendar_container, "Querying Calendar settings...") + self.exchange_section.update() + threading.Thread(target=self.fetch_mailbox, daemon=True).start() + threading.Thread(target=self.fetch_calendar, daemon=True).start() + + # 6. Files (SharePoint & OneDrive) + self.set_tab_loading(self.sharepoint_container, "Downloading SharePoint reports...") + self.set_tab_loading(self.onedrive_container, "Downloading OneDrive reports...") + self.files_section.update() + threading.Thread(target=self.fetch_sharepoint, daemon=True).start() + threading.Thread(target=self.fetch_onedrive, daemon=True).start() + + # 8 & 9. Sensitivity Labels & Retention Policies + self.set_loading(self.labels_section, "Retrieving Sensitivity labels...") + self.labels_section.update() + threading.Thread(target=self.fetch_labels, daemon=True).start() + + self.set_loading(self.retention_section, "Retrieving Retention policies...") + self.retention_section.update() + threading.Thread(target=self.fetch_retention, daemon=True).start() + + # 10. Power Automate + self.set_loading(self.pa_section, "Scanning Power Automate flows...") + self.pa_section.update() + threading.Thread(target=self.fetch_pa, daemon=True).start() + + def mark_complete(self, key, status): + self.fetch_statuses[key] = status + if key in ["mailbox", "calendar"]: + if self.fetch_statuses.get("mailbox") != "pending" and self.fetch_statuses.get("calendar") != "pending": + self.clear_loading(self.exchange_section) + if self.page: + self.exchange_section.update() + elif key in ["sharepoint", "onedrive"]: + if self.fetch_statuses.get("sharepoint") != "pending" and self.fetch_statuses.get("onedrive") != "pending": + self.clear_loading(self.files_section) + if self.page: + self.files_section.update() + self.check_all_done() + + def check_all_done(self): + if not self.fetch_statuses: + return + if "pending" not in self.fetch_statuses.values(): + self.fetch_btn.disabled = False + self.fetch_btn.content = ft.Text("Fetch Report", weight=ft.FontWeight.BOLD) + if self.page: + self.fetch_btn.update() + + # --- Fetching Logic --- + + def fetch_skus(self): + try: + client = GraphClient(tenant_id=self.tenant, client_ids=self.client, client_secrets=self.secret, concurrency=1, retries=30, backoff=2) + client.authenticate(required_scopes=["Organization.Read.All", "Directory.Read.All"]) + dir_service = DirectoryService(client) + sku_data = dir_service.get_subscribed_skus() + client.close() + + items = sku_data.get("value", []) + self.last_licenses_items = items + + if not items: + self.sku_section.content_container.content = ft.Text("No subscribed product configurations found.", color=COLOR_TEXT_SUB) + self.export_sku_btn.disabled = True + else: + items.sort(key=lambda x: len(x.get("servicePlans", [])), reverse=True) + rows = [] + for item in items: + prepaid = item.get("prepaidUnits", {}) + p_str = f"Enabled: {prepaid.get('enabled', 0):,}" + if prepaid.get('warning', 0) > 0: p_str += f"\nWarn: {prepaid.get('warning'):,}" + if prepaid.get('suspended', 0) > 0: p_str += f"\nSusp: {prepaid.get('suspended'):,}" + rows.append(ft.DataRow(cells=[ + ft.DataCell(ft.Text(item.get("skuPartNumber", "UNKNOWN_SKU"), weight=ft.FontWeight.BOLD)), + ft.DataCell(ft.Text(p_str)), + ft.DataCell(ft.Text(f"{item.get('consumedUnits', 0):,}")) + ])) + + table = ft.DataTable( + columns=[ + ft.DataColumn(ft.Text("SKU Part Number", weight=ft.FontWeight.BOLD)), + ft.DataColumn(ft.Text("Units", weight=ft.FontWeight.BOLD)), + ft.DataColumn(ft.Text("Consumed Units", weight=ft.FontWeight.BOLD)), + ], + rows=rows, + border=ft.Border.all(1, COLOR_OUTLINE_LIGHT), + border_radius=8, + heading_row_color=COLOR_TONAL_BG, + ) + self.sku_section.content_container.content = ft.Column([table], scroll=ft.ScrollMode.AUTO, height=300) + self.export_sku_btn.disabled = False + + self.mark_complete("sku", "success") + except Exception as e: + self.set_error(self.sku_section, str(e)) + self.export_sku_btn.disabled = True + self.mark_complete("sku", "error") + finally: + self.clear_loading(self.sku_section) + if self.page: + self.sku_section.update() + self.export_sku_btn.update() + + def fetch_o365(self): + try: + try: + o365_data = usage.run_o365_pipeline(self.client, self.secret, self.tenant) + except Exception as o365_err: + script_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + local_file = os.path.join(script_dir, "telemetry", "reports", f"{self.tenant}_{self.client}", "Office365ActiveUserDetail(180d).csv") + if not os.path.exists(local_file): + local_file = os.path.join(script_dir, "telemetry", "reports", "Office365ActiveUserDetail(180d).csv") + if os.path.exists(local_file): + print(f"Falling back to local O365 file: {local_file}") + o365_data = usage.process_active_user_detail(local_file) + else: + raise o365_err + if not o365_data: + self.o365_section.content_container.content = ft.Text("No O365 usage data found.", color=COLOR_TEXT_SUB) + else: + rows = [] + for row_data in o365_data: + rows.append(ft.DataRow(cells=[ + ft.DataCell(ft.Text(str(row_data[0]), weight=ft.FontWeight.BOLD)), + ft.DataCell(ft.Text(f"{row_data[1]:,}")), + ft.DataCell(ft.Text(f"{row_data[2]:,}")), + ft.DataCell(ft.Text(f"{row_data[3]:,}")) + ])) + + table = ft.DataTable( + columns=[ + ft.DataColumn(ft.Text("Service", weight=ft.FontWeight.BOLD)), + ft.DataColumn(ft.Text("30 Days", weight=ft.FontWeight.BOLD)), + ft.DataColumn(ft.Text("90 Days", weight=ft.FontWeight.BOLD)), + ft.DataColumn(ft.Text("180 Days", weight=ft.FontWeight.BOLD)), + ], + rows=rows, + border=ft.Border.all(1, COLOR_OUTLINE_LIGHT), + border_radius=8, + heading_row_color=COLOR_TONAL_BG, + ) + self.o365_section.content_container.content = table + + self.mark_complete("o365", "success") + except Exception as e: + self.set_error(self.o365_section, str(e)) + self.mark_complete("o365", "error") + finally: + self.clear_loading(self.o365_section) + if self.page: + self.o365_section.update() + + def fetch_trend(self): + try: + try: + trend_data = usage.run_o365_trend_pipeline(self.client, self.secret, self.tenant) + except Exception as trend_err: + script_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + local_file = os.path.join(script_dir, "telemetry", "reports", f"{self.tenant}_{self.client}", "Office365ActiveUserCounts(30d).csv") + if not os.path.exists(local_file): + local_file = os.path.join(script_dir, "telemetry", "reports", "Office365ActiveUserCounts(30d).csv") + if os.path.exists(local_file): + print(f"Falling back to local trend file: {local_file}") + trend_data = usage.process_active_user_counts(local_file) + else: + raise trend_err + if not trend_data or not trend_data.get("dates"): + self.trend_section.content_container.content = ft.Text("No O365 trend data found.", color=COLOR_TEXT_SUB) + else: + dates = trend_data["dates"] + datasets = { + "Office 365": trend_data["office365"], + "Exchange": trend_data["exchange"], + "OneDrive": trend_data["onedrive"], + "SharePoint": trend_data["sharepoint"], + "Teams": trend_data["teams"] + } + colors = { + "Office 365": COLOR_PRIMARY, + "Exchange": "#C2410C", + "OneDrive": "#3B82F6", + "SharePoint": "#15803D", + "Teams": "#9333EA" + } + + chart = CustomLineChart(dates=dates, datasets=datasets, colors=colors, height=300) + + # Legend container + def legend_item(label, color): + return ft.Row([ + ft.Container(width=12, height=12, bgcolor=color, border_radius=3), + ft.Text(label, size=12, weight=ft.FontWeight.W_500, color=COLOR_TEXT_MAIN) + ], spacing=5) + + legend = ft.Row([ + legend_item("Office 365", colors["Office 365"]), + legend_item("Exchange", colors["Exchange"]), + legend_item("OneDrive", colors["OneDrive"]), + legend_item("SharePoint", colors["SharePoint"]), + legend_item("Teams", colors["Teams"]), + ], alignment=ft.MainAxisAlignment.CENTER, spacing=20) + + self.trend_section.content_container.content = ft.Column([ + chart, + ft.Divider(height=10, color="transparent"), + legend + ]) + + self.mark_complete("trend", "success") + except Exception as e: + self.set_error(self.trend_section, str(e)) + self.mark_complete("trend", "error") + finally: + self.clear_loading(self.trend_section) + if self.page: + self.trend_section.update() + + def fetch_m365(self): + try: + try: + m365_data = usage.run_m365_pipeline(self.client, self.secret, self.tenant) + except Exception as m365_err: + script_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + local_file = os.path.join(script_dir, "telemetry", "reports", f"{self.tenant}_{self.client}", "M365AppUserDetail(180d).csv") + if not os.path.exists(local_file): + local_file = os.path.join(script_dir, "telemetry", "reports", "M365AppUserDetail(180d).csv") + if os.path.exists(local_file): + print(f"Falling back to local M365 file: {local_file}") + m365_data = usage.process_m365_app_user_detail(local_file) + else: + raise m365_err + if not m365_data: + self.m365_section.content_container.content = ft.Text("No M365 App usage data found.", color=COLOR_TEXT_SUB) + else: + rows = [] + for row in m365_data: + rows.append(ft.DataRow(cells=[ + ft.DataCell(ft.Text(str(row[0]), weight=ft.FontWeight.BOLD)), + ft.DataCell(ft.Text(f"{row[1]:,}")) + ])) + + table = ft.DataTable( + columns=[ + ft.DataColumn(ft.Text("App / Platform", weight=ft.FontWeight.BOLD)), + ft.DataColumn(ft.Text("Users Count", weight=ft.FontWeight.BOLD)), + ], + rows=rows, + border=ft.Border.all(1, COLOR_OUTLINE_LIGHT), + border_radius=8, + heading_row_color=COLOR_TONAL_BG, + ) + self.m365_section.content_container.content = ft.Column([table], scroll=ft.ScrollMode.AUTO, height=300) + + self.mark_complete("m365", "success") + except Exception as e: + self.set_error(self.m365_section, str(e)) + self.mark_complete("m365", "error") + finally: + self.clear_loading(self.m365_section) + if self.page: + self.m365_section.update() + + def fetch_mailbox(self): + try: + data = run_mailbox_usage_pipeline(self.client, self.secret, self.tenant) + rows_data = [ + ("Total Mailboxes Analyzed", f"{data.get('total_mailboxes', 0):,} Mailboxes"), + ("Total Size of All Mailboxes", data.get("total_storage_formatted", "0.00 Bytes")), + ("Average Mailbox Size", data.get("average_mailbox_size_formatted", "0.00 Bytes")), + ("Total Number of Emails", f"{data.get('total_emails', 0):,} Emails"), + ("Average Emails per User", f"{data.get('average_emails', 0.0):,.0f} Emails") + ] + if data.get("has_powershell"): + rows_data += [ + ("Shared Mailboxes Count", f"{data.get('shared_mailboxes_count', 0):,} Shared Mailboxes"), + ("Total Shared Mailbox Size", data.get('shared_mailboxes_total_formatted', "0.00 Bytes")), + ("Public Folders Count", f"{data.get('public_folders_count', 0):,} Public Folders"), + ("Total Public Folder Size", data.get('public_folders_total_formatted', "0.00 Bytes")) + ] + + rows = [ft.DataRow(cells=[ + ft.DataCell(ft.Text(str(r[0]), weight=ft.FontWeight.BOLD)), + ft.DataCell(ft.Text(str(r[1]))) + ]) for r in rows_data] + + table = ft.DataTable( + columns=[ + ft.DataColumn(ft.Text("Mailbox Metric Description", weight=ft.FontWeight.BOLD)), + ft.DataColumn(ft.Text("Value / Measurement", weight=ft.FontWeight.BOLD)), + ], + rows=rows, + border=ft.Border.all(1, COLOR_OUTLINE_LIGHT), + border_radius=8, + heading_row_color=COLOR_TONAL_BG, + ) + + # Check warnings + if data.get("powershell_error"): + self.emails_container.content = ft.Column([ + ft.Text(f"⚠️ Warning: PowerShell stats failed. {data['powershell_error']}", color=COLOR_WARNING, size=13), + table + ], scroll=ft.ScrollMode.AUTO, height=300) + else: + self.emails_container.content = ft.Column([table], scroll=ft.ScrollMode.AUTO, height=300) + + self.mark_complete("mailbox", "success") + except Exception as e: + self.emails_container.content = ft.Text(f"Error: {e}", color=COLOR_ERROR) + self.mark_complete("mailbox", "error") + finally: + if self.page: + self.emails_container.update() + + def fetch_calendar(self): + try: + data = run_calendar_telemetry_pipeline(self.client, self.secret, self.tenant) + + rooms_err = data.get("RoomsError") + devs_err = data.get("DevicesError") + rooms_count = data.get("RoomsCount", 0) + equip_count = data.get("EquipmentCount", 0) + + if rooms_err and devs_err: + res_val = rooms_err + else: + r_str = "Error" if rooms_err else str(rooms_count) + e_str = "Error" if devs_err else str(equip_count) + tot = "Error" if (rooms_err or devs_err) else str(rooms_count + equip_count) + res_val = f"Total: {tot} ({r_str} Rooms, {e_str} Equipment)" + + reserve_val = data.get("CanUsersReserveRooms") + if isinstance(reserve_val, bool): + reserve_val = "Yes" if reserve_val else "No" + + att_val = data.get("CanShareAttachments") + if isinstance(att_val, bool): + attachments_val = "Yes" if att_val else "No" + else: + attachments_val = att_val + + rows_data = [ + ("Room & Resource Reservation", reserve_val), + ("Calendar Resources", res_val), + ("Integrated Calendar Apps", data.get("IntegratedCalendarApps") or "None found"), + ("Resource Naming Convention", data.get("NamingConvention") or "None found"), + ("Calendar Attachments Enabled", attachments_val), + ] + + rows = [ft.DataRow(cells=[ + ft.DataCell(ft.Text(str(r[0]), weight=ft.FontWeight.BOLD)), + ft.DataCell(ft.Text(str(r[1]))) + ]) for r in rows_data] + + table = ft.DataTable( + columns=[ + ft.DataColumn(ft.Text("Calendar Configuration / Metric", weight=ft.FontWeight.BOLD)), + ft.DataColumn(ft.Text("Value / Configuration", weight=ft.FontWeight.BOLD)), + ], + rows=rows, + border=ft.Border.all(1, COLOR_OUTLINE_LIGHT), + border_radius=8, + heading_row_color=COLOR_TONAL_BG, + ) + + # Check for warnings + if data.get("powershell_error"): + self.calendar_container.content = ft.Column([ + ft.Text(f"⚠️ Warning: Exchange PowerShell query failed: {data['powershell_error']}", color=COLOR_WARNING, size=13), + table + ], scroll=ft.ScrollMode.AUTO, height=300) + else: + self.calendar_container.content = ft.Column([table], scroll=ft.ScrollMode.AUTO, height=300) + + self.mark_complete("calendar", "success") + except Exception as e: + self.calendar_container.content = ft.Text(f"Error: {e}", color=COLOR_ERROR) + self.mark_complete("calendar", "error") + finally: + if self.page: + self.calendar_container.update() + + def fetch_sharepoint(self): + try: + data = run_sharepoint_pipeline(self.client, self.secret, self.tenant) + rows_data = [ + ("Total Sites Count", f"{data.get('total_sites', 0):,} Sites"), + ("Total Storage Used", data.get("total_storage_formatted", "0.00 Bytes")), + ("Total Files Stored", f"{data.get('total_files', 0):,} Files"), + ("Active Files Count", f"{data.get('active_files', 0):,} Files ({data.get('active_files_pct', 0.0):.1f}%)") + ] + rows = [ft.DataRow(cells=[ + ft.DataCell(ft.Text(str(r[0]), weight=ft.FontWeight.BOLD)), + ft.DataCell(ft.Text(str(r[1]))) + ]) for r in rows_data] + + table = ft.DataTable( + columns=[ + ft.DataColumn(ft.Text("SharePoint Site Metric Description", weight=ft.FontWeight.BOLD)), + ft.DataColumn(ft.Text("Value / Measurement", weight=ft.FontWeight.BOLD)), + ], + rows=rows, + border=ft.Border.all(1, COLOR_OUTLINE_LIGHT), + border_radius=8, + heading_row_color=COLOR_TONAL_BG, + ) + self.sharepoint_container.content = ft.Column([table], scroll=ft.ScrollMode.AUTO, height=200) + self.mark_complete("sharepoint", "success") + except Exception as e: + self.sharepoint_container.content = ft.Text(f"Error: {e}", color=COLOR_ERROR) + self.mark_complete("sharepoint", "error") + finally: + if self.page: + self.sharepoint_container.update() + + def fetch_onedrive(self): + try: + data = run_onedrive_pipeline(self.client, self.secret, self.tenant) + rows_data = [ + ("Total Accounts Count", f"{data.get('total_accounts', 0):,} Accounts"), + ("Total Storage Used", data.get("total_storage_formatted", "0.00 Bytes")), + ("Total Files Stored", f"{data.get('total_files', 0):,} Files"), + ("Active Files Count", f"{data.get('active_files', 0):,} Files ({data.get('active_files_pct', 0.0):.1f}%)"), + ("Users with Synced Files", f"{data.get('sync_users', 0):,} Users ({data.get('sync_users_pct', 0.0):.1f}%)"), + ("OneNote Active Users", f"{data.get('onenote_users', 0):,} Users") + ] + rows = [ft.DataRow(cells=[ + ft.DataCell(ft.Text(str(r[0]), weight=ft.FontWeight.BOLD)), + ft.DataCell(ft.Text(str(r[1]))) + ]) for r in rows_data] + + table = ft.DataTable( + columns=[ + ft.DataColumn(ft.Text("OneDrive Metric Description", weight=ft.FontWeight.BOLD)), + ft.DataColumn(ft.Text("Value / Measurement", weight=ft.FontWeight.BOLD)), + ], + rows=rows, + border=ft.Border.all(1, COLOR_OUTLINE_LIGHT), + border_radius=8, + heading_row_color=COLOR_TONAL_BG, + ) + self.onedrive_container.content = ft.Column([table], scroll=ft.ScrollMode.AUTO, height=250) + self.mark_complete("onedrive", "success") + except Exception as e: + self.onedrive_container.content = ft.Text(f"Error: {e}", color=COLOR_ERROR) + self.mark_complete("onedrive", "error") + finally: + if self.page: + self.onedrive_container.update() + + def fetch_labels(self): + try: + from telemetry.data_security_governance import fetch_sensitivity_labels_data + res = fetch_sensitivity_labels_data(self.client, self.secret, self.tenant) + labels = res.get("labels") + err = res.get("error") + + # Populate Sensitivity Labels pagination data + self.flattened_labels = [] + if err: + self.labels_section.content_container.content = ft.Text(f"Error loading labels: {err}", color=COLOR_ERROR) + self.labels_pagination_row.visible = False + self.mark_complete("labels", "error") + elif not labels: + self.labels_section.content_container.content = ft.Text("No Sensitivity Labels configured in this tenant.", color=COLOR_TEXT_SUB) + self.labels_pagination_row.visible = False + self.mark_complete("labels", "success") + else: + for parent in labels: + self.flattened_labels.append({ + "name": parent.get("name", "N/A"), + "description": parent.get("description", "") or parent.get("toolTip", "") or "N/A", + "hasProtection": parent.get("hasProtection", False), + "applicationMode": parent.get("applicationMode", "N/A") or "N/A", + "priority": parent.get("priority", 0), + "applicableTo": parent.get("applicableTo", ""), + "isEnabled": parent.get("isEnabled", True), + "is_sublabel": False + }) + sublabels = parent.get("sublabels", []) + if sublabels: + sublabels_sorted = sorted(sublabels, key=lambda x: x.get("priority", 0), reverse=True) + for sub in sublabels_sorted: + self.flattened_labels.append({ + "name": f" ↳ {sub.get('name', 'N/A')}", + "description": sub.get("description", "") or sub.get("toolTip", "") or "N/A", + "hasProtection": sub.get("hasProtection", False), + "applicationMode": sub.get("applicationMode", "N/A") or "N/A", + "priority": sub.get("priority", 0), + "applicableTo": sub.get("applicableTo", ""), + "isEnabled": sub.get("isEnabled", True), + "is_sublabel": True + }) + self.current_labels_page = 0 + self.render_labels_page() + self.mark_complete("labels", "success") + except Exception as e: + self.set_error(self.labels_section, str(e)) + self.labels_pagination_row.visible = False + self.mark_complete("labels", "error") + finally: + self.clear_loading(self.labels_section) + if self.page: + self.labels_section.update() + + def fetch_retention(self): + try: + from telemetry.data_security_governance import fetch_retention_policies_data + res = fetch_retention_policies_data(self.client, self.secret, self.tenant) + policies = res.get("policies") + err = res.get("error") + + # Populate Retention Policies + if err: + msg = err + if "powershell" in err.lower() or "pwsh" in err.lower(): + msg = "PowerShell Core ('pwsh') is not installed or configured on this machine." + elif "exchangeonlinemanagement" in err.lower(): + msg = "ExchangeOnlineManagement PowerShell module is missing." + self.retention_section.content_container.content = ft.Text(f"Error loading policies: {msg}", color=COLOR_ERROR) + self.mark_complete("retention", "error") + elif not policies: + self.retention_section.content_container.content = ft.Text("No Retention Compliance Policies found.", color=COLOR_TEXT_SUB) + self.mark_complete("retention", "success") + else: + policies_list = policies if isinstance(policies, list) else [policies] + rows = [] + for policy in policies_list: + duration_val = str(policy.get("Duration", "N/A")) + duration_str = duration_val + if duration_val.lower() == "unlimited": + duration_str = "Keep Forever" + elif duration_val.isdigit(): + days = int(duration_val) + if days >= 365: + years = days / 365.0 + duration_str = f"{int(years)} Years ({days} days)" if years.is_integer() else f"{years:.1f} Years ({days} days)" + else: + duration_str = f"{days} days" + + trigger_val = policy.get("RetentionTrigger", "N/A") + if trigger_val and trigger_val != "N/A": + trigger_map = {"DateCreated": "created date", "DateModified": "last modified date", "DateLabeled": "labeled date"} + duration_str += f"\n(from {trigger_map.get(trigger_val, trigger_val)})" + + enabled_val = policy.get("Enabled", True) + is_enabled = enabled_val.lower() == "true" if isinstance(enabled_val, str) else bool(enabled_val) + status_str = "🟢 Enabled" if is_enabled else "🔴 Disabled" + + rows.append(ft.DataRow(cells=[ + ft.DataCell(ft.Column([ + ft.Text(policy.get("Name", "N/A"), weight=ft.FontWeight.BOLD), + ft.Text(policy.get("Comment", ""), size=11, color=COLOR_TEXT_SUB) if policy.get("Comment") else ft.Container() + ], alignment=ft.MainAxisAlignment.CENTER)), + ft.DataCell(ft.Text(policy.get("Workload", "N/A"))), + ft.DataCell(ft.Text(duration_str)), + ft.DataCell(ft.Text(policy.get("DistributionStatus", "Success"))), + ft.DataCell(ft.Text(status_str)) + ])) + + table = ft.DataTable( + columns=[ + ft.DataColumn(ft.Text("Policy Name", weight=ft.FontWeight.BOLD)), + ft.DataColumn(ft.Text("Workloads", weight=ft.FontWeight.BOLD)), + ft.DataColumn(ft.Text("Duration", weight=ft.FontWeight.BOLD)), + ft.DataColumn(ft.Text("Distribution", weight=ft.FontWeight.BOLD)), + ft.DataColumn(ft.Text("Status", weight=ft.FontWeight.BOLD)), + ], + rows=rows, + border=ft.Border.all(1, COLOR_OUTLINE_LIGHT), + border_radius=8, + heading_row_color=COLOR_TONAL_BG, + ) + self.retention_section.content_container.content = ft.Column([table], scroll=ft.ScrollMode.AUTO, height=300) + self.mark_complete("retention", "success") + except Exception as e: + self.set_error(self.retention_section, str(e)) + self.mark_complete("retention", "error") + finally: + self.clear_loading(self.retention_section) + if self.page: + self.retention_section.update() + + def render_labels_page(self): + total_items = len(self.flattened_labels) + total_pages = (total_items + self.labels_per_page - 1) // self.labels_per_page + if total_pages < 1: + total_pages = 1 + + start_idx = self.current_labels_page * self.labels_per_page + end_idx = min(start_idx + self.labels_per_page, total_items) + page_items = self.flattened_labels[start_idx:end_idx] + + rows = [] + for item in page_items: + protection = "🛡️ Yes" if item["hasProtection"] else "🔓 No" + status_str = "🟢 Enabled" if item["isEnabled"] else "🔴 Disabled" + + name_color = COLOR_TEXT_MAIN if not item["is_sublabel"] else COLOR_TEXT_SUB + name_weight = ft.FontWeight.BOLD if not item["is_sublabel"] else ft.FontWeight.NORMAL + + rows.append(ft.DataRow(cells=[ + ft.DataCell(ft.Text(item["name"], weight=name_weight, color=name_color)), + ft.DataCell(ft.Text(item["description"])), + ft.DataCell(ft.Text(protection)), + ft.DataCell(ft.Text(str(item["applicationMode"]).capitalize())), + ft.DataCell(ft.Text(str(item["priority"]))), + ft.DataCell(ft.Text(", ".join([x.capitalize() for x in item["applicableTo"].split(",") if x.strip()]) or "N/A")), + ft.DataCell(ft.Text(status_str)) + ])) + + table = ft.DataTable( + columns=[ + ft.DataColumn(ft.Text("Sensitivity Label", weight=ft.FontWeight.BOLD)), + ft.DataColumn(ft.Text("Description", weight=ft.FontWeight.BOLD)), + ft.DataColumn(ft.Text("Protection", weight=ft.FontWeight.BOLD)), + ft.DataColumn(ft.Text("Mode", weight=ft.FontWeight.BOLD)), + ft.DataColumn(ft.Text("Priority", weight=ft.FontWeight.BOLD)), + ft.DataColumn(ft.Text("Applicable Targets", weight=ft.FontWeight.BOLD)), + ft.DataColumn(ft.Text("Status", weight=ft.FontWeight.BOLD)), + ], + rows=rows, + border=ft.Border.all(1, COLOR_OUTLINE_LIGHT), + border_radius=8, + heading_row_color=COLOR_TONAL_BG, + ) + self.labels_section.content_container.content = ft.Column([table], scroll=ft.ScrollMode.AUTO, height=350) + + # Update Pagination Control status + self.labels_pagination_info.value = f"Page {self.current_labels_page + 1} of {total_pages}" + self.labels_prev_btn.disabled = (self.current_labels_page <= 0) + self.labels_next_btn.disabled = (self.current_labels_page >= total_pages - 1) + self.labels_pagination_row.visible = (total_items > self.labels_per_page) + + if self.page: + self.labels_section.update() + self.labels_pagination_row.update() + + def handle_labels_prev(self, e): + if self.current_labels_page > 0: + self.current_labels_page -= 1 + self.render_labels_page() + + def handle_labels_next(self, e): + total_items = len(self.flattened_labels) + total_pages = (total_items + self.labels_per_page - 1) // self.labels_per_page + if self.current_labels_page < total_pages - 1: + self.current_labels_page += 1 + self.render_labels_page() + + def fetch_pa(self): + try: + scanner = PowerAutomateScanner(self.tenant, self.client, self.secret) + results = scanner.scan_flows() + if not results: + self.pa_section.content_container.content = ft.Text("No Power Automate data found.", color=COLOR_TEXT_SUB) + self.export_pa_btn.disabled = True + else: + total_envs = results.get("total_environments", 0) + counts = results.get("counts", {}) + total_flows = counts.get("Cloud Flows", 0) + counts.get("Desktop Flows", 0) + premium_conns = results.get("premium_connectors", []) + custom_conns = results.get("custom_connectors", []) + self.last_complex_flows = results.get("complex_logic_flows", []) + + prem_str = ", ".join(premium_conns) if premium_conns else "0" + cust_str = ", ".join(custom_conns) if custom_conns else "0" + + rows_data = [ + ("Total Environments Scanned", str(total_envs)), + ("Total Flows (Active + Inactive)", str(total_flows)), + ("Premium Connectors In Use", prem_str), + ("Custom Connectors In Use", cust_str), + ] + rows = [ft.DataRow(cells=[ + ft.DataCell(ft.Text(str(r[0]), weight=ft.FontWeight.BOLD)), + ft.DataCell(ft.Text(str(r[1]))) + ]) for r in rows_data] + + table = ft.DataTable( + columns=[ + ft.DataColumn(ft.Text("Metric", weight=ft.FontWeight.BOLD)), + ft.DataColumn(ft.Text("Value", weight=ft.FontWeight.BOLD)), + ], + rows=rows, + border=ft.Border.all(1, COLOR_OUTLINE_LIGHT), + border_radius=8, + heading_row_color=COLOR_TONAL_BG, + ) + self.pa_section.content_container.content = table + self.export_pa_btn.disabled = (len(self.last_complex_flows) == 0) + + self.mark_complete("pa", "success") + except Exception as e: + self.set_error(self.pa_section, str(e)) + self.export_pa_btn.disabled = True + self.mark_complete("pa", "error") + finally: + self.clear_loading(self.pa_section) + if self.page: + self.pa_section.update() + self.export_pa_btn.update() + + # --- CSV Export Handlers --- + + def handle_export_skus(self, e): + def on_save_result(save_event: ft.FilePickerResultEvent): + if save_event.path: + try: + headers = ["SKU Part Number", "Units", "Consumed Units", "Included Service Plans", "Applies To"] + rows = [] + for item in self.last_licenses_items: + sku_name = item.get("skuPartNumber", "UNKNOWN_SKU") + prepaid = item.get("prepaidUnits", {}) + enabled_units = prepaid.get("enabled", 0) + warn_units = prepaid.get("warning", 0) + susp_units = prepaid.get("suspended", 0) + + prepaid_str = f"Enabled: {enabled_units:,}" + if warn_units > 0: prepaid_str += f"\nWarn: {warn_units:,}" + if susp_units > 0: prepaid_str += f"\nSusp: {susp_units:,}" + consumed_str = f"{item.get('consumedUnits', 0):,}" + + plans = item.get("servicePlans", []) + + if not plans: + rows.append([sku_name, prepaid_str, consumed_str, "None designated.", "-"]) + else: + for idx, p in enumerate(plans): + p_name = p.get("servicePlanName", "UnnamedPlan") + p_scope = p.get("appliesTo", "Unknown") + if idx == 0: + rows.append([sku_name, prepaid_str, consumed_str, p_name, p_scope]) + else: + rows.append(["", "", "", p_name, p_scope]) + + with open(save_event.path, 'w', newline='', encoding='utf-8') as csvfile: + writer = csv.writer(csvfile) + writer.writerow(headers) + writer.writerows(rows) + + except Exception as ex: + print(f"Failed to export SKUs: {ex}") + + picker = ft.FilePicker(on_result=on_save_result) + e.page.overlay.append(picker) + e.page.update() + ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + picker.save_file(file_name=f"licenses_inventory_{ts}.csv") + + def handle_export_pa(self, e): + def on_save_result(save_event: ft.FilePickerResultEvent): + if save_event.path: + try: + headers = ["Environment", "Name", "Type", "Tier", "Active", "Reason"] + rows = [] + for flow in self.last_complex_flows: + rows.append([ + flow.get("Environment"), + flow.get("Name"), + flow.get("Type"), + flow.get("Tier"), + flow.get("Active"), + flow.get("Reason") + ]) + + with open(save_event.path, 'w', newline='', encoding='utf-8') as csvfile: + writer = csv.writer(csvfile) + writer.writerow(headers) + writer.writerows(rows) + except Exception as ex: + print(f"Failed to export complex flows: {ex}") + + picker = ft.FilePicker(on_result=on_save_result) + e.page.overlay.append(picker) + e.page.update() + ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + picker.save_file(file_name=f"complex_flows_{ts}.csv") diff --git a/flet_app/main.py b/flet_app/main.py new file mode 100644 index 00000000..076184d2 --- /dev/null +++ b/flet_app/main.py @@ -0,0 +1,113 @@ +import flet as ft +import sys +import os + +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from flet_app.styles import get_theme, COLOR_BACKGROUND +from flet_app.auth_view import AuthView +from flet_app.cert_instructions_view import CertInstructionsView +from flet_app.dashboard import DashboardView +from core.cert_auth import check_certificate_exists, generate_certificate, load_certificate + +def main(page: ft.Page): + page.title = "Deal Assistant (Flet)" + page.theme_mode = ft.ThemeMode.LIGHT + page.theme = get_theme() + page.bgcolor = COLOR_BACKGROUND + page.padding = 20 + + # Store session variables + page.session.store.set("tenant", "") + page.session.store.set("client", "") + page.session.store.set("secret", "") + + def show_error_dialog(title, message): + def close_dialog(e): + dialog.open = False + page.update() + dialog = ft.AlertDialog( + title=ft.Text(title, weight=ft.FontWeight.BOLD), + content=ft.Text(message), + actions=[ft.TextButton("Close", on_click=close_dialog)], + actions_alignment=ft.MainAxisAlignment.END, + ) + page.overlay.append(dialog) + dialog.open = True + page.update() + + def show_auth(): + page.controls.clear() + page.add(AuthView(on_connect_clicked=handle_connect)) + page.update() + + def show_dashboard(): + tenant = page.session.store.get("tenant") + client = page.session.store.get("client") + secret = page.session.store.get("secret") + page.controls.clear() + page.add(DashboardView(tenant, client, secret, on_disconnect=handle_disconnect)) + page.update() + + def show_cert_instructions(pem_path): + client = page.session.store.get("client") + page.controls.clear() + page.add(CertInstructionsView(pem_path=pem_path, client_id=client, on_continue=handle_cert_continue)) + page.update() + + def handle_connect(tenant, client, secret): + page.session.store.set("tenant", tenant) + page.session.store.set("client", client) + page.session.store.set("secret", secret) + + if check_certificate_exists(tenant_id=tenant, client_id=client): + try: + # Decrypt the PFX certificate using the client secret + load_certificate(secret, tenant_id=tenant, client_id=client) + show_dashboard() + except Exception as e: + show_error_dialog( + "Certificate Decryption Error", + f"Unable to unlock certificate with Client Secret. Proceeding with standard Client Secret authentication fallback.\n\nError: {e}" + ) + show_dashboard() + else: + try: + # Generate new certificate and pfx encrypted with the client secret + pem_path, _ = generate_certificate(secret, tenant_id=tenant, client_id=client) + # Show instructions UI + show_cert_instructions(pem_path) + except Exception as e: + show_error_dialog( + "Certificate Generation Error", + f"Unable to generate certificate. Proceeding with standard Client Secret authentication fallback.\n\nError: {e}" + ) + show_dashboard() + + def handle_cert_continue(): + tenant = page.session.store.get("tenant") + client = page.session.store.get("client") + secret = page.session.store.get("secret") + try: + load_certificate(secret, tenant_id=tenant, client_id=client) + except Exception as e: + show_error_dialog( + "Certificate Verification Error", + f"Unable to verify certificate. Proceeding with standard Client Secret authentication fallback.\n\nError: {e}" + ) + show_dashboard() + + def handle_disconnect(): + page.session.store.set("tenant", "") + page.session.store.set("client", "") + page.session.store.set("secret", "") + show_auth() + + # Start app on auth page + show_auth() + +if __name__ == "__main__": + # Ensure matplotlib backend doesn't crash Flet if graph operations are pulled + import matplotlib + matplotlib.use("Agg") + + ft.run(main) diff --git a/flet_app/sidebar.py b/flet_app/sidebar.py new file mode 100644 index 00000000..841ef69e --- /dev/null +++ b/flet_app/sidebar.py @@ -0,0 +1,61 @@ +import flet as ft +from flet_app.styles import * + +class Sidebar(ft.Container): + def __init__(self, on_disconnect): + super().__init__() + self.on_disconnect = on_disconnect + + self.width = 300 + self.bgcolor = COLOR_SURFACE + self.border_radius = 12 + self.border = ft.Border.all(1, COLOR_OUTLINE_LIGHT) + self.padding = 20 + + menu_items = [ + ("Usage and adoption", ft.Icons.BAR_CHART, True), + ("Workforce analysis", ft.Icons.PEOPLE, False), + ("Cost savings plan", ft.Icons.ATTACH_MONEY, False), + ("Migration planner", ft.Icons.ROCKET_LAUNCH, False) + ] + + self.menu_column = ft.Column(spacing=10) + + for label, icon, is_active in menu_items: + bg = COLOR_TONAL_BG if is_active else "transparent" + text_col = COLOR_PRIMARY if is_active else COLOR_TEXT_SUB + weight = ft.FontWeight.BOLD if is_active else ft.FontWeight.NORMAL + + btn = ft.Container( + content=ft.Row([ + ft.Icon(icon, color=text_col, size=20), + ft.Text(label, color=text_col, weight=weight, size=14) + ]), + bgcolor=bg, + padding=ft.Padding.symmetric(horizontal=15, vertical=12), + border_radius=8, + ink=True if not is_active else False, + ) + self.menu_column.controls.append(btn) + + self.content = ft.Column( + controls=[ + ft.Row([ + ft.Text("🤝", size=24), + ft.Text("Deal Assistant", size=18, weight=ft.FontWeight.BOLD, color=COLOR_TEXT_MAIN) + ], alignment=ft.MainAxisAlignment.START), + ft.Divider(height=30, color="transparent"), + self.menu_column, + ft.Container(expand=True), # Spacer + ft.Container( + content=ft.Row([ + ft.Icon(ft.Icons.LOGOUT, color=COLOR_ERROR, size=20), + ft.Text("Disconnect", color=COLOR_ERROR, weight=ft.FontWeight.W_500, size=14) + ]), + padding=ft.Padding.symmetric(horizontal=15, vertical=12), + border_radius=8, + ink=True, + on_click=lambda _: self.on_disconnect() + ) + ] + ) diff --git a/flet_app/styles.py b/flet_app/styles.py new file mode 100644 index 00000000..9381c155 --- /dev/null +++ b/flet_app/styles.py @@ -0,0 +1,27 @@ +import flet as ft + +COLOR_PRIMARY = "#1E3A8A" +COLOR_PRIMARY_HOVER = "#172554" +COLOR_SECONDARY = "#3B82F6" +COLOR_SECONDARY_HOVER = "#DBEAFE" +COLOR_BACKGROUND = "#F8FAFC" +COLOR_SURFACE = "#FFFFFF" +COLOR_SURFACE_VARIANT = "#F1F5F9" +COLOR_ERROR = "#DC2626" +COLOR_SUCCESS = "#10B981" +COLOR_WARNING = "#F59E0B" +COLOR_TEXT_MAIN = "#0F172A" +COLOR_TEXT_SUB = "#64748B" +COLOR_OUTLINE = "#CBD5E1" +COLOR_OUTLINE_LIGHT = "#E2E8F0" +COLOR_TONAL_BG = "#EFF6FF" +COLOR_TONAL_TEXT = "#1E40AF" + +def get_theme(): + return ft.Theme( + color_scheme=ft.ColorScheme( + primary=COLOR_PRIMARY, + surface=COLOR_SURFACE, + error=COLOR_ERROR, + ) + ) diff --git a/migration_planner.py b/migration_planner.py index 656204a3..042381b3 100644 --- a/migration_planner.py +++ b/migration_planner.py @@ -54,6 +54,11 @@ import subprocess import sys import customtkinter as ctk + +# Performance optimizations for CustomTkinter across OS +ctk.set_window_scaling(1.0) +ctk.set_widget_scaling(1.0) + from util.constants import * class SelectorApp(ctk.CTk): diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..472e18c3 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,13 @@ +customtkinter +pandas +requests +reportlab +matplotlib +psutil +aiohttp +sortedcontainers +msal +aiosqlite +Pillow +urllib3 +certifi diff --git a/scripts/app_creation_script.ps1 b/scripts/app_creation_script.ps1 new file mode 100644 index 00000000..39ddf75f --- /dev/null +++ b/scripts/app_creation_script.ps1 @@ -0,0 +1,301 @@ +<# +.SYNOPSIS +Automates the creation of a Single-Tenant Entra ID App for Workspace Migration and Deal Assistant Telemetry. +Strictly forces account selection, verifies Admin roles, configures Graph & Exchange API permissions, grants admin consent, +and automatically assigns required Entra ID Directory Roles to the Service Principal. +#> + +# Check if the Microsoft Graph module is installed +if (-not (Get-Module -ListAvailable -Name Microsoft.Graph.Authentication)) { + Write-Host "Microsoft Graph module is NOT installed." -ForegroundColor Yellow + $UserResponse = Read-Host "Would you like to try installing Microsoft Graph? (Y/N)" + + if ($UserResponse -ieq "Y") { + try { + Install-Module -Name Microsoft.Graph -Scope CurrentUser -Force -AllowClobber + Write-Host "Installation complete!" -ForegroundColor Green + } + catch { + Write-Error "Policy is blocking installation. Please contact IT to install Microsoft.Graph module." + Read-Host "Press Enter to exit"; exit + } + } + else { + exit + } +} else { + Write-Host "Microsoft Graph modules detected. Proceeding..." -ForegroundColor Green +} + +# --- STEP 0: THE "DEEP" LOGOUT --- +Write-Host "Forcing session cleanup..." -ForegroundColor Gray +Disconnect-MgGraph -ErrorAction SilentlyContinue + +# Force clear the local token cache folder if it exists +$CachePath = "$env:USERPROFILE\.mg" +if (Test-Path $CachePath) { + try { Remove-Item $CachePath -Recurse -Force -ErrorAction SilentlyContinue } catch {} +} + +Write-Host "Opening Microsoft Login... (Please select an administrative account)" -ForegroundColor Cyan + +$RequiredScopes = @( + "Application.ReadWrite.All", + "AppRoleAssignment.ReadWrite.All", + "Directory.Read.All", + "RoleManagement.Read.Directory", + "RoleManagement.ReadWrite.Directory" +) + +try { + Connect-MgGraph -Scopes $RequiredScopes -ContextScope Process + + $Context = Get-MgContext + if ($null -eq $Context) { throw "Login was cancelled or failed." } + + $UserPrincipal = $Context.Account + Write-Host "Logged in as: $UserPrincipal" -ForegroundColor Green + + # --- ROLE VALIDATION --- + Write-Host "Verifying Directory Roles of authenticating user..." -ForegroundColor Gray + $UserRoles = Get-MgUserMemberOf -UserId $Context.Account -All | Where-Object { $_.AdditionalProperties.displayName -ne $null } + + $Authorized = $false + $RequiredRoles = @("Global Administrator", "Privileged Role Administrator") + + foreach ($role in $UserRoles) { + $roleName = $role.AdditionalProperties.displayName + if ($roleName -in $RequiredRoles) { + $Authorized = $true + Write-Host "Access Granted: $roleName" -ForegroundColor Green + break + } + } + + if (-not $Authorized) { + Write-Host "`nCRITICAL ERROR: Insufficient Privileges." -ForegroundColor Red + Write-Host "Account must be 'Global Administrator' or 'Privileged Role Administrator'." -ForegroundColor Yellow + Disconnect-MgGraph + Read-Host "`nPress Enter to exit"; exit + } + +} catch { + Write-Error "Login failed: $_" + Read-Host "Press Enter to exit"; exit +} + +# --- USER INPUT --- +Write-Host "`n--- APPLICATION SETUP ---" -ForegroundColor Cyan +$InputName = Read-Host "Enter the name for your new Entra ID Application (Default: Workspace Migration App)" +$AppName = if ([string]::IsNullOrWhiteSpace($InputName)) { "Workspace Migration App" } else { $InputName } + +# --- CONFIGURATION --- +$GraphAppRoles = @( + "Reports.Read.All", "Directory.Read.All", "Policy.Read.All", "NetworkAccess.Read.All", + "DeviceManagementConfiguration.Read.All", "DeviceManagementServiceConfig.Read.All", + "DeviceManagementApps.Read.All", "DeviceManagementManagedDevices.Read.All", + "Organization.Read.All", "Place.Read.All", "Calendars.ReadBasic.All", "Sites.Read.All", + "AuditLog.Read.All", "SensitivityLabels.Read.All", "Application.Read.All", "User.Read.All", + "Group.Read.All", "Mail.Read", "Contacts.Read", "Calendars.Read", "MailboxFolder.Read.All", + "MailboxSettings.Read", "Chat.Read.All", "ChannelMessage.Read.All", "ChannelSettings.Read.All", + "TeamsActivity.Read.All", "TeamMember.Read.All", "Files.Read.All", "LicenseAssignment.Read.All" +) + +$GraphDelegatedScopes = @( + "eDiscovery.Read.All", "Policy.Read.All", "offline_access" +) + +$ExchangeAppRoles = @( + "Exchange.ManageAsApp", "Exchange.ManageAsAppV2" +) + +$DirectoryRolesToAssign = @( + "Global Reader", + "Compliance Administrator", + "Compliance Data Administrator" +) + +$TenantId = $Context.TenantId + +try { + # --- STEP 1: REGISTER APPLICATION --- + Write-Host "Creating Application: $AppName..." -ForegroundColor Cyan + $Application = New-MgApplication -BodyParameter @{ + displayName = $AppName + signInAudience = "AzureADMyOrg" + web = @{ + redirectUris = @("http://localhost") + } + } + + # --- STEP 2: PREPARE SERVICE PRINCIPAL --- + $NewServicePrincipal = New-MgServicePrincipal -BodyParameter @{ appId = $Application.AppId } + + Write-Host "Waiting 10 seconds for service principal replication..." -ForegroundColor DarkGray + Start-Sleep -Seconds 10 + + # --- STEP 3: CONFIGURE & GRANT API PERMISSIONS --- + Write-Host "Configuring API Permissions & Granting Admin Consent..." -ForegroundColor Cyan + + $AllRequiredResourceAccess = @() + + # 1. Graph API Permissions + $GraphSP = Get-MgServicePrincipal -Filter "AppId eq '00000003-0000-0000-c000-000000000000'" | Select-Object -First 1 + $GraphResourceAccess = @() + + foreach ($RoleName in $GraphAppRoles) { + $Role = $GraphSP.AppRoles | Where-Object { $_.Value -eq $RoleName } + if ($Role) { + $GraphResourceAccess += @{ id = $Role.Id; type = "Role" } + New-MgServicePrincipalAppRoleAssignment -ServicePrincipalId $NewServicePrincipal.Id -BodyParameter @{ + principalId = $NewServicePrincipal.Id; resourceId = $GraphSP.Id; appRoleId = $Role.Id + } | Out-Null + Write-Host " - Granted App Role (Graph): $RoleName" -ForegroundColor Gray + } + } + + foreach ($ScopeName in $GraphDelegatedScopes) { + $Scope = $GraphSP.Oauth2PermissionScopes | Where-Object { $_.Value -eq $ScopeName } + if ($Scope) { + $GraphResourceAccess += @{ id = $Scope.Id; type = "Scope" } + Write-Host " - Added Delegated Scope (Graph): $ScopeName" -ForegroundColor Gray + } + } + + if ($GraphResourceAccess.Count -gt 0) { + $AllRequiredResourceAccess += @{ resourceAppId = "00000003-0000-0000-c000-000000000000"; resourceAccess = $GraphResourceAccess } + } + + # 2. Exchange Online API Permissions + $ExchangeSP = Get-MgServicePrincipal -Filter "AppId eq '00000002-0000-0ff1-ce00-000000000000'" | Select-Object -First 1 + $ExchangeResourceAccess = @() + + if ($ExchangeSP) { + foreach ($RoleName in $ExchangeAppRoles) { + $Role = $ExchangeSP.AppRoles | Where-Object { $_.Value -eq $RoleName } + if ($Role) { + $ExchangeResourceAccess += @{ id = $Role.Id; type = "Role" } + New-MgServicePrincipalAppRoleAssignment -ServicePrincipalId $NewServicePrincipal.Id -BodyParameter @{ + principalId = $NewServicePrincipal.Id; resourceId = $ExchangeSP.Id; appRoleId = $Role.Id + } | Out-Null + Write-Host " - Granted App Role (Exchange): $RoleName" -ForegroundColor Gray + } + } + if ($ExchangeResourceAccess.Count -gt 0) { + $AllRequiredResourceAccess += @{ resourceAppId = "00000002-0000-0ff1-ce00-000000000000"; resourceAccess = $ExchangeResourceAccess } + } + } else { + Write-Host " - Warning: Exchange Online Service Principal not found. Skipping Exchange permissions." -ForegroundColor Yellow + } + + # Update the Application Registration with all configured scopes and roles + Update-MgApplication -ApplicationId $Application.Id -RequiredResourceAccess $AllRequiredResourceAccess + + # 3. Grant Admin Consent for Delegated Scopes + if ($GraphDelegatedScopes.Count -gt 0) { + $ScopeString = $GraphDelegatedScopes -join " " + New-MgOauth2PermissionGrant -BodyParameter @{ + clientId = $NewServicePrincipal.Id + consentType = "AllPrincipals" + resourceId = $GraphSP.Id + scope = $ScopeString + } | Out-Null + Write-Host " - Admin Consent Granted for Delegated Scopes" -ForegroundColor Gray + } + + # --- STEP 4: AUTOMATED DIRECTORY ROLE ASSIGNMENTS --- + Write-Host "`nAssigning Directory Roles to Service Principal..." -ForegroundColor Cyan + + foreach ($RoleName in $DirectoryRolesToAssign) { + try { + # Check if directory role is enabled in tenant + $Role = Get-MgDirectoryRole -Filter "displayName eq '$RoleName'" -ErrorAction SilentlyContinue + if (-not $Role) { + # Enable role from template if not instantiated + $Template = Get-MgDirectoryRoleTemplate -Filter "displayName eq '$RoleName'" -ErrorAction SilentlyContinue + if ($Template) { + $Role = New-MgDirectoryRole -RoleTemplateId $Template.Id -ErrorAction SilentlyContinue + } + } + + if ($Role) { + # Add service principal as member + New-MgDirectoryRoleMemberByRef -DirectoryRoleId $Role.Id -OdataId "https://graph.microsoft.com/v1.0/directoryObjects/$($NewServicePrincipal.Id)" -ErrorAction Stop + Write-Host " - Successfully Assigned Directory Role: $RoleName" -ForegroundColor Green + } else { + # Fallback via REST if template resolution requires raw REST API call + $RoleRest = (Invoke-MgRestMethod -Uri "https://graph.microsoft.com/v1.0/directoryRoles?`$filter=displayName eq '$RoleName'").value[0] + if (-not $RoleRest) { + $TemplateRest = (Invoke-MgRestMethod -Uri "https://graph.microsoft.com/v1.0/directoryRoleTemplates?`$filter=displayName eq '$RoleName'").value[0] + if ($TemplateRest) { + $RoleRest = Invoke-MgRestMethod -Uri "https://graph.microsoft.com/v1.0/directoryRoles" -Method Post -Body @{ roleTemplateId = $TemplateRest.id } + } + } + if ($RoleRest) { + $MemberBody = @{ "`@odata.id" = "https://graph.microsoft.com/v1.0/directoryObjects/$($NewServicePrincipal.Id)" } + Invoke-MgRestMethod -Uri "https://graph.microsoft.com/v1.0/directoryRoles/$($RoleRest.id)/members/`$ref" -Method Post -Body $MemberBody | Out-Null + Write-Host " - Successfully Assigned Directory Role (REST): $RoleName" -ForegroundColor Green + } else { + Write-Host " - Warning: Unable to resolve Directory Role template for '$RoleName'." -ForegroundColor Yellow + } + } + } catch { + if ($_.ToString() -match "already exists" -or $_.ToString() -match "Request_BadRequest") { + Write-Host " - Directory Role '$RoleName' is already assigned." -ForegroundColor DarkGreen + } else { + Write-Host " - Failed to assign Directory Role '$RoleName': $_" -ForegroundColor Yellow + } + } + } + + # --- STEP 5: CREATE CLIENT SECRET --- + Write-Host "`nGenerating Client Secret..." -ForegroundColor Cyan + $ExpiryDate = (Get-Date).AddYears(2).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ") + $PasswordCred = Add-MgApplicationPassword -ApplicationId $Application.Id -BodyParameter @{ + passwordCredential = @{ + displayName = "MigrationToolSecret" + endDateTime = $ExpiryDate + } + } + + # --- OUTPUT --- + Write-Host "`n-------------------------------------------------------" -ForegroundColor Yellow + Write-Host " SETUP COMPLETE - SAVE THESE DETAILS" -ForegroundColor Yellow + Write-Host "-------------------------------------------------------" -ForegroundColor Yellow + Write-Host "Application Name : $AppName" + Write-Host "Application (Client) ID : $($Application.AppId)" + Write-Host "Client Secret Value : $($PasswordCred.SecretText)" + Write-Host "Directory (Tenant) ID : $TenantId" + Write-Host "Directory Roles Assigned: $($DirectoryRolesToAssign -join ', ')" -ForegroundColor Green + Write-Warning "IMPORTANT: Copy the Client Secret Value immediately." + +} +catch { + Write-Error "Operation failed: $_" +} + +# --- STEP 6: POWER PLATFORM MANAGEMENT APP (OPTIONAL) --- +Write-Host "`n--- POWER AUTOMATE & DATAVERSE CONFIGURATION ---" -ForegroundColor Cyan +$PromptPower = Read-Host "Would you like to register this app for Power Automate telemetry? (Y/N)" +if ($PromptPower -ieq "Y") { + try { + Write-Host "Installing PowerApps Administration Module..." -ForegroundColor Gray + Install-Module -Name Microsoft.PowerApps.Administration.PowerShell -Scope CurrentUser -AllowClobber -Force -ErrorAction Stop + + Write-Host "Logging into PowerApps... (This may open a browser window)" -ForegroundColor Gray + Add-PowerAppsAccount -Endpoint prod -TenantID $TenantId + + Write-Host "Registering App as Management App..." -ForegroundColor Gray + New-PowerAppManagementApp -ApplicationId $Application.AppId -ErrorAction Stop + Write-Host "Power Automate Management App Registration Complete!" -ForegroundColor Green + Write-Host "`nNote: For Desktop Flow scanning, ensure this App Registration is added as an Application User with the 'System Administrator' security role in your Dataverse environment(s)." -ForegroundColor Yellow + } catch { + Write-Warning "Power Automate registration failed: $_" + Write-Host "You can safely ignore this error. The main App Registration was created successfully." -ForegroundColor Yellow + Write-Host "To retry later, follow the Power Platform steps in the README." -ForegroundColor Yellow + } +} + +# --- FINAL DISCONNECT --- +Disconnect-MgGraph +Read-Host "`nPress Enter to close this window" diff --git a/telemetry/README.md b/telemetry/README.md new file mode 100644 index 00000000..96e8b0b4 --- /dev/null +++ b/telemetry/README.md @@ -0,0 +1,77 @@ +# Telemetry Module + +## Prerequisites + +The certificate authentication flow, reports parsing, and user interface require the following Python libraries: +* `customtkinter` +* `requests` +* `pandas` +* `psutil` +* `matplotlib` +* `cryptography` +* `msal` + +You can install them via pip: +```bash +pip install customtkinter requests pandas psutil matplotlib cryptography msal +``` + +## Architecture & Optimizations +For large tenant scopes (e.g., millions of records or 100K+ flows), this module utilizes aggressive disk-caching mechanisms out-of-the-box, ensuring the application remains lightweight on RAM: +- **SQLite UI Pagination**: Data grids are lazily fetched from `sqlite3` temp databases rather than hoarding UI components in Python lists. +- **Disk Streaming Pipelines**: Complex parsing arrays are continuously streamed to local `.jsonl` temp files and pushed natively to `pandas.DataFrame` chunking logic for exports. +- **Lazy Garbage Collection**: Core navigation state handles `gc.collect()` passively between UI tab cycles. + + +## Setup & Execution + +### 1. Entra ID Permissions (Tenant-Wide Cloud Flows) + +1. Navigate to the [Microsoft Entra ID Portal](https://www.google.com/search?q=https://entra.microsoft.com/) > **Roles and administrators**. +2. Assign the **Power Platform Administrator** role to your App Registration. + +### 2. Dataverse Permissions (Desktop Flows) + +*Perform this in each environment where you need to scan Desktop Flows:* + +1. Navigate to the [Power Platform Admin Center](https://www.google.com/search?q=https://admin.powerplatform.microsoft.com/) > **Environments** > [Select Environment] > **Settings**. +2. Under **Users + permissions** > **Application users**, click **+ New app user**. +3. Add your App Registration and assign it the **System Administrator** role. + +### 3. Certificate Setup (Hybrid Authentication) + +The telemetry planner uses local certificate-based authentication for connecting securely to Microsoft APIs: + +1. When running the Telemetry tool, it checks for a directory named `certificate/{tenantId}_{clientId}` containing `passkey.pfx` under the root of `migration-planner`. +2. If this file does not exist, the app automatically generates a self-signed public certificate (`certificate.pem`) and an encrypted private key bundle (`passkey.pfx`) under the dynamic `certificate/{tenantId}_{clientId}` directory using the provided Client Secret as the password. +3. You will be prompted in the UI to upload `certificate.pem` to Microsoft Entra ID: + - Navigate to the **Microsoft Entra ID Portal** > **App registrations** > [Select your Application]. + - Click **Certificates & secrets** > **Certificates** tab > **Upload certificate**. + - Select and upload the generated `certificate.pem` file. +4. Click **Continue** in the application interface to complete the connection flow. + +### 4. Entra ID App & PowerShell Permissions (Calendar & Mailbox Telemetry) + +For the core telemetry scanners (Calendar Telemetry, Active Users, Mailbox/SharePoint Usage, etc.) to query successfully: + +#### A. Microsoft Graph API Permissions (Application Scopes) +Ensure the following **Application** API permissions are granted and admin-consented in your App Registration: +- `Place.Read.All`: Used to list meeting rooms and resource device counts. +- `User.Read.All`: Used to read user directory identities to aggregate settings. +- `Calendars.ReadBasic.All`: Used to audit organizational calendar permissions. +- `Reports.Read.All`: Used to retrieve active user trends and mailbox/SharePoint usage reports. +- `Directory.Read.All`: Used to read tenant organization configuration data. + +#### B. Exchange Online PowerShell Roles +The certificate-based PowerShell client requires administrative roles to read Exchange policies (OWA, default apps, sharing policies). +In the **Microsoft Entra ID Portal** > **Roles and administrators**, assign one of the following directory roles to your App Registration: +- **Global Reader** (Recommended, read-only) +- **Exchange Administrator** + +## Logging +Logs are appended to `telemetry/logs/power_automate_log.txt`. + +## Troubleshooting +If you encounter a `400 Client Error: Bad Request` when querying the workflows endpoint, please verify: +1. The App is properly allowlisted in the Power Platform Admin Center. +2. The Environment URL is correct and accessible. \ No newline at end of file diff --git a/telemetry/active_users_usage.py b/telemetry/active_users_usage.py new file mode 100644 index 00000000..35d23f19 --- /dev/null +++ b/telemetry/active_users_usage.py @@ -0,0 +1,25 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Backward compatibility facade for M365 Apps active users usage telemetry.""" + +# Re-export pipeline functions from core backend +from core.graph.m365_apps.active_users import run_o365_pipeline, process_active_user_detail +from core.graph.m365_apps.active_users_trend import run_o365_trend_pipeline, process_active_user_counts +from core.graph.m365_apps.app_usage import run_m365_pipeline, process_m365_app_user_detail + +# Re-export UI subframes from telemetry package +from telemetry.m365_apps.active_users import ActiveUsersUsageFrame +from telemetry.m365_apps.active_users_trend import ActiveUsersTrendFrame +from telemetry.m365_apps.app_usage import M365AppUsageFrame diff --git a/telemetry/calendar_telemetry.py b/telemetry/calendar_telemetry.py new file mode 100644 index 00000000..beafd275 --- /dev/null +++ b/telemetry/calendar_telemetry.py @@ -0,0 +1,21 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Backward compatibility facade for Exchange Online Calendar telemetry.""" + +# Re-export pipeline from core backend +from core.graph.exchange.calendar import run_calendar_telemetry_pipeline + +# Re-export UI subframe from telemetry package +from telemetry.exchange.calendar import CalendarTelemetryFrame diff --git a/telemetry/data_security_governance.py b/telemetry/data_security_governance.py new file mode 100644 index 00000000..9520d476 --- /dev/null +++ b/telemetry/data_security_governance.py @@ -0,0 +1,97 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Backward compatibility facade for Data Security & Governance telemetry.""" + +import logging +from core.graph.security.sensitivity_labels import run_sensitivity_labels_pipeline +from core.graph.security.retention_policies import run_retention_policies_pipeline +from core.graph.security.dlp_policies import run_dlp_policies_pipeline +from core.graph.security.sensitive_info_types import run_sensitive_info_types_pipeline +from core.graph.security.authentication import run_authentication_pipeline +from core.graph.security.service_principals_sso import run_service_principals_sso_pipeline +from telemetry.security import DataSecurityGovernanceFrame + +logger = logging.getLogger(__name__) + +def run_security_governance_pipeline(client_id, client_secret, tenant_id) -> dict: + """Legacy pipeline helper.""" + labels = None + labels_error = None + try: + from core.graph.client import GraphClient + from core.graph.security import SecurityService + c = GraphClient(tenant_id=tenant_id, client_ids=client_id, client_secrets=client_secret, concurrency=1) + c.authenticate() + svc = SecurityService(c) + labels = svc.fetch_sensitivity_labels() + c.close() + except Exception as e: + labels_error = str(e) + + policies = None + policies_error = None + try: + policies = run_retention_policies_pipeline(client_id, client_secret, tenant_id) + except Exception as e: + policies_error = str(e) + + return { + "labels": labels, + "labels_error": labels_error, + "policies": policies, + "policies_error": policies_error + } + +def fetch_sensitivity_labels_data(client_id, client_secret, tenant_id, csv_path=None, on_page_callback=None, is_cancelled_callback=None) -> dict: + try: + run_sensitivity_labels_pipeline(client_id, client_secret, tenant_id, csv_path, on_page_callback, is_cancelled_callback) + return {"labels": [], "error": None} + except Exception as e: + return {"labels": None, "error": str(e)} + +def fetch_service_principals_sso_data(client_id, client_secret, tenant_id, csv_path=None, on_page_callback=None, is_cancelled_callback=None) -> dict: + try: + run_service_principals_sso_pipeline(client_id, client_secret, tenant_id, csv_path, on_page_callback, is_cancelled_callback) + return {"sso": [], "error": None} + except Exception as e: + return {"sso": None, "error": str(e)} + +def fetch_retention_policies_data(client_id, client_secret, tenant_id) -> dict: + try: + policies = run_retention_policies_pipeline(client_id, client_secret, tenant_id) + return {"policies": policies, "error": None} + except Exception as e: + return {"policies": None, "error": str(e)} + +def fetch_dlp_policies_data(client_id, client_secret, tenant_id) -> dict: + try: + policies = run_dlp_policies_pipeline(client_id, client_secret, tenant_id) + return {"policies": policies, "error": None} + except Exception as e: + return {"policies": None, "error": str(e)} + +def fetch_sensitive_info_types_data(client_id, client_secret, tenant_id) -> dict: + try: + data = run_sensitive_info_types_pipeline(client_id, client_secret, tenant_id) + return {"sit_data": data, "error": None} + except Exception as e: + return {"sit_data": None, "error": str(e)} + +def fetch_authentication_data(client_id, client_secret, tenant_id, csv_path=None, on_page_callback=None, is_cancelled_callback=None) -> dict: + try: + run_authentication_pipeline(client_id, client_secret, tenant_id, csv_path, on_page_callback, is_cancelled_callback) + return {"auth_data": {"ca_policies": []}, "error": None} + except Exception as e: + return {"auth_data": None, "error": str(e)} diff --git a/telemetry/devices_apps_telemetry.py b/telemetry/devices_apps_telemetry.py new file mode 100644 index 00000000..7462e134 --- /dev/null +++ b/telemetry/devices_apps_telemetry.py @@ -0,0 +1,27 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Backward compatibility facade for Microsoft Entra Data telemetry.""" + +# Re-export pipeline from core backend +from core.graph.entra import run_devices_apps_pipeline +from core.graph.client import GraphClient +from core.graph.reports import ReportsService + +# Re-export UI subframes and main container from telemetry package +from telemetry.entra.auth_methods import AuthMethodsSubFrame +from telemetry.entra.app_signins import AppSigninsSubFrame +from telemetry.entra.user_signins import UserSigninsSubFrame +from telemetry.entra.app_registrations import AppRegistrationsSubFrame +from telemetry.entra import DevicesAppsTelemetryFrame diff --git a/telemetry/directory/__init__.py b/telemetry/directory/__init__.py new file mode 100644 index 00000000..ed01cc14 --- /dev/null +++ b/telemetry/directory/__init__.py @@ -0,0 +1,193 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Consolidated Directory Telemetry Orchestrator Container.""" + +import logging +import customtkinter as ctk + +from telemetry.directory.organization import DirectoryOrganizationFrame +from telemetry.directory.domains import DirectoryDomainsFrame +from telemetry.directory.user_logs import DirectoryUserLogsFrame +from telemetry.directory.provisioning_logs import DirectoryProvisioningLogsFrame +from telemetry.directory.users_groups import DirectoryUsersGroupsFrame +from telemetry.styles import * + +usage_logger = logging.getLogger("M365TelemetryAsyncLogger.DirectoryUI") + +class DirectoryFrame(ctk.CTkFrame): + """Self-contained container wrapping the 5 independent Directory telemetry sub-frames.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, retries_var=None, backoff_var=None, **kwargs): + self.semaphore = kwargs.pop("concurrency_semaphore", None) + super().__init__(master, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=12, **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.retries = retries_var + self.backoff = backoff_var + self.status = None # 'loading', 'success', 'error', None + + self.build_ui() + + def build_ui(self): + self.pack(fill="x", expand=True, pady=10) + + self.inner_pad = ctk.CTkFrame(self, fg_color="transparent") + self.inner_pad.pack(fill="both", expand=True, padx=20, pady=20) + + # Main Title Header + self.header_frame = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.header_frame.pack(fill="x", pady=(0, 15)) + ctk.CTkLabel( + self.header_frame, + text="Directory Summary", + font=FONT_HEADER_SMALL, + text_color=COLOR_TEXT_MAIN + ).pack(side="left") + + # 1. Organization Subframe + self.organization_frame = DirectoryOrganizationFrame( + self.inner_pad, + log_callback=self.log_msg, + credentials_callback=self.get_credentials, + status_change_callback=self._subframe_status_changed, + semaphore=self.semaphore + ) + self.organization_frame.pack(fill="x", pady=(0, 10)) + + # Divider 1 + self.divider1 = ctk.CTkFrame(self.inner_pad, height=1, fg_color=COLOR_OUTLINE_LIGHT) + self.divider1.pack(fill="x", pady=15) + + # 2. Domains Subframe + self.domains_frame = DirectoryDomainsFrame( + self.inner_pad, + log_callback=self.log_msg, + credentials_callback=self.get_credentials, + status_change_callback=self._subframe_status_changed, + semaphore=self.semaphore + ) + self.domains_frame.pack(fill="x", pady=(0, 10)) + + # Divider 2 + self.divider2 = ctk.CTkFrame(self.inner_pad, height=1, fg_color=COLOR_OUTLINE_LIGHT) + self.divider2.pack(fill="x", pady=15) + + # 3. User Logs Subframe + self.user_logs_frame = DirectoryUserLogsFrame( + self.inner_pad, + log_callback=self.log_msg, + credentials_callback=self.get_credentials, + status_change_callback=self._subframe_status_changed, + semaphore=self.semaphore + ) + self.user_logs_frame.pack(fill="x", pady=(0, 10)) + + # Divider 3 + self.divider3 = ctk.CTkFrame(self.inner_pad, height=1, fg_color=COLOR_OUTLINE_LIGHT) + self.divider3.pack(fill="x", pady=15) + + # 4. Provisioning Logs Subframe + self.provisioning_logs_frame = DirectoryProvisioningLogsFrame( + self.inner_pad, + log_callback=self.log_msg, + credentials_callback=self.get_credentials, + status_change_callback=self._subframe_status_changed, + semaphore=self.semaphore + ) + self.provisioning_logs_frame.pack(fill="x", pady=(0, 10)) + + # Divider 4 + self.divider4 = ctk.CTkFrame(self.inner_pad, height=1, fg_color=COLOR_OUTLINE_LIGHT) + self.divider4.pack(fill="x", pady=15) + + # 5. Users & Groups Subframe + self.users_groups_frame = DirectoryUsersGroupsFrame( + self.inner_pad, + log_callback=self.log_msg, + credentials_callback=self.get_credentials, + status_change_callback=self._subframe_status_changed, + semaphore=self.semaphore + ) + self.users_groups_frame.pack(fill="x", pady=(0, 10)) + + def _subframe_status_changed(self): + statuses = [ + self.organization_frame.status, + self.domains_frame.status, + self.user_logs_frame.status, + self.provisioning_logs_frame.status, + self.users_groups_frame.status + ] + if "loading" in statuses: + self.status = "loading" + elif "error" in statuses: + self.status = "error" + elif "success" in statuses: + self.status = "success" + else: + self.status = None + self.on_status_change() + + def reset_view(self): + self.pack_forget() + self.status = None + self.organization_frame.reset_view() + self.domains_frame.reset_view() + self.user_logs_frame.reset_view() + self.provisioning_logs_frame.reset_view() + self.users_groups_frame.reset_view() + + def trigger_fetch(self, tenant, client_id, client_secret): + usage_logger.info("DirectoryFrame trigger_fetch: propagating to subframes...") + self.pack(fill="x", expand=True, pady=10) + self.organization_frame.trigger_fetch(tenant, client_id, client_secret) + self.domains_frame.trigger_fetch(tenant, client_id, client_secret) + self.user_logs_frame.trigger_fetch(tenant, client_id, client_secret) + self.provisioning_logs_frame.trigger_fetch(tenant, client_id, client_secret) + self.users_groups_frame.trigger_fetch(tenant, client_id, client_secret) + + def cancel(self): + usage_logger.info("DirectoryFrame cancel: propagating to subframes...") + self.organization_frame.cancel() + self.domains_frame.cancel() + self.user_logs_frame.cancel() + self.provisioning_logs_frame.cancel() + self.users_groups_frame.cancel() + + # Properties maintained for exact backward compatibility with reporting and export logic + @property + def last_organization(self): + return self.organization_frame.last_data + + @property + def last_domains(self): + return self.domains_frame.last_data + + @property + def last_user_creation_logs(self): + return self.user_logs_frame.last_data + + @property + def last_provisioning_logs(self): + return self.provisioning_logs_frame.last_data + + @property + def last_group_counts(self): + return self.users_groups_frame.last_data.get("group_counts", {}) + + @property + def last_user_counts(self): + return self.users_groups_frame.last_data.get("user_counts", {}) diff --git a/telemetry/directory/domains.py b/telemetry/directory/domains.py new file mode 100644 index 00000000..c96bd341 --- /dev/null +++ b/telemetry/directory/domains.py @@ -0,0 +1,402 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""UI frame for Microsoft Entra ID Domains telemetry.""" + +import os +import csv +import logging +import threading +import asyncio +import webbrowser +from typing import Optional +import customtkinter as ctk + +from core.graph.client import GraphClient +from core.graph.directory.domains import DomainsService +from core.graph.db import import_csv_to_sqlite, query_page_sync +from telemetry.styles import * + +usage_logger = logging.getLogger("M365TelemetryAsyncLogger.DirectoryDomainsUI") + +class DirectoryDomainsFrame(ctk.CTkFrame): + """Sub-frame showing Domains list with pagination.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, semaphore=None, **kwargs): + super().__init__(master, fg_color="transparent", **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.semaphore = semaphore + self.status = None + self.is_cancelled = False + self.current_request_id = 0 + + self.ITEMS_PER_PAGE = 10 + self.current_page = 0 + self._cached_domains = [] + self.csv_path = None + + self.build_ui() + + def build_ui(self): + self.header_frame = ctk.CTkFrame(self, fg_color="transparent") + self.header_frame.pack(fill="x", pady=(0, 5)) + + self.title_lbl = ctk.CTkLabel(self.header_frame, text="Domains", font=FONT_HEADER_SMALL, text_color=COLOR_TEXT_MAIN) + self.title_lbl.pack(side="left") + + self.reference_link = ctk.CTkLabel( + self.header_frame, + text="Domain API Reference ↗", + font=FONT_BODY_BOLD, + text_color=COLOR_PRIMARY, + cursor="hand2" + ) + self.reference_link.pack(side="left", padx=(15, 0)) + self.reference_link.bind("", lambda e: webbrowser.open("https://learn.microsoft.com/en-us/graph/api/resources/domain?view=graph-rest-1.0#properties")) + self.reference_link.bind("", lambda e: self.reference_link.configure(text_color=COLOR_PRIMARY_HOVER)) + self.reference_link.bind("", lambda e: self.reference_link.configure(text_color=COLOR_PRIMARY)) + + self.btn_refresh = ctk.CTkButton( + self.header_frame, state="disabled", text="↻ Reload", width=80, height=24, + font=ctk.CTkFont(family="Segoe UI", size=12), + fg_color="transparent", border_width=1, text_color="#2563EB", hover_color="#DBEAFE", + command=self.trigger_fetch_individual + ) + self.btn_refresh.pack(side="right") + + self.body_frame = ctk.CTkFrame(self, fg_color="transparent") + self.body_frame.pack(fill="x") + + self.reset_view() + + def reset_view(self): + self.status = None + self.is_cancelled = False + self.csv_path = None + self.current_page = 0 + self._cached_domains = [] + for w in self.body_frame.winfo_children(): + w.destroy() + + def _set_state_loading(self, msg="Loading..."): + for w in self.body_frame.winfo_children(): + w.destroy() + loading_lbl = ctk.CTkLabel(self.body_frame, text=f"⏳ {msg}", text_color="#6b7280", font=ctk.CTkFont(family="Segoe UI", size=13)) + loading_lbl.pack(pady=(15, 5)) + pb = ctk.CTkProgressBar(self.body_frame, mode="indeterminate", width=250, fg_color="#F3F4F6", progress_color="#2563EB") + pb.pack(pady=(0, 15)) + pb.start() + + def _set_state_error(self, error_msg): + for w in self.body_frame.winfo_children(): + w.destroy() + display_msg = error_msg + if "401" in error_msg or "403" in error_msg or "unauthorized" in error_msg.lower() or "forbidden" in error_msg.lower(): + display_msg = "Directory read permissions required.\nPlease grant the 'Directory.Read.All' permission to your App Registration in Entra ID." + ctk.CTkLabel(self.body_frame, text=f"✖ {display_msg}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center").pack(pady=(15, 5)) + ctk.CTkButton(self.body_frame, text="Try Again", command=self.trigger_fetch_individual, width=120, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 15)) + + def trigger_fetch_individual(self): + tenant, clients, secrets = self.get_credentials() + if tenant: + self.trigger_fetch(tenant, clients[0], secrets[0]) + else: + self._set_state_error("Missing connection credentials.") + + def trigger_fetch(self, tenant, client_id, client_secret): + self.status = "loading" + self.is_cancelled = False + self.current_page = 0 + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + if os.path.basename(script_dir) == "directory": + script_dir = os.path.dirname(script_dir) + reports_dir = os.path.join(script_dir, "reports", f"{tenant}_{client_id}") + self.csv_path = os.path.join(reports_dir, "directory_domains.csv") + + self._set_state_loading("Fetching directory domains list...") + self.on_status_change() + if hasattr(self, "btn_refresh") and self.btn_refresh.winfo_exists(): + self.btn_refresh.configure(state="disabled") + + threading.Thread( + target=self._execute_worker, + args=(tenant, client_id, client_secret, self.current_request_id), + daemon=True + ).start() + + def _execute_worker(self, tenant, client_id, client_secret, request_id): + if self.semaphore: + self.semaphore.acquire() + try: + if self.is_cancelled or request_id != self.current_request_id: + return + + client = GraphClient( + tenant_id=tenant, + client_ids=client_id, + client_secrets=client_secret, + concurrency=1, + retries=5, + backoff=2 + ) + client.authenticate(required_scopes=["Directory.Read.All"]) + + domains_service = DomainsService(client) + domains_list = domains_service.get_domains(self.log_msg) + client.close() + + if self.is_cancelled or request_id != self.current_request_id: + return + + # Write Domains CSV + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + if os.path.basename(script_dir) == "directory": + script_dir = os.path.dirname(script_dir) + reports_dir = os.path.join(script_dir, "reports", f"{tenant}_{client_id}") + os.makedirs(reports_dir, exist_ok=True) + + headers = ["Domain ID", "Authentication Type", "Admin Managed", "Default", "Verified", "Supported Services", "Federation Display Name", "Federation Issuer URI"] + rows = [] + for domain in domains_list: + auth_type = domain.get("authenticationType", "N/A") or "N/A" + admin_managed = "Yes" if domain.get("isAdminManaged") else "No" + is_default = "Yes" if domain.get("isDefault") else "No" + is_verified = "Yes" if domain.get("isVerified") else "No" + services = domain.get("supportedServices", []) + services_str = ", ".join(services) if services else "-" + fed_idp = domain.get("federationDisplayName") or "-" + fed_issuer = domain.get("federationIssuerUri") or "-" + rows.append([domain.get("id", "-"), auth_type, admin_managed, is_default, is_verified, services_str, fed_idp, fed_issuer]) + + with open(self.csv_path, 'w', encoding='utf-8', newline='') as f: + writer = csv.writer(f) + writer.writerow(headers) + writer.writerows(rows) + + db_path = os.path.join(reports_dir, "telemetry_cache.db") + asyncio.run(import_csv_to_sqlite(self.csv_path, db_path, "directory_domains")) + + self.after(0, self._render_success, domains_list, request_id) + except Exception as e: + usage_logger.error(f"Error fetching Domains list: {e}", exc_info=True) + if not self.is_cancelled and request_id == self.current_request_id: + self.after(0, self._render_error, str(e), request_id) + finally: + if self.semaphore: + self.semaphore.release() + + def _render_success(self, domains_list, request_id): + if self.is_cancelled or request_id != self.current_request_id: + return + self.status = "success" + self._cached_domains = domains_list + self._update_domains_ui_paginated() + self.on_status_change() + if hasattr(self, "btn_refresh") and self.btn_refresh.winfo_exists(): + self.btn_refresh.configure(state="normal") + + def _render_error(self, err_msg, request_id): + if self.is_cancelled or request_id != self.current_request_id: + return + self.status = "error" + self._set_state_error(err_msg) + self.on_status_change() + if hasattr(self, "btn_refresh") and self.btn_refresh.winfo_exists(): + self.btn_refresh.configure(state="normal") + + def _load_page_from_sqlite(self, page): + if not self.csv_path: + return [], 0 + + reports_dir = os.path.dirname(self.csv_path) + db_path = os.path.join(reports_dir, "telemetry_cache.db") + if not os.path.exists(db_path): + return [], 0 + + try: + rows, total_count = query_page_sync(db_path, "directory_domains", page, self.ITEMS_PER_PAGE) + domains = [] + for row in rows: + services_str = row.get("Supported_Services", "") + domains.append({ + "id": row.get("Domain_ID", "-"), + "authenticationType": row.get("Authentication_Type", "-"), + "isAdminManaged": row.get("Admin_Managed") == "Yes", + "isDefault": row.get("Default") == "Yes", + "isVerified": row.get("Verified") == "Yes", + "supportedServices": [s.strip() for s in services_str.split(",")] if services_str and services_str != "-" else [], + "federationDisplayName": row.get("Federation_Display_Name", "-"), + "federationIssuerUri": row.get("Federation_Issuer_URI", "-") + }) + return domains, total_count + except Exception as e: + usage_logger.error(f"Error reading SQLite for Domains pagination: {e}") + return [], 0 + + def _update_domains_ui_paginated(self): + for w in self.body_frame.winfo_children(): + w.destroy() + + domains_grid = ctk.CTkFrame(self.body_frame, fg_color=COLOR_OUTLINE_LIGHT, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + domains_grid.pack(fill="x", expand=True, pady=(5, 10)) + + domains_grid.grid_columnconfigure((0, 5, 6, 7), weight=3) + domains_grid.grid_columnconfigure((1, 2, 3, 4), weight=2) + + domains_headers = ["Domain ID", "Auth Type", "Admin Managed", "Default", "Verified", "Supported Services", "Federation Display Name", "Federation Issuer URI"] + for col_idx, head_text in enumerate(domains_headers): + cell = ctk.CTkFrame(domains_grid, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + page_data, total_count = self._load_page_from_sqlite(self.current_page) + + if not page_data: + empty_cell = ctk.CTkFrame(domains_grid, fg_color="transparent") + empty_cell.grid(row=1, column=0, columnspan=8, sticky="nsew", pady=15) + ctk.CTkLabel(empty_cell, text="No domains found under the organization.", text_color=COLOR_TEXT_SUB).pack() + else: + for item_idx, domain in enumerate(page_data, start=1): + bg_style = COLOR_SURFACE if item_idx % 2 == 0 else COLOR_SURFACE_VARIANT + + auth_type = domain.get("authenticationType", "N/A") + admin_managed = "Yes" if domain.get("isAdminManaged") else "No" + is_default = "Yes" if domain.get("isDefault") else "No" + is_verified = "Yes" if domain.get("isVerified") else "No" + services = domain.get("supportedServices", []) + services_str = ", ".join(services) if services else "-" + + # Domain ID + c0 = ctk.CTkFrame(domains_grid, fg_color=bg_style, corner_radius=0) + c0.grid(row=item_idx, column=0, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c0, text=domain.get("id", "-"), font=FONT_BODY_BOLD, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=8, anchor="nw") + + # Auth Type + c1 = ctk.CTkFrame(domains_grid, fg_color=bg_style, corner_radius=0) + c1.grid(row=item_idx, column=1, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c1, text=auth_type, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=8, anchor="nw") + + # Admin Managed + c2 = ctk.CTkFrame(domains_grid, fg_color=bg_style, corner_radius=0) + c2.grid(row=item_idx, column=2, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c2, text=admin_managed, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=8, anchor="nw") + + # Default + c3 = ctk.CTkFrame(domains_grid, fg_color=bg_style, corner_radius=0) + c3.grid(row=item_idx, column=3, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c3, text=is_default, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=8, anchor="nw") + + # Verified + c4 = ctk.CTkFrame(domains_grid, fg_color=bg_style, corner_radius=0) + c4.grid(row=item_idx, column=4, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c4, text=is_verified, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=8, anchor="nw") + + # Supported Services + c5 = ctk.CTkFrame(domains_grid, fg_color=bg_style, corner_radius=0) + c5.grid(row=item_idx, column=5, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c5, text=services_str, text_color=COLOR_TEXT_MAIN, justify="left", wraplength=180).pack(padx=10, pady=8, anchor="nw") + + # Federated IdP Name + c6 = ctk.CTkFrame(domains_grid, fg_color=bg_style, corner_radius=0) + c6.grid(row=item_idx, column=6, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c6, text=domain.get("federationDisplayName", "-"), text_color=COLOR_TEXT_MAIN, justify="left", wraplength=180).pack(padx=10, pady=8, anchor="nw") + + # Federated Issuer URI + c7 = ctk.CTkFrame(domains_grid, fg_color=bg_style, corner_radius=0) + c7.grid(row=item_idx, column=7, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c7, text=domain.get("federationIssuerUri", "-"), text_color=COLOR_TEXT_MAIN, justify="left", wraplength=200).pack(padx=10, pady=8, anchor="nw") + + # Draw pagination controls if we have multiple pages + if total_count > 0: + self._draw_pagination_controls(total_count) + + domains_footnote = ctk.CTkLabel( + self.body_frame, + text="* AuthenticationType=Managed indicates a cloud managed domain where Microsoft Entra ID performs user authentication. Federated indicates authentication is federated with an identity provider (eg. AD FS, Okta etc.)", + font=FONT_BODY_SMALL, + text_color=COLOR_TEXT_SUB, + anchor="w", + justify="left", + wraplength=1100 + ) + domains_footnote.pack(fill="x", padx=10, pady=(0, 5)) + + def _draw_pagination_controls(self, total_count): + total_pages = (total_count + self.ITEMS_PER_PAGE - 1) // self.ITEMS_PER_PAGE + if total_pages <= 1: + return + + pagination_frame = ctk.CTkFrame(self.body_frame, fg_color="transparent") + pagination_frame.pack(fill="x", pady=(2, 5)) + + left_spacer = ctk.CTkFrame(pagination_frame, fg_color="transparent") + left_spacer.pack(side="left", fill="x", expand=True) + + center_container = ctk.CTkFrame(pagination_frame, fg_color="transparent") + center_container.pack(side="left") + + prev_state = "normal" if self.current_page > 0 else "disabled" + btn_prev = ctk.CTkButton( + center_container, text="◀ Prev", width=70, height=22, corner_radius=6, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, + state=prev_state, + command=lambda: self._change_page(-1) + ) + btn_prev.pack(side="left", padx=5) + + page_lbl = ctk.CTkLabel( + center_container, + text=f"Page {self.current_page + 1} of {total_pages} ({total_count} domains)", + font=FONT_BODY_MEDIUM, + text_color=COLOR_TEXT_SUB + ) + page_lbl.pack(side="left", padx=15) + + next_state = "normal" if self.current_page < total_pages - 1 else "disabled" + btn_next = ctk.CTkButton( + center_container, text="Next ▶", width=70, height=22, corner_radius=6, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, + state=next_state, + command=lambda: self._change_page(1) + ) + btn_next.pack(side="left", padx=5) + + right_spacer = ctk.CTkFrame(pagination_frame, fg_color="transparent") + right_spacer.pack(side="right", fill="x", expand=True) + + def _change_page(self, delta): + self.current_page += delta + self._update_domains_ui_paginated() + + def cancel(self): + self.is_cancelled = True + self.current_request_id += 1 + if self.status == "loading": + self.status = "cancelled" + self._update_domains_ui_paginated() + if hasattr(self, "btn_refresh") and self.btn_refresh.winfo_exists(): + self.btn_refresh.configure(state="normal") + + @property + def last_data(self): + if hasattr(self, "_cached_domains") and self._cached_domains: + return self._cached_domains + # Fallback load from SQLite + page_data, _ = self._load_page_from_sqlite(0) + return page_data diff --git a/telemetry/directory/organization.py b/telemetry/directory/organization.py new file mode 100644 index 00000000..c790d082 --- /dev/null +++ b/telemetry/directory/organization.py @@ -0,0 +1,352 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""UI frame for Microsoft Entra ID Organization telemetry.""" + +import os +import csv +import logging +import threading +import asyncio +import sqlite3 +import webbrowser +from typing import Optional +import customtkinter as ctk + +from core.graph.client import GraphClient +from core.graph.directory.organization import OrganizationService +from core.graph.db import import_csv_to_sqlite +from telemetry.styles import * + +usage_logger = logging.getLogger("M365TelemetryAsyncLogger.DirectoryOrganizationUI") + +class DirectoryOrganizationFrame(ctk.CTkFrame): + """Sub-frame showing Organization properties.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, semaphore=None, **kwargs): + super().__init__(master, fg_color="transparent", **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.semaphore = semaphore + self.status = None + self.is_cancelled = False + self.current_request_id = 0 + self._cached_org_data = [] + self.csv_path = None + + self.build_ui() + + def build_ui(self): + self.header_frame = ctk.CTkFrame(self, fg_color="transparent") + self.header_frame.pack(fill="x", pady=(0, 5)) + + self.title_lbl = ctk.CTkLabel(self.header_frame, text="Organization", font=FONT_HEADER_SMALL, text_color=COLOR_TEXT_MAIN) + self.title_lbl.pack(side="left") + + self.reference_link = ctk.CTkLabel( + self.header_frame, + text="Organization API Reference ↗", + font=FONT_BODY_BOLD, + text_color=COLOR_PRIMARY, + cursor="hand2" + ) + self.reference_link.pack(side="left", padx=(15, 0)) + self.reference_link.bind("", lambda e: webbrowser.open("https://learn.microsoft.com/en-us/graph/api/resources/organization?view=graph-rest-1.0#properties")) + self.reference_link.bind("", lambda e: self.reference_link.configure(text_color=COLOR_PRIMARY_HOVER)) + self.reference_link.bind("", lambda e: self.reference_link.configure(text_color=COLOR_PRIMARY)) + + self.btn_refresh = ctk.CTkButton( + self.header_frame, state="disabled", text="↻ Reload", width=80, height=24, + font=ctk.CTkFont(family="Segoe UI", size=12), + fg_color="transparent", border_width=1, text_color="#2563EB", hover_color="#DBEAFE", + command=self.trigger_fetch_individual + ) + self.btn_refresh.pack(side="right") + + self.body_frame = ctk.CTkFrame(self, fg_color="transparent") + self.body_frame.pack(fill="x") + + self.reset_view() + + def reset_view(self): + self.status = None + self.is_cancelled = False + self.csv_path = None + self._cached_org_data = [] + for w in self.body_frame.winfo_children(): + w.destroy() + + def _set_state_loading(self, msg="Loading..."): + for w in self.body_frame.winfo_children(): + w.destroy() + loading_lbl = ctk.CTkLabel(self.body_frame, text=f"⏳ {msg}", text_color="#6b7280", font=ctk.CTkFont(family="Segoe UI", size=13)) + loading_lbl.pack(pady=(15, 5)) + pb = ctk.CTkProgressBar(self.body_frame, mode="indeterminate", width=250, fg_color="#F3F4F6", progress_color="#2563EB") + pb.pack(pady=(0, 15)) + pb.start() + + def _set_state_error(self, error_msg): + for w in self.body_frame.winfo_children(): + w.destroy() + display_msg = error_msg + if "401" in error_msg or "403" in error_msg or "unauthorized" in error_msg.lower() or "forbidden" in error_msg.lower(): + display_msg = "Directory read permissions required.\nPlease grant the 'Directory.Read.All' permission to your App Registration in Entra ID." + ctk.CTkLabel(self.body_frame, text=f"✖ {display_msg}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center").pack(pady=(15, 5)) + ctk.CTkButton(self.body_frame, text="Try Again", command=self.trigger_fetch_individual, width=120, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 15)) + + def trigger_fetch_individual(self): + tenant, clients, secrets = self.get_credentials() + if tenant: + self.trigger_fetch(tenant, clients[0], secrets[0]) + else: + self._set_state_error("Missing connection credentials.") + + def trigger_fetch(self, tenant, client_id, client_secret): + self.status = "loading" + self.is_cancelled = False + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + # Navigate up to telemetry parent folder if inside telemetry/directory/ + if os.path.basename(script_dir) == "directory": + script_dir = os.path.dirname(script_dir) + reports_dir = os.path.join(script_dir, "reports", f"{tenant}_{client_id}") + self.csv_path = os.path.join(reports_dir, "directory_organization.csv") + + self._set_state_loading("Fetching directory organization configuration...") + self.on_status_change() + if hasattr(self, "btn_refresh") and self.btn_refresh.winfo_exists(): + self.btn_refresh.configure(state="disabled") + + threading.Thread( + target=self._execute_worker, + args=(tenant, client_id, client_secret, self.current_request_id), + daemon=True + ).start() + + def _execute_worker(self, tenant, client_id, client_secret, request_id): + if self.semaphore: + self.semaphore.acquire() + try: + if self.is_cancelled or request_id != self.current_request_id: + return + + client = GraphClient( + tenant_id=tenant, + client_ids=client_id, + client_secrets=client_secret, + concurrency=1, + retries=5, + backoff=2 + ) + client.authenticate(required_scopes=["Directory.Read.All"]) + + org_service = OrganizationService(client) + org_list = org_service.get_organization_info(self.log_msg) + client.close() + + if self.is_cancelled or request_id != self.current_request_id: + return + + # Write Organization CSV + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + if os.path.basename(script_dir) == "directory": + script_dir = os.path.dirname(script_dir) + reports_dir = os.path.join(script_dir, "reports", f"{tenant}_{client_id}") + os.makedirs(reports_dir, exist_ok=True) + + org_headers = [ + "displayName", + "isMultipleDataLocationsForServicesEnabled", "onPremisesSyncEnabled", + "onPremisesLastSyncDateTime", "partnerTenantType", "tenantType", + "provisionedPlans_service", "provisionedPlans_capabilityStatus", "provisionedPlans_provisioningStatus" + ] + org_rows = [] + + def format_csv_val(v): + return "null" if v is None else str(v) + + for org in org_list: + disp_name = format_csv_val(org.get("displayName")) + multi_loc = format_csv_val(org.get("isMultipleDataLocationsForServicesEnabled")) + sync_enabled = format_csv_val(org.get("onPremisesSyncEnabled")) + last_sync = format_csv_val(org.get("onPremisesLastSyncDateTime")) + partner_type = format_csv_val(org.get("partnerTenantType")) + tenant_type = format_csv_val(org.get("tenantType")) + + plans = org.get("provisionedPlans", []) + if not plans: + org_rows.append([disp_name, multi_loc, sync_enabled, last_sync, partner_type, tenant_type, "null", "null", "null"]) + else: + for plan in plans: + service = format_csv_val(plan.get("service")) + cap_status = format_csv_val(plan.get("capabilityStatus")) + prov_status = format_csv_val(plan.get("provisioningStatus")) + org_rows.append([disp_name, multi_loc, sync_enabled, last_sync, partner_type, tenant_type, service, cap_status, prov_status]) + + with open(self.csv_path, 'w', encoding='utf-8', newline='') as f: + writer = csv.writer(f) + writer.writerow(org_headers) + writer.writerows(org_rows) + + db_path = os.path.join(reports_dir, "telemetry_cache.db") + asyncio.run(import_csv_to_sqlite(self.csv_path, db_path, "directory_organization")) + + self.after(0, self._render_success, org_list, request_id) + except Exception as e: + usage_logger.error(f"Error fetching Organization info: {e}", exc_info=True) + if not self.is_cancelled and request_id == self.current_request_id: + self.after(0, self._render_error, str(e), request_id) + finally: + if self.semaphore: + self.semaphore.release() + + def _render_success(self, org_list, request_id): + if self.is_cancelled or request_id != self.current_request_id: + return + self.status = "success" + self._cached_org_data = org_list + self._update_ui() + self.on_status_change() + if hasattr(self, "btn_refresh") and self.btn_refresh.winfo_exists(): + self.btn_refresh.configure(state="normal") + + def _render_error(self, err_msg, request_id): + if self.is_cancelled or request_id != self.current_request_id: + return + self.status = "error" + self._set_state_error(err_msg) + self.on_status_change() + if hasattr(self, "btn_refresh") and self.btn_refresh.winfo_exists(): + self.btn_refresh.configure(state="normal") + + def _update_ui(self): + for w in self.body_frame.winfo_children(): + w.destroy() + + org_grid = ctk.CTkFrame(self.body_frame, fg_color=COLOR_OUTLINE_LIGHT, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + org_grid.pack(fill="x", expand=True, pady=(5, 10)) + + org_grid.grid_columnconfigure(0, weight=1) + org_grid.grid_columnconfigure(1, weight=3) + + org_headers = ["Property", "Value"] + for col_idx, head_text in enumerate(org_headers): + cell = ctk.CTkFrame(org_grid, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + org = self._cached_org_data[0] if self._cached_org_data else {} + + # Unique provisioned plans + plans = org.get("provisionedPlans", []) + plan_services = sorted(list(set( + plan.get("service") for plan in plans + if plan.get("service") and str(plan.get("capabilityStatus")).lower() in ["enabled", "warning"] + ))) + plan_services_str = ", ".join(plan_services) if plan_services else "null" + + def format_ui_val(v): + return "null" if v is None else str(v) + + rows_data = [ + ("displayName", format_ui_val(org.get("displayName"))), + ("isMultipleDataLocationsForServicesEnabled", format_ui_val(org.get("isMultipleDataLocationsForServicesEnabled"))), + ("onPremisesSyncEnabled", format_ui_val(org.get("onPremisesSyncEnabled"))), + ("onPremisesLastSyncDateTime", format_ui_val(org.get("onPremisesLastSyncDateTime"))), + ("partnerTenantType", format_ui_val(org.get("partnerTenantType"))), + ("tenantType", format_ui_val(org.get("tenantType"))), + ("provisionedPlans", plan_services_str) + ] + + for r_idx, (prop_name, val) in enumerate(rows_data, start=1): + bg_style = COLOR_SURFACE if r_idx % 2 == 0 else COLOR_SURFACE_VARIANT + + c0 = ctk.CTkFrame(org_grid, fg_color=bg_style, corner_radius=0) + c0.grid(row=r_idx, column=0, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c0, text=prop_name, font=FONT_BODY_BOLD, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=8, anchor="nw") + + c1 = ctk.CTkFrame(org_grid, fg_color=bg_style, corner_radius=0) + c1.grid(row=r_idx, column=1, sticky="nsew", padx=0, pady=(0, 1)) + + wraplen = 600 if prop_name == "provisionedPlans" else None + lbl = ctk.CTkLabel(c1, text=str(val), font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN, justify="left") + if wraplen: + lbl.configure(wraplength=wraplen) + lbl.pack(padx=10, pady=8, anchor="nw") + + org_footnote = ctk.CTkLabel( + self.body_frame, + text="* If OnPremisesSyncEnabled returns True, on-premises Active Directory is a primary source of truth. If it returns Null or False, the directory is cloud-managed or driven by a 3rd-party application.", + font=FONT_BODY_SMALL, + text_color=COLOR_TEXT_SUB, + anchor="w", + justify="left", + wraplength=1100 + ) + org_footnote.pack(fill="x", padx=10, pady=(0, 5)) + + def cancel(self): + self.is_cancelled = True + self.current_request_id += 1 + if self.status == "loading": + self.status = "cancelled" + self._update_ui() + if hasattr(self, "btn_refresh") and self.btn_refresh.winfo_exists(): + self.btn_refresh.configure(state="normal") + + @property + def last_data(self): + if hasattr(self, "_cached_org_data") and self._cached_org_data: + return self._cached_org_data + # Fallback load from SQLite + if not self.csv_path: + return [] + + reports_dir = os.path.dirname(self.csv_path) + db_path = os.path.join(reports_dir, "telemetry_cache.db") + if not os.path.exists(db_path): + return [] + + items = [] + try: + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + cursor = conn.cursor() + cursor.execute("SELECT * FROM directory_organization") + + org_map = {} + for row in cursor.fetchall(): + disp_name = row["displayName"] + if disp_name not in org_map: + org_map[disp_name] = { + "displayName": disp_name, + "isMultipleDataLocationsForServicesEnabled": row["isMultipleDataLocationsForServicesEnabled"] if row["isMultipleDataLocationsForServicesEnabled"] != "null" else None, + "onPremisesSyncEnabled": row["onPremisesSyncEnabled"] if row["onPremisesSyncEnabled"] != "null" else None, + "onPremisesLastSyncDateTime": row["onPremisesLastSyncDateTime"] if row["onPremisesLastSyncDateTime"] != "null" else None, + "partnerTenantType": row["partnerTenantType"] if row["partnerTenantType"] != "null" else None, + "tenantType": row["tenantType"] if row["tenantType"] != "null" else None, + "provisionedPlans": [] + } + if row["provisionedPlans_service"] != "null": + org_map[disp_name]["provisionedPlans"].append({ + "service": row["provisionedPlans_service"], + "capabilityStatus": row["provisionedPlans_capabilityStatus"], + "provisioningStatus": row["provisionedPlans_provisioningStatus"] + }) + items = list(org_map.values()) + conn.close() + except Exception: + pass + return items diff --git a/telemetry/directory/provisioning_logs.py b/telemetry/directory/provisioning_logs.py new file mode 100644 index 00000000..381b621e --- /dev/null +++ b/telemetry/directory/provisioning_logs.py @@ -0,0 +1,370 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""UI frame for Microsoft Entra ID Provisioning Logs telemetry.""" + +import os +import csv +import logging +import threading +import webbrowser +from typing import Optional +import customtkinter as ctk + +from core.graph.client import GraphClient +from core.graph.directory.provisioning_logs import ProvisioningLogsService +from core.graph.db import import_csv_to_sqlite, query_page_sync +from telemetry.styles import * + +usage_logger = logging.getLogger("M365TelemetryAsyncLogger.DirectoryProvisioningLogsUI") + +class DirectoryProvisioningLogsFrame(ctk.CTkFrame): + """Sub-frame showing Provisioning logs with pagination.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, semaphore=None, **kwargs): + super().__init__(master, fg_color="transparent", **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.semaphore = semaphore + self.status = None + self.is_cancelled = False + self.current_request_id = 0 + + self.ITEMS_PER_PAGE = 10 + self.current_page = 0 + self._cached_logs = [] + self.csv_path = None + + self.build_ui() + + def build_ui(self): + self.header_frame = ctk.CTkFrame(self, fg_color="transparent") + self.header_frame.pack(fill="x", pady=(0, 5)) + + self.title_lbl = ctk.CTkLabel(self.header_frame, text="Provisioning Logs", font=FONT_HEADER_SMALL, text_color=COLOR_TEXT_MAIN) + self.title_lbl.pack(side="left") + + self.reference_link = ctk.CTkLabel( + self.header_frame, + text="Provisioning Logs API Reference ↗", + font=FONT_BODY_SMALL_UNDERLINED, + text_color=COLOR_PRIMARY, + cursor="hand2" + ) + self.reference_link.pack(side="left", padx=(15, 0)) + self.reference_link.bind("", lambda e: webbrowser.open("https://learn.microsoft.com/en-us/graph/api/resources/provisioningobjectsummary?view=graph-rest-1.0")) + self.reference_link.bind("", lambda e: self.reference_link.configure(text_color=COLOR_PRIMARY_HOVER)) + self.reference_link.bind("", lambda e: self.reference_link.configure(text_color=COLOR_PRIMARY)) + + self.btn_refresh = ctk.CTkButton( + self.header_frame, state="disabled", text="↻ Reload", width=80, height=24, + font=ctk.CTkFont(family="Segoe UI", size=12), + fg_color="transparent", border_width=1, text_color="#2563EB", hover_color="#DBEAFE", + command=self.trigger_fetch_individual + ) + self.btn_refresh.pack(side="right") + + self.body_frame = ctk.CTkFrame(self, fg_color="transparent") + self.body_frame.pack(fill="x") + + self.reset_view() + + def reset_view(self): + self.status = None + self.is_cancelled = False + self.csv_path = None + self.current_page = 0 + self._cached_logs = [] + for w in self.body_frame.winfo_children(): + w.destroy() + + def _set_state_loading(self, msg="Loading..."): + for w in self.body_frame.winfo_children(): + w.destroy() + loading_lbl = ctk.CTkLabel(self.body_frame, text=f"⏳ {msg}", text_color="#6b7280", font=ctk.CTkFont(family="Segoe UI", size=13)) + loading_lbl.pack(pady=(15, 5)) + pb = ctk.CTkProgressBar(self.body_frame, mode="indeterminate", width=250, fg_color="#F3F4F6", progress_color="#2563EB") + pb.pack(pady=(0, 15)) + pb.start() + + def _set_state_error(self, error_msg): + for w in self.body_frame.winfo_children(): + w.destroy() + display_msg = error_msg + if "401" in error_msg or "403" in error_msg or "unauthorized" in error_msg.lower() or "forbidden" in error_msg.lower(): + display_msg = "AuditLog.Read.All application permission required.\nPlease grant the 'AuditLog.Read.All' permission to your App Registration in Entra ID." + ctk.CTkLabel(self.body_frame, text=f"✖ {display_msg}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center").pack(pady=(15, 5)) + ctk.CTkButton(self.body_frame, text="Try Again", command=self.trigger_fetch_individual, width=120, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 15)) + + def trigger_fetch_individual(self): + tenant, clients, secrets = self.get_credentials() + if tenant: + self.trigger_fetch(tenant, clients[0], secrets[0]) + else: + self._set_state_error("Missing connection credentials.") + + def trigger_fetch(self, tenant, client_id, client_secret): + self.status = "loading" + self.is_cancelled = False + self.current_page = 0 + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + if os.path.basename(script_dir) == "directory": + script_dir = os.path.dirname(script_dir) + reports_dir = os.path.join(script_dir, "reports", f"{tenant}_{client_id}") + self.csv_path = os.path.join(reports_dir, "directory_provisioning_logs.csv") + + self._set_state_loading("Fetching Provisioning Audit Logs...") + self.on_status_change() + if hasattr(self, "btn_refresh") and self.btn_refresh.winfo_exists(): + self.btn_refresh.configure(state="disabled") + + threading.Thread( + target=self._execute_worker, + args=(tenant, client_id, client_secret, self.current_request_id), + daemon=True + ).start() + + def _execute_worker(self, tenant, client_id, client_secret, request_id): + if self.semaphore: + self.semaphore.acquire() + try: + if self.is_cancelled or request_id != self.current_request_id: + return + + client = GraphClient( + tenant_id=tenant, + client_ids=client_id, + client_secrets=client_secret, + concurrency=1, + retries=5, + backoff=2 + ) + client.authenticate() + + prov_service = ProvisioningLogsService(client) + + provisioning_logs = [] + def handle_provisioning_page(page_rows): + provisioning_logs.extend(page_rows) + + prov_service.fetch_provisioning_logs( + csv_path=self.csv_path, + max_rows=200, + on_page_callback=handle_provisioning_page, + is_cancelled_callback=lambda: self.is_cancelled or request_id != self.current_request_id + ) + client.close() + + if self.is_cancelled or request_id != self.current_request_id: + return + + import asyncio + db_dir = os.path.dirname(self.csv_path) + db_path = os.path.join(db_dir, "telemetry_cache.db") + asyncio.run(import_csv_to_sqlite(self.csv_path, db_path, "provisioning_logs", "initiatedBy")) + + if self.is_cancelled or request_id != self.current_request_id: + return + + self.after(0, self._render_success, provisioning_logs, request_id) + except Exception as e: + usage_logger.error(f"Error fetching Provisioning logs: {e}", exc_info=True) + if not self.is_cancelled and request_id == self.current_request_id: + self.after(0, self._render_error, str(e), request_id) + finally: + if self.semaphore: + self.semaphore.release() + + def _render_success(self, logs, request_id): + if self.is_cancelled or request_id != self.current_request_id: + return + self.status = "success" + self._cached_logs = logs + self._update_provisioning_ui_paginated() + self.on_status_change() + if hasattr(self, "btn_refresh") and self.btn_refresh.winfo_exists(): + self.btn_refresh.configure(state="normal") + + def _render_error(self, err_msg, request_id): + if self.is_cancelled or request_id != self.current_request_id: + return + self.status = "error" + self._set_state_error(err_msg) + self.on_status_change() + if hasattr(self, "btn_refresh") and self.btn_refresh.winfo_exists(): + self.btn_refresh.configure(state="normal") + + def _load_provisioning_page_from_csv(self, page): + if not self.csv_path or not os.path.exists(self.csv_path): + return [], 0 + + db_dir = os.path.dirname(self.csv_path) + db_path = os.path.join(db_dir, "telemetry_cache.db") + + try: + page_data, total_count = query_page_sync(db_path, "provisioning_logs", page, self.ITEMS_PER_PAGE) + # Map keys to match expected dictionary format of UI rendering + mapped_data = [] + for row in page_data: + mapped_data.append({ + "initiatedBy": row.get("initiatedBy", ""), + "provisioningAction": row.get("provisioningAction", ""), + "provisioningSteps": row.get("provisioningSteps", ""), + "servicePrincipal": row.get("servicePrincipal", ""), + "sourceSystem": row.get("sourceSystem", ""), + "targetSystem": row.get("targetSystem", ""), + "tenantId": row.get("tenantId", ""), + "provisioningStatusInfo": row.get("provisioningStatusInfo", "") + }) + return mapped_data, total_count + except Exception as e: + usage_logger.error(f"Error loading provisioning page from SQLite cache: {e}") + return [], 0 + + def _update_provisioning_ui_paginated(self): + for w in self.body_frame.winfo_children(): + w.destroy() + + provisioning_grid = ctk.CTkFrame(self.body_frame, fg_color=COLOR_OUTLINE_LIGHT, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + provisioning_grid.pack(fill="x", expand=True, pady=(5, 10)) + + provisioning_grid.grid_columnconfigure(0, weight=2) + provisioning_grid.grid_columnconfigure(1, weight=2) + provisioning_grid.grid_columnconfigure(2, weight=3) + provisioning_grid.grid_columnconfigure(3, weight=2) + provisioning_grid.grid_columnconfigure(4, weight=1) + provisioning_grid.grid_columnconfigure(5, weight=1) + provisioning_grid.grid_columnconfigure(6, weight=1) + provisioning_grid.grid_columnconfigure(7, weight=2) + + headers = ["Initiated By", "Action", "Steps", "Service Principal", "Source System", "Target System", "Tenant ID", "Status Info"] + for col_idx, head_text in enumerate(headers): + cell = ctk.CTkFrame(provisioning_grid, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=8, pady=8, anchor="w") + + page_data, total_count = self._load_provisioning_page_from_csv(self.current_page) + + if not page_data: + empty_cell = ctk.CTkFrame(provisioning_grid, fg_color="transparent") + empty_cell.grid(row=1, column=0, columnspan=8, sticky="nsew", pady=15) + ctk.CTkLabel(empty_cell, text="No provisioning logs found.", text_color=COLOR_TEXT_SUB).pack() + elif page_data[0].get("initiatedBy") == "ERROR": + err_msg = page_data[0].get("provisioningAction") + error_cell = ctk.CTkFrame(provisioning_grid, fg_color="transparent") + error_cell.grid(row=1, column=0, columnspan=8, sticky="nsew", pady=15) + ctk.CTkLabel(error_cell, text=f"⚠️ {err_msg}", font=FONT_BODY_MEDIUM, text_color="#DC2626", justify="left", wraplength=1000).pack(padx=10, pady=5) + else: + for item_idx, log in enumerate(page_data, start=1): + bg_style = COLOR_SURFACE if item_idx % 2 == 0 else COLOR_SURFACE_VARIANT + + vals = [ + log.get("initiatedBy", "-"), + log.get("provisioningAction", "-"), + log.get("provisioningSteps", "-"), + log.get("servicePrincipal", "-"), + log.get("sourceSystem", "-"), + log.get("targetSystem", "-"), + log.get("tenantId", "-"), + log.get("provisioningStatusInfo", "-") + ] + + for col_idx, val in enumerate(vals): + c = ctk.CTkFrame(provisioning_grid, fg_color=bg_style, corner_radius=0) + c.grid(row=item_idx, column=col_idx, sticky="nsew", padx=0, pady=(0, 1)) + + wraplen = 220 if col_idx in [0, 1, 2, 3, 7] else 100 + ctk.CTkLabel(c, text=val, font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN, justify="left", wraplength=wraplen).pack(padx=8, pady=8, anchor="nw") + + # Draw pagination controls if we have multiple pages + if total_count > 0: + self._draw_pagination_controls(total_count) + + provisioning_footnote = ctk.CTkLabel( + self.body_frame, + text="* Based on sampled data collected from audit logs.", + font=FONT_BODY_SMALL, + text_color=COLOR_TEXT_SUB, + anchor="w", + justify="left", + wraplength=1100 + ) + provisioning_footnote.pack(fill="x", padx=10, pady=(0, 5)) + + def _draw_pagination_controls(self, total_count): + total_pages = (total_count + self.ITEMS_PER_PAGE - 1) // self.ITEMS_PER_PAGE + if total_pages <= 1: + return + + pagination_frame = ctk.CTkFrame(self.body_frame, fg_color="transparent") + pagination_frame.pack(fill="x", pady=(2, 5)) + + left_spacer = ctk.CTkFrame(pagination_frame, fg_color="transparent") + left_spacer.pack(side="left", fill="x", expand=True) + + center_container = ctk.CTkFrame(pagination_frame, fg_color="transparent") + center_container.pack(side="left") + + prev_state = "normal" if self.current_page > 0 else "disabled" + btn_prev = ctk.CTkButton( + center_container, text="◀ Prev", width=70, height=22, corner_radius=6, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, + state=prev_state, + command=lambda: self._change_page(-1) + ) + btn_prev.pack(side="left", padx=5) + + page_lbl = ctk.CTkLabel( + center_container, + text=f"Page {self.current_page + 1} of {total_pages} ({total_count} logs)", + font=FONT_BODY_MEDIUM, + text_color=COLOR_TEXT_SUB + ) + page_lbl.pack(side="left", padx=15) + + next_state = "normal" if self.current_page < total_pages - 1 else "disabled" + btn_next = ctk.CTkButton( + center_container, text="Next ▶", width=70, height=22, corner_radius=6, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, + state=next_state, + command=lambda: self._change_page(1) + ) + btn_next.pack(side="left", padx=5) + + right_spacer = ctk.CTkFrame(pagination_frame, fg_color="transparent") + right_spacer.pack(side="right", fill="x", expand=True) + + def _change_page(self, delta): + self.current_page += delta + self._update_provisioning_ui_paginated() + + def cancel(self): + self.is_cancelled = True + self.current_request_id += 1 + if self.status == "loading": + self.status = "cancelled" + self._update_provisioning_ui_paginated() + if hasattr(self, "btn_refresh") and self.btn_refresh.winfo_exists(): + self.btn_refresh.configure(state="normal") + + @property + def last_data(self): + if hasattr(self, "_cached_logs") and self._cached_logs: + return self._cached_logs + page_data, _ = self._load_provisioning_page_from_csv(0) + return page_data diff --git a/telemetry/directory/user_logs.py b/telemetry/directory/user_logs.py new file mode 100644 index 00000000..ea4840e0 --- /dev/null +++ b/telemetry/directory/user_logs.py @@ -0,0 +1,352 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""UI frame for Microsoft Entra ID User Creation/Deletion Logs telemetry.""" + +import os +import csv +import logging +import threading +import webbrowser +from typing import Optional +import customtkinter as ctk + +from core.graph.client import GraphClient +from core.graph.directory.user_logs import UserLogsService +from core.graph.db import import_csv_to_sqlite, query_page_sync +from telemetry.styles import * + +usage_logger = logging.getLogger("M365TelemetryAsyncLogger.DirectoryUserLogsUI") + +class DirectoryUserLogsFrame(ctk.CTkFrame): + """Sub-frame showing User Creation/Deletion audit logs with pagination.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, semaphore=None, **kwargs): + super().__init__(master, fg_color="transparent", **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.semaphore = semaphore + self.status = None + self.is_cancelled = False + self.current_request_id = 0 + + self.ITEMS_PER_PAGE = 10 + self.current_page = 0 + self._cached_logs = [] + self.csv_path = None + + self.build_ui() + + def build_ui(self): + self.header_frame = ctk.CTkFrame(self, fg_color="transparent") + self.header_frame.pack(fill="x", pady=(0, 5)) + + self.title_lbl = ctk.CTkLabel(self.header_frame, text="User Creation/Deletion logs", font=FONT_HEADER_SMALL, text_color=COLOR_TEXT_MAIN) + self.title_lbl.pack(side="left") + + self.reference_link = ctk.CTkLabel( + self.header_frame, + text="Directory Audit API Reference ↗", + font=FONT_BODY_BOLD, + text_color=COLOR_PRIMARY, + cursor="hand2" + ) + self.reference_link.pack(side="left", padx=(15, 0)) + self.reference_link.bind("", lambda e: webbrowser.open("https://learn.microsoft.com/en-us/graph/api/resources/directoryaudit?view=graph-rest-1.0")) + self.reference_link.bind("", lambda e: self.reference_link.configure(text_color=COLOR_PRIMARY_HOVER)) + self.reference_link.bind("", lambda e: self.reference_link.configure(text_color=COLOR_PRIMARY)) + + self.btn_refresh = ctk.CTkButton( + self.header_frame, state="disabled", text="↻ Reload", width=80, height=24, + font=ctk.CTkFont(family="Segoe UI", size=12), + fg_color="transparent", border_width=1, text_color="#2563EB", hover_color="#DBEAFE", + command=self.trigger_fetch_individual + ) + self.btn_refresh.pack(side="right") + + self.body_frame = ctk.CTkFrame(self, fg_color="transparent") + self.body_frame.pack(fill="x") + + self.reset_view() + + def reset_view(self): + self.status = None + self.is_cancelled = False + self.csv_path = None + self.current_page = 0 + self._cached_logs = [] + for w in self.body_frame.winfo_children(): + w.destroy() + + def _set_state_loading(self, msg="Loading..."): + for w in self.body_frame.winfo_children(): + w.destroy() + loading_lbl = ctk.CTkLabel(self.body_frame, text=f"⏳ {msg}", text_color="#6b7280", font=ctk.CTkFont(family="Segoe UI", size=13)) + loading_lbl.pack(pady=(15, 5)) + pb = ctk.CTkProgressBar(self.body_frame, mode="indeterminate", width=250, fg_color="#F3F4F6", progress_color="#2563EB") + pb.pack(pady=(0, 15)) + pb.start() + + def _set_state_error(self, error_msg): + for w in self.body_frame.winfo_children(): + w.destroy() + display_msg = error_msg + if "401" in error_msg or "403" in error_msg or "unauthorized" in error_msg.lower() or "forbidden" in error_msg.lower(): + display_msg = "AuditLog.Read.All application permission required.\nPlease grant the 'AuditLog.Read.All' permission to your App Registration in Entra ID." + ctk.CTkLabel(self.body_frame, text=f"✖ {display_msg}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center").pack(pady=(15, 5)) + ctk.CTkButton(self.body_frame, text="Try Again", command=self.trigger_fetch_individual, width=120, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 15)) + + def trigger_fetch_individual(self): + tenant, clients, secrets = self.get_credentials() + if tenant: + self.trigger_fetch(tenant, clients[0], secrets[0]) + else: + self._set_state_error("Missing connection credentials.") + + def trigger_fetch(self, tenant, client_id, client_secret): + self.status = "loading" + self.is_cancelled = False + self.current_page = 0 + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + if os.path.basename(script_dir) == "directory": + script_dir = os.path.dirname(script_dir) + reports_dir = os.path.join(script_dir, "reports", f"{tenant}_{client_id}") + self.csv_path = os.path.join(reports_dir, "directory_user_creation_logs.csv") + + self._set_state_loading("Fetching User Creation & Deletion Audit Logs...") + self.on_status_change() + if hasattr(self, "btn_refresh") and self.btn_refresh.winfo_exists(): + self.btn_refresh.configure(state="disabled") + + threading.Thread( + target=self._execute_worker, + args=(tenant, client_id, client_secret, self.current_request_id), + daemon=True + ).start() + + def _execute_worker(self, tenant, client_id, client_secret, request_id): + if self.semaphore: + self.semaphore.acquire() + try: + if self.is_cancelled or request_id != self.current_request_id: + return + + client = GraphClient( + tenant_id=tenant, + client_ids=client_id, + client_secrets=client_secret, + concurrency=1, + retries=5, + backoff=2 + ) + client.authenticate() + + user_logs_service = UserLogsService(client) + + user_creation_logs = [] + def handle_user_creation_page(page_rows): + user_creation_logs.extend(page_rows) + + user_logs_service.fetch_user_creation_logs( + csv_path=self.csv_path, + max_rows=50, + on_page_callback=handle_user_creation_page, + is_cancelled_callback=lambda: self.is_cancelled or request_id != self.current_request_id + ) + client.close() + + if self.is_cancelled or request_id != self.current_request_id: + return + + import asyncio + db_dir = os.path.dirname(self.csv_path) + db_path = os.path.join(db_dir, "telemetry_cache.db") + asyncio.run(import_csv_to_sqlite(self.csv_path, db_path, "user_logs", "activity")) + + if self.is_cancelled or request_id != self.current_request_id: + return + + self.after(0, self._render_success, user_creation_logs, request_id) + except Exception as e: + usage_logger.error(f"Error fetching User Creation logs: {e}", exc_info=True) + if not self.is_cancelled and request_id == self.current_request_id: + self.after(0, self._render_error, str(e), request_id) + finally: + if self.semaphore: + self.semaphore.release() + + def _render_success(self, logs, request_id): + if self.is_cancelled or request_id != self.current_request_id: + return + self.status = "success" + self._cached_logs = logs + self._update_user_creation_ui_paginated() + self.on_status_change() + if hasattr(self, "btn_refresh") and self.btn_refresh.winfo_exists(): + self.btn_refresh.configure(state="normal") + + def _render_error(self, err_msg, request_id): + if self.is_cancelled or request_id != self.current_request_id: + return + self.status = "error" + self._set_state_error(err_msg) + self.on_status_change() + if hasattr(self, "btn_refresh") and self.btn_refresh.winfo_exists(): + self.btn_refresh.configure(state="normal") + + def _load_user_creation_page_from_csv(self, page): + if not self.csv_path or not os.path.exists(self.csv_path): + return [], 0 + + db_dir = os.path.dirname(self.csv_path) + db_path = os.path.join(db_dir, "telemetry_cache.db") + + try: + page_data, total_count = query_page_sync(db_path, "user_logs", page, self.ITEMS_PER_PAGE) + # Map keys to match expected dictionary format of UI rendering + mapped_data = [] + for row in page_data: + mapped_data.append({ + "activity": row.get("activity", ""), + "initiatedBy": row.get("initiatedBy", "") + }) + return mapped_data, total_count + except Exception as e: + usage_logger.error(f"Error loading user creation page from SQLite cache: {e}") + return [], 0 + + def _update_user_creation_ui_paginated(self): + for w in self.body_frame.winfo_children(): + w.destroy() + + user_creation_grid = ctk.CTkFrame(self.body_frame, fg_color=COLOR_OUTLINE_LIGHT, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + user_creation_grid.pack(fill="x", expand=True, pady=(5, 10)) + + user_creation_grid.grid_columnconfigure(0, weight=1) + user_creation_grid.grid_columnconfigure(1, weight=3) + + headers = ["Activity", "Initiated By"] + for col_idx, head_text in enumerate(headers): + cell = ctk.CTkFrame(user_creation_grid, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + page_data, total_count = self._load_user_creation_page_from_csv(self.current_page) + + if not page_data: + empty_cell = ctk.CTkFrame(user_creation_grid, fg_color="transparent") + empty_cell.grid(row=1, column=0, columnspan=2, sticky="nsew", pady=15) + ctk.CTkLabel(empty_cell, text="No user creation/deletion logs found.", text_color=COLOR_TEXT_SUB).pack() + elif page_data[0].get("activity") == "ERROR": + err_msg = page_data[0].get("initiatedBy") + error_cell = ctk.CTkFrame(user_creation_grid, fg_color="transparent") + error_cell.grid(row=1, column=0, columnspan=2, sticky="nsew", pady=15) + ctk.CTkLabel(error_cell, text=f"⚠️ {err_msg}", font=FONT_BODY_MEDIUM, text_color="#DC2626", justify="left", wraplength=1000).pack(padx=10, pady=5) + else: + for item_idx, log in enumerate(page_data, start=1): + bg_style = COLOR_SURFACE if item_idx % 2 == 0 else COLOR_SURFACE_VARIANT + + vals = [ + log.get("activity", "-"), + log.get("initiatedBy", "-") + ] + + for col_idx, val in enumerate(vals): + c = ctk.CTkFrame(user_creation_grid, fg_color=bg_style, corner_radius=0) + c.grid(row=item_idx, column=col_idx, sticky="nsew", padx=0, pady=(0, 1)) + + wraplen = 600 if col_idx == 1 else 180 + ctk.CTkLabel(c, text=val, font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN, justify="left", wraplength=wraplen).pack(padx=10, pady=8, anchor="nw") + + # Draw pagination controls if we have multiple pages + if total_count > 0: + self._draw_pagination_controls(total_count) + + user_creation_footnote = ctk.CTkLabel( + self.body_frame, + text="* Based on sampled data collected from audit logs.", + font=FONT_BODY_SMALL, + text_color=COLOR_TEXT_SUB, + anchor="w", + justify="left", + wraplength=1100 + ) + user_creation_footnote.pack(fill="x", padx=10, pady=(0, 5)) + + def _draw_pagination_controls(self, total_count): + total_pages = (total_count + self.ITEMS_PER_PAGE - 1) // self.ITEMS_PER_PAGE + if total_pages <= 1: + return + + pagination_frame = ctk.CTkFrame(self.body_frame, fg_color="transparent") + pagination_frame.pack(fill="x", pady=(2, 5)) + + left_spacer = ctk.CTkFrame(pagination_frame, fg_color="transparent") + left_spacer.pack(side="left", fill="x", expand=True) + + center_container = ctk.CTkFrame(pagination_frame, fg_color="transparent") + center_container.pack(side="left") + + prev_state = "normal" if self.current_page > 0 else "disabled" + btn_prev = ctk.CTkButton( + center_container, text="◀ Prev", width=70, height=22, corner_radius=6, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, + state=prev_state, + command=lambda: self._change_page(-1) + ) + btn_prev.pack(side="left", padx=5) + + page_lbl = ctk.CTkLabel( + center_container, + text=f"Page {self.current_page + 1} of {total_pages} ({total_count} logs)", + font=FONT_BODY_MEDIUM, + text_color=COLOR_TEXT_SUB + ) + page_lbl.pack(side="left", padx=15) + + next_state = "normal" if self.current_page < total_pages - 1 else "disabled" + btn_next = ctk.CTkButton( + center_container, text="Next ▶", width=70, height=22, corner_radius=6, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, + state=next_state, + command=lambda: self._change_page(1) + ) + btn_next.pack(side="left", padx=5) + + right_spacer = ctk.CTkFrame(pagination_frame, fg_color="transparent") + right_spacer.pack(side="right", fill="x", expand=True) + + def _change_page(self, delta): + self.current_page += delta + self._update_user_creation_ui_paginated() + + def cancel(self): + self.is_cancelled = True + self.current_request_id += 1 + if self.status == "loading": + self.status = "cancelled" + self._update_user_creation_ui_paginated() + if hasattr(self, "btn_refresh") and self.btn_refresh.winfo_exists(): + self.btn_refresh.configure(state="normal") + + @property + def last_data(self): + if hasattr(self, "_cached_logs") and self._cached_logs: + return self._cached_logs + page_data, _ = self._load_user_creation_page_from_csv(0) + return page_data diff --git a/telemetry/directory/users_groups.py b/telemetry/directory/users_groups.py new file mode 100644 index 00000000..b7b6fac0 --- /dev/null +++ b/telemetry/directory/users_groups.py @@ -0,0 +1,321 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""UI frame for Microsoft Entra ID Users & Groups counts telemetry.""" + +import os +import csv +import logging +import threading +import asyncio +import sqlite3 +from typing import Optional +import customtkinter as ctk + +from core.graph.client import GraphClient +from core.graph.directory.users_groups import UsersGroupsService +from core.graph.db import import_csv_to_sqlite +from telemetry.styles import * + +usage_logger = logging.getLogger("M365TelemetryAsyncLogger.DirectoryUsersGroupsUI") + +class DirectoryUsersGroupsFrame(ctk.CTkFrame): + """Sub-frame showing Users & Groups counts table.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, semaphore=None, **kwargs): + super().__init__(master, fg_color="transparent", **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.semaphore = semaphore + self.status = None + self.is_cancelled = False + self.current_request_id = 0 + + self.last_group_counts = {} + self.last_user_counts = {} + self.csv_path = None + + self.build_ui() + + def build_ui(self): + self.header_frame = ctk.CTkFrame(self, fg_color="transparent") + self.header_frame.pack(fill="x", pady=(0, 5)) + + self.title_lbl = ctk.CTkLabel(self.header_frame, text="Groups & Users", font=FONT_HEADER_SMALL, text_color=COLOR_TEXT_MAIN) + self.title_lbl.pack(side="left") + + self.btn_refresh = ctk.CTkButton( + self.header_frame, state="disabled", text="↻ Reload", width=80, height=24, + font=ctk.CTkFont(family="Segoe UI", size=12), + fg_color="transparent", border_width=1, text_color="#2563EB", hover_color="#DBEAFE", + command=self.trigger_fetch_individual + ) + self.btn_refresh.pack(side="right") + + self.body_frame = ctk.CTkFrame(self, fg_color="transparent") + self.body_frame.pack(fill="x") + + self.reset_view() + + def reset_view(self): + self.status = None + self.is_cancelled = False + self.csv_path = None + self.last_group_counts = {} + self.last_user_counts = {} + for w in self.body_frame.winfo_children(): + w.destroy() + + def _set_state_loading(self, msg="Loading..."): + for w in self.body_frame.winfo_children(): + w.destroy() + loading_lbl = ctk.CTkLabel(self.body_frame, text=f"⏳ {msg}", text_color="#6b7280", font=ctk.CTkFont(family="Segoe UI", size=13)) + loading_lbl.pack(pady=(15, 5)) + pb = ctk.CTkProgressBar(self.body_frame, mode="indeterminate", width=250, fg_color="#F3F4F6", progress_color="#2563EB") + pb.pack(pady=(0, 15)) + pb.start() + + def _set_state_error(self, error_msg): + for w in self.body_frame.winfo_children(): + w.destroy() + display_msg = error_msg + if "401" in error_msg or "403" in error_msg or "unauthorized" in error_msg.lower() or "forbidden" in error_msg.lower(): + display_msg = "Directory read permissions required.\nPlease grant the 'Directory.Read.All' permission to your App Registration in Entra ID." + ctk.CTkLabel(self.body_frame, text=f"✖ {display_msg}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center").pack(pady=(15, 5)) + ctk.CTkButton(self.body_frame, text="Try Again", command=self.trigger_fetch_individual, width=120, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 15)) + + def trigger_fetch_individual(self): + tenant, clients, secrets = self.get_credentials() + if tenant: + self.trigger_fetch(tenant, clients[0], secrets[0]) + else: + self._set_state_error("Missing connection credentials.") + + def trigger_fetch(self, tenant, client_id, client_secret): + self.status = "loading" + self.is_cancelled = False + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + if os.path.basename(script_dir) == "directory": + script_dir = os.path.dirname(script_dir) + reports_dir = os.path.join(script_dir, "reports", f"{tenant}_{client_id}") + self.csv_path = os.path.join(reports_dir, "directory_users_groups.csv") + + self._set_state_loading("Fetching Users & Groups Counts...") + self.on_status_change() + if hasattr(self, "btn_refresh") and self.btn_refresh.winfo_exists(): + self.btn_refresh.configure(state="disabled") + + threading.Thread( + target=self._execute_worker, + args=(tenant, client_id, client_secret, self.current_request_id), + daemon=True + ).start() + + def _execute_worker(self, tenant, client_id, client_secret, request_id): + if self.semaphore: + self.semaphore.acquire() + try: + if self.is_cancelled or request_id != self.current_request_id: + return + + client = GraphClient( + tenant_id=tenant, + client_ids=client_id, + client_secrets=client_secret, + concurrency=1, + retries=5, + backoff=2 + ) + client.authenticate(required_scopes=["Directory.Read.All"]) + + ug_service = UsersGroupsService(client) + counts_dict = ug_service.get_users_groups_counts(self.log_msg) + client.close() + + if self.is_cancelled or request_id != self.current_request_id: + return + + # Write Users/Groups CSV + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + if os.path.basename(script_dir) == "directory": + script_dir = os.path.dirname(script_dir) + reports_dir = os.path.join(script_dir, "reports", f"{tenant}_{client_id}") + os.makedirs(reports_dir, exist_ok=True) + + user_c = counts_dict.get("user_counts", {}) + group_c = counts_dict.get("group_counts", {}) + + rows = [ + ("Total Users", user_c.get("total", 0)), + ("Enabled Users", user_c.get("enabled", 0)), + ("Disabled Users", user_c.get("disabled", 0)), + ("Member Users", user_c.get("member", 0)), + ("Guest Users", user_c.get("guest", 0)), + ("Total Groups", group_c.get("total", 0)), + ("Microsoft 365 Groups (Unified)", group_c.get("m365", 0)), + ("Security Groups (Static, non-mail-enabled)", group_c.get("security", 0)), + ("Mail-enabled Security Groups", group_c.get("mail_enabled_security", 0)), + ("Distribution Groups", group_c.get("distribution", 0)), + ("Dynamic Groups (Dynamic Membership)", group_c.get("dynamic", 0)), + ] + + with open(self.csv_path, 'w', encoding='utf-8', newline='') as f: + writer = csv.writer(f) + writer.writerow(["Category", "Count"]) + for cat, count in rows: + writer.writerow([cat, count]) + + db_path = os.path.join(reports_dir, "telemetry_cache.db") + asyncio.run(import_csv_to_sqlite(self.csv_path, db_path, "directory_users_groups")) + + self.after(0, self._render_success, user_c, group_c, request_id) + except Exception as e: + usage_logger.error(f"Error fetching Users & Groups: {e}", exc_info=True) + if not self.is_cancelled and request_id == self.current_request_id: + self.after(0, self._render_error, str(e), request_id) + finally: + if self.semaphore: + self.semaphore.release() + + def _render_success(self, user_c, group_c, request_id): + if self.is_cancelled or request_id != self.current_request_id: + return + self.status = "success" + self.last_user_counts = user_c + self.last_group_counts = group_c + self._update_ui() + self.on_status_change() + if hasattr(self, "btn_refresh") and self.btn_refresh.winfo_exists(): + self.btn_refresh.configure(state="normal") + + def _render_error(self, err_msg, request_id): + if self.is_cancelled or request_id != self.current_request_id: + return + self.status = "error" + self._set_state_error(err_msg) + self.on_status_change() + if hasattr(self, "btn_refresh") and self.btn_refresh.winfo_exists(): + self.btn_refresh.configure(state="normal") + + def _update_ui(self): + for w in self.body_frame.winfo_children(): + w.destroy() + + groups_users_grid = ctk.CTkFrame(self.body_frame, fg_color=COLOR_OUTLINE_LIGHT, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + groups_users_grid.pack(fill="x", expand=True, pady=(5, 10)) + + groups_users_grid.grid_columnconfigure(0, weight=3) + groups_users_grid.grid_columnconfigure(1, weight=1) + + groups_users_headers = ["Category", "Count"] + for col_idx, head_text in enumerate(groups_users_headers): + cell = ctk.CTkFrame(groups_users_grid, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + rows_data = [ + ("Total Users", self.last_user_counts.get("total", 0), True), + ("Enabled Users", self.last_user_counts.get("enabled", 0), False), + ("Disabled Users", self.last_user_counts.get("disabled", 0), False), + ("Member Users", self.last_user_counts.get("member", 0), False), + ("Guest Users", self.last_user_counts.get("guest", 0), False), + (None, None, False), + ("Total Groups", self.last_group_counts.get("total", 0), True), + ("Microsoft 365 Groups (Unified)", self.last_group_counts.get("m365", 0), False), + ("Security Groups (Static, non-mail-enabled)", self.last_group_counts.get("security", 0), False), + ("Mail-enabled Security Groups", self.last_group_counts.get("mail_enabled_security", 0), False), + ("Distribution Groups", self.last_group_counts.get("distribution", 0), False), + ("Dynamic Groups (Dynamic Membership)", self.last_group_counts.get("dynamic", 0), False) + ] + + current_row = 1 + for item in rows_data: + metric_name, val, is_bold = item + if metric_name is None: + for c_idx in range(2): + c = ctk.CTkFrame(groups_users_grid, fg_color=COLOR_OUTLINE_LIGHT, corner_radius=0, height=2) + c.grid(row=current_row, column=c_idx, sticky="nsew", padx=0, pady=(0, 1)) + current_row += 1 + continue + + bg_style = COLOR_SURFACE if current_row % 2 == 0 else COLOR_SURFACE_VARIANT + font_style = FONT_BODY_BOLD if is_bold else FONT_BODY_MEDIUM + + c0 = ctk.CTkFrame(groups_users_grid, fg_color=bg_style, corner_radius=0) + c0.grid(row=current_row, column=0, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c0, text=metric_name, font=font_style, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=8, anchor="nw") + + c1 = ctk.CTkFrame(groups_users_grid, fg_color=bg_style, corner_radius=0) + c1.grid(row=current_row, column=1, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c1, text=f"{val:,}", font=font_style, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=8, anchor="nw") + + current_row += 1 + + def cancel(self): + self.is_cancelled = True + self.current_request_id += 1 + if self.status == "loading": + self.status = "cancelled" + self._update_ui() + if hasattr(self, "btn_refresh") and self.btn_refresh.winfo_exists(): + self.btn_refresh.configure(state="normal") + + @property + def last_data(self): + # We can reconstruct group_counts and user_counts from self.last_group_counts / self.last_user_counts, + # or load from SQLite if empty + if self.last_group_counts or self.last_user_counts: + return {"group_counts": self.last_group_counts, "user_counts": self.last_user_counts} + + # Load from SQLite fallback + if not self.csv_path: + return {"group_counts": {}, "user_counts": {}} + + reports_dir = os.path.dirname(self.csv_path) + db_path = os.path.join(reports_dir, "telemetry_cache.db") + if not os.path.exists(db_path): + return {"group_counts": {}, "user_counts": {}} + + group_c = {} + user_c = {} + + try: + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + cursor = conn.cursor() + cursor.execute("SELECT Category, Count FROM directory_users_groups") + for row in cursor.fetchall(): + cat = row["Category"] + try: + val = int(row["Count"]) + except ValueError: + continue + if cat == "Total Users": user_c["total"] = val + elif cat == "Enabled Users": user_c["enabled"] = val + elif cat == "Disabled Users": user_c["disabled"] = val + elif cat == "Member Users": user_c["member"] = val + elif cat == "Guest Users": user_c["guest"] = val + elif cat == "Total Groups": group_c["total"] = val + elif cat == "Microsoft 365 Groups (Unified)": group_c["m365"] = val + elif cat == "Security Groups (Static, non-mail-enabled)": group_c["security"] = val + elif cat == "Mail-enabled Security Groups": group_c["mail_enabled_security"] = val + elif cat == "Distribution Groups": group_c["distribution"] = val + elif cat == "Dynamic Groups (Dynamic Membership)": group_c["dynamic"] = val + conn.close() + except Exception: + pass + + return {"group_counts": group_c, "user_counts": user_c} diff --git a/telemetry/ediscovery_ui.py b/telemetry/ediscovery_ui.py new file mode 100644 index 00000000..b7b50607 --- /dev/null +++ b/telemetry/ediscovery_ui.py @@ -0,0 +1,292 @@ +import customtkinter as ctk +import time +import threading +import logging +from telemetry.styles import * +from core.graph.delegated_auth import DelegatedAuthClient +from core.graph.ediscovery import EDiscoveryFetcher + +logger = logging.getLogger("M365Telemetry.EDiscoveryUI") + +class EDiscoveryFrame(ctk.CTkFrame): + """Component for rendering Microsoft Purview eDiscovery Cases using Delegated Auth.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, **kwargs): + self.semaphore = kwargs.pop("concurrency_semaphore", None) + self.get_delegated_auth = kwargs.pop("delegated_auth_callback", None) + super().__init__(master, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=12, **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change_cb = status_change_callback + self.status = None + self.error_msg = None + self.loading = False + + self.page_index = 0 + self.page_size = 5 + self.last_data = [] + + self.build_ui() + + def build_ui(self): + self.pack(fill="x", expand=True, pady=10) + + self.inner_pad = ctk.CTkFrame(self, fg_color="transparent") + self.inner_pad.pack(fill="both", expand=True, padx=20, pady=20) + + self.header = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.header.pack(fill="x", pady=(0, 10)) + + self.title_lbl = ctk.CTkLabel( + self.header, + text="Microsoft Purview eDiscovery Cases", + font=FONT_HEADER_SMALL, + text_color=COLOR_TEXT_MAIN + ) + self.title_lbl.pack(side="left") + + self.reload_btn = ctk.CTkButton( + self.header, text="↻ Reload", width=80, height=28, corner_radius=6, + fg_color="transparent", border_width=1, border_color=COLOR_PRIMARY, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, + command=self._retry_fetch + ) + self.reload_btn.pack(side="right") + + self.state_frame = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + + self.grid_frame = ctk.CTkFrame(self.inner_pad, fg_color=COLOR_SURFACE, corner_radius=12, border_width=1, border_color=COLOR_OUTLINE_LIGHT) + self.grid_frame.pack_forget() + + self.pagination_frame = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.pagination_frame.pack_forget() + + self.loading_label = None + self.progress = None + self.render_ui_state() + + def trigger_fetch(self, tenant, client_id, secret, use_delegated_auth=False): + if not use_delegated_auth: + self.error_msg = "eDiscovery scanning requires Delegated Authentication. Please recreate the connection with Delegated Auth enabled." + self.status = "error" + self.render_ui_state() + return + + self.pack(fill="x", expand=True, pady=(0, 5)) + self.status = "loading" + self.loading = True + self.error_msg = None + self.render_ui_state() + threading.Thread(target=self._fetch_data, args=(tenant, client_id, secret), daemon=True).start() + + def _fetch_data(self, tenant, client_id, secret): + if self.semaphore: + self.semaphore.acquire() + try: + if hasattr(self, 'loading_label') and self.loading_label.winfo_exists(): + self.loading_label.configure(text="⏳ Authenticating via MSAL...") + + auth_client = DelegatedAuthClient(tenant, client_id, secret) + token = auth_client.get_token(scopes=["https://graph.microsoft.com/.default"]) + + if not token: + raise Exception("Failed to acquire delegated token. User may have cancelled or app is misconfigured.") + + if hasattr(self, 'loading_label') and self.loading_label.winfo_exists(): + self.loading_label.configure(text="⏳ Fetching eDiscovery Cases...") + + import os + csv_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'reports', f"{tenant}_{client_id}") + os.makedirs(csv_dir, exist_ok=True) + self.csv_path = os.path.join(csv_dir, "ediscovery_cases.csv") + self.total_items = 0 + + def update_progress(items): + self.total_items += len(items) + if hasattr(self, 'loading_label') and self.loading_label.winfo_exists(): + self.loading_label.configure(text=f"⏳ Fetched {self.total_items} eDiscovery cases...") + + fetcher = EDiscoveryFetcher(token) + res = fetcher.fetch_cases(csv_path=self.csv_path, on_page_callback=update_progress) + + if not res.get("success", False): + raise Exception(res.get("error", "Unknown error fetching eDiscovery cases")) + + self.last_data = [] # Free memory + self.after(0, self._render_success) + + except Exception as e: + logger.error(f"Error fetching eDiscovery cases: {e}", exc_info=True) + self.after(0, self._render_error, str(e)) + finally: + if self.semaphore: + self.semaphore.release() + + def _render_success(self): + self.status = "success" + self.loading = False + self.page_index = 0 + self.render_ui_state() + if self.on_status_change_cb: + self.on_status_change_cb() + + def _render_error(self, err_msg): + self.error_msg = err_msg + self.status = "error" + self.loading = False + self.render_ui_state() + if self.on_status_change_cb: + self.on_status_change_cb() + + def reset_view(self): + self.pack_forget() + self.state_frame.pack_forget() + self.grid_frame.pack_forget() + self.pagination_frame.pack_forget() + self.status = None + self.error_msg = None + + def _retry_fetch(self): + tenant, clients, secrets = self.get_credentials() + use_delegated = self.get_delegated_auth() if self.get_delegated_auth else False + if tenant and clients and secrets: + if hasattr(self, 'reload_btn') and self.reload_btn.winfo_exists(): + self.reload_btn.configure(state="disabled") + self.trigger_fetch(tenant, clients[0], secrets[0], use_delegated_auth=use_delegated) + + def render_ui_state(self): + self.state_frame.pack_forget() + self.grid_frame.pack_forget() + self.pagination_frame.pack_forget() + + for widget in self.state_frame.winfo_children(): + widget.destroy() + for widget in self.grid_frame.winfo_children(): + widget.destroy() + for widget in self.pagination_frame.winfo_children(): + widget.destroy() + + if not self.loading and hasattr(self, 'reload_btn') and self.reload_btn.winfo_exists(): + self.reload_btn.configure(state="normal") + + if self.loading: + self.loading_label = ctk.CTkLabel(self.state_frame, text="⏳ Initializing...", text_color="#6b7280", font=ctk.CTkFont(family="Segoe UI", size=13)) + self.loading_label.pack(pady=(20, 5)) + self.progress = ctk.CTkProgressBar(self.state_frame, mode="indeterminate", width=250, fg_color=COLOR_SURFACE_VARIANT, progress_color=COLOR_PRIMARY) + self.progress.pack(pady=(0, 20)) + self.progress.start() + self.state_frame.pack(fill="x", expand=True) + return + + if self.error_msg: + ctk.CTkLabel(self.state_frame, text=f"✖ {self.error_msg}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center", wraplength=700).pack(pady=(20, 5)) + ctk.CTkButton(self.state_frame, text="Try Again", command=self._retry_fetch, width=120, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 20)) + self.state_frame.pack(fill="x", expand=True) + return + + self.grid_frame.pack(fill="x", pady=(0, 10)) + self.pagination_frame.pack(fill="x", pady=(5, 0)) + self._update_ui_paginated() + + def _update_ui_paginated(self): + for widget in self.grid_frame.winfo_children(): + widget.destroy() + for widget in self.pagination_frame.winfo_children(): + widget.destroy() + + total_items = getattr(self, 'total_items', 0) + + if total_items == 0: + c = ctk.CTkFrame(self.grid_frame, fg_color=COLOR_SURFACE, corner_radius=0) + c.pack(fill="x", expand=True) + ctk.CTkLabel(c, text="No eDiscovery Cases discovered.", font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN, justify="center").pack(padx=10, pady=20) + return + + metrics_grid = ctk.CTkFrame(self.grid_frame, fg_color=COLOR_SURFACE, corner_radius=0) + metrics_grid.pack(fill="x", padx=10, pady=(5, 5)) + + headers = ["Display Name", "Status", "Created DateTime", "Closed By"] + for i in range(4): + metrics_grid.grid_columnconfigure(i, weight=1 if i > 0 else 2) + + for col_idx, head_text in enumerate(headers): + cell = ctk.CTkFrame(metrics_grid, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + start_idx = self.page_index * self.page_size + end_idx = min(start_idx + self.page_size, total_items) + + page_items = [] + if getattr(self, 'csv_path', None): + try: + import csv + with open(self.csv_path, 'r', encoding='utf-8') as f: + reader = csv.DictReader(f) + for i, row in enumerate(reader): + if i >= start_idx and i < end_idx: + page_items.append(row) + elif i >= end_idx: + break + except Exception: + pass + + for r_idx, case in enumerate(page_items, start=1): + bg_style = COLOR_SURFACE if r_idx % 2 != 0 else COLOR_SURFACE_VARIANT + + c0 = ctk.CTkFrame(metrics_grid, fg_color=bg_style, corner_radius=0) + c0.grid(row=r_idx, column=0, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c0, text=str(case.get("displayName", "-")), font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN, justify="left", wraplength=250).pack(padx=10, pady=12, anchor="w") + + c1 = ctk.CTkFrame(metrics_grid, fg_color=bg_style, corner_radius=0) + c1.grid(row=r_idx, column=1, sticky="nsew", padx=0, pady=(0, 1)) + status_text = case.get("status", "-") + status_color = "#16a34a" if status_text.lower() == "active" else COLOR_TEXT_SUB + ctk.CTkLabel(c1, text=status_text, font=FONT_BODY_MEDIUM, text_color=status_color).pack(padx=10, pady=12, anchor="w") + + c2 = ctk.CTkFrame(metrics_grid, fg_color=bg_style, corner_radius=0) + c2.grid(row=r_idx, column=2, sticky="nsew", padx=0, pady=(0, 1)) + dt = str(case.get("createdDateTime", "-")).split("T")[0] + ctk.CTkLabel(c2, text=dt, font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=12, anchor="w") + + c3 = ctk.CTkFrame(metrics_grid, fg_color=bg_style, corner_radius=0) + c3.grid(row=r_idx, column=3, sticky="nsew", padx=0, pady=(0, 1)) + closed_by = case.get("closedBy", {}) + user_info = closed_by.get("user", {}) if closed_by else {} + user_disp = user_info.get("displayName", "-") + ctk.CTkLabel(c3, text=user_disp, font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=12, anchor="w") + + if total_items >= 0: + total_pages = max(1, (total_items + self.page_size - 1) // self.page_size) + + center_container = ctk.CTkFrame(self.pagination_frame, fg_color="transparent") + center_container.pack(pady=(5, 10)) + + prev_btn = ctk.CTkButton(center_container, text="◀ Prev", width=70, height=22, corner_radius=6, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, + state="normal" if self.page_index > 0 else "disabled", + command=self._prev_page) + prev_btn.pack(side="left", padx=5) + + ctk.CTkLabel(center_container, text=f"Page {self.page_index + 1} of {total_pages}", + font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_SUB).pack(side="left", padx=15) + + next_btn = ctk.CTkButton(center_container, text="Next ▶", width=70, height=22, corner_radius=6, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, + state="normal" if self.page_index < total_pages - 1 else "disabled", + command=self._next_page) + next_btn.pack(side="left", padx=5) + + def _prev_page(self): + if self.page_index > 0: + self.page_index -= 1 + self._update_ui_paginated() + + def _next_page(self): + total_items = len(self.last_data) + total_pages = max(1, (total_items + self.page_size - 1) // self.page_size) + if self.page_index < total_pages - 1: + self.page_index += 1 + self._update_ui_paginated() diff --git a/telemetry/email_client_support.py b/telemetry/email_client_support.py new file mode 100644 index 00000000..49a8c2a8 --- /dev/null +++ b/telemetry/email_client_support.py @@ -0,0 +1,22 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Backward compatibility facade for Exchange Online Supported Email Clients and PST files telemetry.""" + +# Re-export pipelines from core backend +from core.graph.exchange.email_clients import run_email_client_usage_pipeline +from core.graph.exchange.pst_files import run_pst_discovery_pipeline + +# Re-export UI subframe from telemetry package +from telemetry.exchange.email_clients import EmailClientSupportFrame diff --git a/telemetry/entra/__init__.py b/telemetry/entra/__init__.py new file mode 100644 index 00000000..b4f43204 --- /dev/null +++ b/telemetry/entra/__init__.py @@ -0,0 +1,149 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Consolidated Microsoft Entra Data (Sign-in & Authentication Methods) Telemetry Orchestrator Container.""" + +import logging +import customtkinter as ctk + +from telemetry.styles import * +from telemetry.entra.auth_methods import AuthMethodsSubFrame +from telemetry.entra.app_signins import AppSigninsSubFrame +from telemetry.entra.user_signins import UserSigninsSubFrame +from telemetry.entra.app_registrations import AppRegistrationsSubFrame + +usage_logger = logging.getLogger("M365TelemetryAsyncLogger.DevicesAppsUI") + +class DevicesAppsTelemetryFrame(ctk.CTkFrame): + """Self-contained component wrapping Microsoft Entra Data UI with independent sub-sections.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, **kwargs): + self.semaphore = kwargs.pop("concurrency_semaphore", None) + super().__init__(master, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=12, **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.status = None + + self.build_ui() + + def build_ui(self): + self.pack(fill="x", expand=True, pady=10) + + self.inner_pad = ctk.CTkFrame(self, fg_color="transparent") + self.inner_pad.pack(fill="both", expand=True, padx=20, pady=20) + + ctk.CTkLabel(self.inner_pad, text="Microsoft Entra Data", font=FONT_HEADER_SMALL, text_color=COLOR_TEXT_MAIN).pack(anchor="w", pady=(0, 10)) + + # 1. Authentication Methods at the top + self.auth_methods_subframe = AuthMethodsSubFrame( + self.inner_pad, + log_callback=self.log_msg, + credentials_callback=self.get_credentials, + status_change_callback=self._subframe_status_changed, + semaphore=self.semaphore + ) + self.auth_methods_subframe.pack(fill="x", pady=(10, 15)) + + # Divider 1 + self.divider1 = ctk.CTkFrame(self.inner_pad, fg_color=COLOR_OUTLINE_LIGHT, height=1) + self.divider1.pack(fill="x", pady=15) + + # 2. App Registrations in the middle (between Auth Methods and App Sign Ins) + self.app_registrations_subframe = AppRegistrationsSubFrame( + self.inner_pad, + log_callback=self.log_msg, + credentials_callback=self.get_credentials, + status_change_callback=self._subframe_status_changed, + semaphore=self.semaphore + ) + self.app_registrations_subframe.pack(fill="x", pady=(0, 15)) + + # Divider 1.5 + self.divider1_5 = ctk.CTkFrame(self.inner_pad, fg_color=COLOR_OUTLINE_LIGHT, height=1) + self.divider1_5.pack(fill="x", pady=15) + + # 3. App Sign Ins below App Registrations + self.app_signins_subframe = AppSigninsSubFrame( + self.inner_pad, + log_callback=self.log_msg, + credentials_callback=self.get_credentials, + status_change_callback=self._subframe_status_changed, + semaphore=self.semaphore + ) + self.app_signins_subframe.pack(fill="x", pady=(0, 15)) + + # Divider 2 + self.divider2 = ctk.CTkFrame(self.inner_pad, fg_color=COLOR_OUTLINE_LIGHT, height=1) + self.divider2.pack(fill="x", pady=15) + + # 4. User Sign-Ins below App Sign Ins + self.user_signins_subframe = UserSigninsSubFrame( + self.inner_pad, + log_callback=self.log_msg, + credentials_callback=self.get_credentials, + status_change_callback=self._subframe_status_changed, + semaphore=self.semaphore + ) + self.user_signins_subframe.pack(fill="x", pady=(0, 15)) + + def _subframe_status_changed(self): + statuses = [ + self.auth_methods_subframe.status, + self.app_registrations_subframe.status, + self.app_signins_subframe.status, + self.user_signins_subframe.status + ] + if "loading" in statuses: + self.status = "loading" + elif "error" in statuses: + self.status = "error" + elif "success" in statuses: + self.status = "success" + else: + self.status = None + self.on_status_change() + + def reset_view(self): + self.pack_forget() + self.status = None + self.auth_methods_subframe.reset_view() + self.app_registrations_subframe.reset_view() + self.app_signins_subframe.reset_view() + self.user_signins_subframe.reset_view() + + def trigger_fetch(self, tenant, client_id, client_secret): + usage_logger.info("Entra Data trigger_fetch called. Propagating to independent sub-sections...") + self.pack(fill="x", expand=True, pady=10) + self.auth_methods_subframe.trigger_fetch(tenant, client_id, client_secret) + self.app_registrations_subframe.trigger_fetch(tenant, client_id, client_secret) + self.app_signins_subframe.trigger_fetch(tenant, client_id, client_secret) + self.user_signins_subframe.trigger_fetch(tenant, client_id, client_secret) + + def cancel(self): + usage_logger.info("Entra Data cancel called. Propagating to independent sub-sections...") + self.auth_methods_subframe.cancel() + self.app_registrations_subframe.cancel() + self.app_signins_subframe.cancel() + self.user_signins_subframe.cancel() + + @property + def last_data(self): + return { + "app_signins": self.app_signins_subframe.last_data, + "auth_methods": self.auth_methods_subframe.last_data, + "auth_methods_period": getattr(self.auth_methods_subframe, "period", "D7"), + "app_registrations": self.app_registrations_subframe.last_data, + "user_signins": self.user_signins_subframe.last_data + } diff --git a/telemetry/entra/app_registrations.py b/telemetry/entra/app_registrations.py new file mode 100644 index 00000000..38ec613c --- /dev/null +++ b/telemetry/entra/app_registrations.py @@ -0,0 +1,382 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""UI frame for Microsoft Entra App Registrations telemetry.""" + +import os +import csv +import logging +import threading +import asyncio +import sqlite3 +import customtkinter as ctk + +from core.graph.entra.app_registrations import run_app_registrations_pipeline +from core.graph.db import import_csv_to_sqlite, query_page_sync +from telemetry.styles import * + +usage_logger = logging.getLogger("M365TelemetryAsyncLogger.AppRegistrationsUI") + +class AppRegistrationsSubFrame(ctk.CTkFrame): + """Sub-frame for Microsoft Entra App Registrations with UI Pagination.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, semaphore=None, **kwargs): + super().__init__(master, fg_color="transparent", **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.semaphore = semaphore + self.status = None + self._cached_app_registrations = [] + self.is_cancelled = False + self.current_request_id = 0 + + # Pagination variables (5 rows per page) + self.ITEMS_PER_PAGE = 5 + self.current_page = 0 + self.csv_path = None + + self.build_ui() + + def build_ui(self): + self.header_frame = ctk.CTkFrame(self, fg_color="transparent") + self.header_frame.pack(fill="x", pady=(0, 5)) + + ctk.CTkLabel(self.header_frame, text="App Registrations", font=FONT_SUBSECTION_HEADER, text_color=COLOR_PRIMARY).pack(side="left") + + self.btn_refresh = ctk.CTkButton( + self.header_frame, state="disabled", text="↻ Reload", width=80, height=26, corner_radius=13, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, + command=self.trigger_fetch_individual + ) + self.btn_refresh.pack(side="right", padx=(10, 0)) + + self.body_frame = ctk.CTkFrame(self, fg_color="transparent") + self.body_frame.pack(fill="x") + + self.reset_view() + + def reset_view(self): + self.status = None + self._cached_app_registrations = [] + self.is_cancelled = False + self.current_page = 0 + self.csv_path = None + for w in self.body_frame.winfo_children(): + w.destroy() + + def _set_state_loading(self, msg="Loading..."): + for w in self.body_frame.winfo_children(): + w.destroy() + ctk.CTkLabel(self.body_frame, text=f"⏳ {msg}", text_color=COLOR_TEXT_SUB, font=FONT_BODY_MEDIUM).pack(pady=10) + + def _set_state_error(self, error_msg): + for w in self.body_frame.winfo_children(): + w.destroy() + display_msg = error_msg + if "401" in error_msg or "403" in error_msg or "unauthorized" in error_msg.lower() or "forbidden" in error_msg.lower() or "application.read" in error_msg.lower(): + display_msg = "Application.Read.All application permission required.\nPlease grant 'Application.Read.All' to your App Registration in Microsoft Entra ID." + ctk.CTkLabel(self.body_frame, text=f"✖ {display_msg}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center").pack(pady=(10, 5)) + ctk.CTkButton(self.body_frame, text="Try Again", command=self.trigger_fetch_individual, width=120, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 10)) + + def trigger_fetch_individual(self): + tenant, clients, secrets = self.get_credentials() + if tenant: + self.trigger_fetch(tenant, clients[0], secrets[0]) + else: + self._set_state_error("Missing credentials. Please submit the credentials above.") + + def trigger_fetch(self, tenant, client_id, client_secret): + self.status = "loading" + self.is_cancelled = False + self.current_page = 0 + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + if os.path.basename(script_dir) == "entra": + script_dir = os.path.dirname(script_dir) + reports_dir = os.path.join(script_dir, "reports", f"{tenant}_{client_id}") + self.csv_path = os.path.join(reports_dir, "entra_app_registrations.csv") + + self._set_state_loading("Downloading and parsing App Registrations...") + self.on_status_change() + if hasattr(self, "btn_refresh") and self.btn_refresh.winfo_exists(): + self.btn_refresh.configure(state="disabled") + + threading.Thread( + target=self._execute_worker, + args=(tenant, client_id, client_secret, self.current_request_id), + daemon=True + ).start() + + def _execute_worker(self, tenant, client_id, client_secret, request_id): + if self.semaphore: + self.semaphore.acquire() + try: + if self.is_cancelled or request_id != self.current_request_id: + return + + app_registrations = [] + page_count = 0 + def handle_page(value_list): + nonlocal app_registrations, page_count + if self.is_cancelled or request_id != self.current_request_id: + return + for item in value_list: + displayName = item.get("displayName") or "" + appId = item.get("appId") or "" + createdDateTime = item.get("createdDateTime") or "" + signInAudience = item.get("signInAudience") or "" + + secrets_cnt = len(item.get("passwordCredentials", [])) + certs_cnt = len(item.get("keyCredentials", [])) + credentials_str = f"{secrets_cnt} Secrets, {certs_cnt} Certs" + + app_registrations.append((displayName, appId, createdDateTime, signInAudience, credentials_str)) + page_count += 1 + if page_count % 3 == 0: + self.after(0, self._render_partial, list(app_registrations), request_id) + + temp_csv_path = self.csv_path + ".tmp" + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + if os.path.basename(script_dir) == "entra": + script_dir = os.path.dirname(script_dir) + reports_dir = os.path.join(script_dir, "reports", f"{tenant}_{client_id}") + os.makedirs(reports_dir, exist_ok=True) + + with open(temp_csv_path, 'w', encoding='utf-8', newline='') as f: + writer = csv.writer(f) + writer.writerow(["displayName", "appId", "createdDateTime", "signInAudience", "credentials"]) + + run_app_registrations_pipeline( + client_id=client_id, + client_secret=client_secret, + tenant_id=tenant, + csv_path=temp_csv_path, + max_rows=5000, + on_page_callback=handle_page, + is_cancelled_callback=lambda: self.is_cancelled or request_id != self.current_request_id + ) + + if not self.is_cancelled and request_id == self.current_request_id: + if os.path.exists(temp_csv_path): + if os.path.exists(self.csv_path): + os.remove(self.csv_path) + os.rename(temp_csv_path, self.csv_path) + import asyncio + db_dir = os.path.dirname(self.csv_path) + db_path = os.path.join(db_dir, "telemetry_cache.db") + asyncio.run(import_csv_to_sqlite(self.csv_path, db_path, "app_registrations", "displayName")) + + if self.is_cancelled or request_id != self.current_request_id: + return + self.after(0, self._render_success, app_registrations, request_id) + except Exception as e: + usage_logger.error(f"Error fetching app registrations: {e}", exc_info=True) + if not self.is_cancelled and request_id == self.current_request_id: + self.after(0, self._render_error, str(e), request_id) + finally: + if 'temp_csv_path' in locals() and os.path.exists(temp_csv_path): + try: + os.remove(temp_csv_path) + except Exception: + pass + if self.semaphore: + self.semaphore.release() + + def _render_partial(self, data, request_id): + if self.is_cancelled or request_id != self.current_request_id: + return + self._update_ui_paginated(data, is_partial=True) + + def _render_success(self, data, request_id): + if self.is_cancelled or request_id != self.current_request_id: + return + self.status = "success" + self._cached_app_registrations = data + self._update_ui_paginated(data=None, is_partial=False) + self.on_status_change() + if hasattr(self, "btn_refresh") and self.btn_refresh.winfo_exists(): + self.btn_refresh.configure(state="normal") + + def _render_error(self, err_msg, request_id): + if self.is_cancelled or request_id != self.current_request_id: + return + self.status = "error" + self._set_state_error(err_msg) + self.on_status_change() + if hasattr(self, "btn_refresh") and self.btn_refresh.winfo_exists(): + self.btn_refresh.configure(state="normal") + + def _load_page_from_csv(self, page): + if not self.csv_path or not os.path.exists(self.csv_path): + return [], 0 + + db_dir = os.path.dirname(self.csv_path) + db_path = os.path.join(db_dir, "telemetry_cache.db") + try: + page_data, total_count = query_page_sync(db_path, "app_registrations", page, self.ITEMS_PER_PAGE) + mapped_data = [] + for r in page_data: + mapped_data.append(( + r.get("displayName", ""), + r.get("appId", ""), + r.get("createdDateTime", ""), + r.get("signInAudience", ""), + r.get("credentials", "") + )) + return mapped_data, total_count + except Exception as e: + usage_logger.error(f"Error loading page from SQLite cache: {e}") + return [], 0 + + def _update_ui_paginated(self, data=None, is_partial=False): + for w in self.body_frame.winfo_children(): + w.destroy() + + if is_partial: + progress_frame = ctk.CTkFrame(self.body_frame, fg_color=COLOR_TONAL_BG, height=26, corner_radius=6) + progress_frame.pack(fill="x", pady=(0, 6)) + ctk.CTkLabel( + progress_frame, + text="⏳ Querying App Registrations in the background... UI will auto-refresh.", + font=FONT_BODY_SMALL, + text_color=COLOR_TONAL_TEXT + ).pack(padx=10, pady=2, anchor="w") + + # Get the page slice + if data is not None: + total_count = len(data) + start_idx = self.current_page * self.ITEMS_PER_PAGE + end_idx = start_idx + self.ITEMS_PER_PAGE + page_data = data[start_idx:end_idx] + else: + page_data, total_count = self._load_page_from_csv(self.current_page) + + # Draw the table grid + metrics_grid = ctk.CTkFrame(self.body_frame, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + metrics_grid.pack(fill="x", pady=(5, 10)) + + headers = ["App Name", "Application ID", "Created Date", "Sign In Audience", "Credentials"] + for i in range(5): + metrics_grid.grid_columnconfigure(i, weight=1) + + for col_idx, head_text in enumerate(headers): + cell = ctk.CTkFrame(metrics_grid, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + if not page_data: + c = ctk.CTkFrame(metrics_grid, fg_color=COLOR_SURFACE, corner_radius=0) + c.grid(row=1, column=0, columnspan=5, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c, text="No app registrations detected.", font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN, justify="center").pack(padx=10, pady=12) + else: + for r_idx, (name, app_id, created, audience, creds) in enumerate(page_data, start=1): + bg_style = COLOR_SURFACE if r_idx % 2 != 0 else COLOR_SURFACE_VARIANT + formatted_created = created[:10] if created else "" + vals = [name, app_id, formatted_created, audience, creds] + for c_idx, val in enumerate(vals): + c = ctk.CTkFrame(metrics_grid, fg_color=bg_style, corner_radius=0) + c.grid(row=r_idx, column=c_idx, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c, text=val, font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN, justify="left", wraplength=180).pack(padx=10, pady=8, anchor="nw") + + # Draw the pagination controls if we have items + if total_count > 0: + self._draw_pagination_controls(total_count, data, is_partial) + + def _draw_pagination_controls(self, total_count, data, is_partial): + total_pages = (total_count + self.ITEMS_PER_PAGE - 1) // self.ITEMS_PER_PAGE + if total_pages <= 1: + return + + control_frame = ctk.CTkFrame(self.body_frame, fg_color=COLOR_SURFACE) + control_frame.pack(fill="x", pady=0) + + center_container = ctk.CTkFrame(control_frame, fg_color="transparent") + center_container.pack(pady=(5, 10)) + + prev_state = "normal" if self.current_page > 0 else "disabled" + btn_prev = ctk.CTkButton( + center_container, text="◀ Prev", width=70, height=22, corner_radius=6, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, + state=prev_state, + command=lambda: self._change_page(-1, data, is_partial) + ) + btn_prev.pack(side="left", padx=5) + + page_lbl = ctk.CTkLabel( + center_container, + text=f"Page {self.current_page + 1} of {total_pages} ({total_count} items)", + font=FONT_BODY_MEDIUM, + text_color=COLOR_TEXT_SUB + ) + page_lbl.pack(side="left", padx=15) + + next_state = "normal" if self.current_page < total_pages - 1 else "disabled" + btn_next = ctk.CTkButton( + center_container, text="Next ▶", width=70, height=22, corner_radius=6, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, + state=next_state, + command=lambda: self._change_page(1, data, is_partial) + ) + btn_next.pack(side="left", padx=5) + + def _change_page(self, delta, data, is_partial): + self.current_page += delta + self._update_ui_paginated(data, is_partial) + + def cancel(self): + self.is_cancelled = True + self.current_request_id += 1 + if self.status == "loading": + self.status = "cancelled" + self._update_ui_paginated(self.last_data) + if hasattr(self, "btn_refresh") and self.btn_refresh.winfo_exists(): + self.btn_refresh.configure(state="normal") + + def _load_data_from_csv(self): + if not self.csv_path or not os.path.exists(self.csv_path): + tenant, clients, secrets = self.get_credentials() + if tenant and clients: + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + if os.path.basename(script_dir) == "entra": + script_dir = os.path.dirname(script_dir) + self.csv_path = os.path.join(script_dir, "reports", f"{tenant}_{clients[0]}", "entra_app_registrations.csv") + else: + return [] + + db_dir = os.path.dirname(self.csv_path) + db_path = os.path.join(db_dir, "telemetry_cache.db") + if not os.path.exists(db_path): + return [] + + items = [] + try: + conn = sqlite3.connect(db_path) + cursor = conn.cursor() + cursor.execute("SELECT displayName, appId, createdDateTime, signInAudience, credentials FROM app_registrations") + for r in cursor.fetchall(): + items.append((r[0], r[1], r[2], r[3], r[4])) + conn.close() + except Exception as e: + usage_logger.error(f"Error reading SQLite cache for AppRegistrationsSubFrame: {e}", exc_info=True) + return items + + @property + def last_data(self): + if hasattr(self, "_cached_app_registrations") and self._cached_app_registrations: + return self._cached_app_registrations + return self._load_data_from_csv() diff --git a/telemetry/entra/app_signins.py b/telemetry/entra/app_signins.py new file mode 100644 index 00000000..add382d3 --- /dev/null +++ b/telemetry/entra/app_signins.py @@ -0,0 +1,370 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""UI frame for Microsoft Entra App Sign-ins telemetry.""" + +import os +import csv +import logging +import threading +import customtkinter as ctk + +from core.graph.entra.app_signins import run_app_signins_pipeline +from core.graph.db import import_csv_to_sqlite, query_page_sync +from telemetry.styles import * + +usage_logger = logging.getLogger("M365TelemetryAsyncLogger.AppSigninsUI") + +class AppSigninsSubFrame(ctk.CTkFrame): + """Sub-frame for Microsoft Entra App Sign Ins with UI Pagination.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, semaphore=None, **kwargs): + super().__init__(master, fg_color="transparent", **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.semaphore = semaphore + self.status = None + self._cached_app_signins = [] + self.is_cancelled = False + self.current_request_id = 0 + + # Pagination variables + self.ITEMS_PER_PAGE = 10 + self.current_page = 0 + self.csv_path = None + + self.build_ui() + + def build_ui(self): + self.header_frame = ctk.CTkFrame(self, fg_color="transparent") + self.header_frame.pack(fill="x", pady=(0, 5)) + + ctk.CTkLabel(self.header_frame, text="App Sign Ins", font=FONT_SUBSECTION_HEADER, text_color=COLOR_PRIMARY).pack(side="left") + + self.btn_refresh = ctk.CTkButton( + self.header_frame, state="disabled", text="↻ Reload", width=80, height=26, corner_radius=13, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, + command=self.trigger_fetch_individual + ) + self.btn_refresh.pack(side="right", padx=(10, 0)) + + self.body_frame = ctk.CTkFrame(self, fg_color="transparent") + self.body_frame.pack(fill="x") + + self.reset_view() + + def reset_view(self): + self.status = None + self._cached_app_signins = [] + self.is_cancelled = False + self.current_page = 0 + self.csv_path = None + for w in self.body_frame.winfo_children(): + w.destroy() + + def _set_state_loading(self, msg="Loading..."): + for w in self.body_frame.winfo_children(): + w.destroy() + ctk.CTkLabel(self.body_frame, text=f"⏳ {msg}", text_color=COLOR_TEXT_SUB, font=FONT_BODY_MEDIUM).pack(pady=10) + + def _set_state_error(self, error_msg): + for w in self.body_frame.winfo_children(): + w.destroy() + display_msg = error_msg + if "401" in error_msg or "403" in error_msg or "unauthorized" in error_msg.lower() or "forbidden" in error_msg.lower() or "reports.read.all" in error_msg.lower(): + display_msg = "Reports.Read.All application permission required.\nPlease grant 'Reports.Read.All' to your App Registration in Microsoft Entra ID." + ctk.CTkLabel(self.body_frame, text=f"✖ {display_msg}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center").pack(pady=(10, 5)) + ctk.CTkButton(self.body_frame, text="Try Again", command=self.trigger_fetch_individual, width=120, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 10)) + + def trigger_fetch_individual(self): + tenant, clients, secrets = self.get_credentials() + if tenant: + self.trigger_fetch(tenant, clients[0], secrets[0]) + else: + self._set_state_error("Missing credentials. Please submit the credentials above.") + + def trigger_fetch(self, tenant, client_id, client_secret): + self.status = "loading" + self.is_cancelled = False + self.current_page = 0 + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + if os.path.basename(script_dir) == "entra": + script_dir = os.path.dirname(script_dir) + reports_dir = os.path.join(script_dir, "reports", f"{tenant}_{client_id}") + self.csv_path = os.path.join(reports_dir, "entra_app_signins.csv") + + self._set_state_loading("Downloading and parsing App Sign Ins...") + self.on_status_change() + if hasattr(self, "btn_refresh") and self.btn_refresh.winfo_exists(): + self.btn_refresh.configure(state="disabled") + + threading.Thread( + target=self._execute_worker, + args=(tenant, client_id, client_secret, self.current_request_id), + daemon=True + ).start() + + def _execute_worker(self, tenant, client_id, client_secret, request_id): + if self.semaphore: + self.semaphore.acquire() + try: + if self.is_cancelled or request_id != self.current_request_id: + return + + app_signins = [] + page_count = 0 + def handle_page(value_list): + nonlocal app_signins, page_count + if self.is_cancelled or request_id != self.current_request_id: + return + for item in value_list: + app_name = item.get("appDisplayName") or "" + success = str(item.get("successfulSignInCount") or 0) + app_signins.append((app_name, success)) + page_count += 1 + if page_count % 3 == 0: + self.after(0, self._render_partial, list(app_signins), request_id) + + temp_csv_path = self.csv_path + ".tmp" + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + if os.path.basename(script_dir) == "entra": + script_dir = os.path.dirname(script_dir) + reports_dir = os.path.join(script_dir, "reports", f"{tenant}_{client_id}") + os.makedirs(reports_dir, exist_ok=True) + + with open(temp_csv_path, 'w', encoding='utf-8', newline='') as f: + writer = csv.writer(f) + writer.writerow(["appDisplayName", "successSignInCount"]) + + run_app_signins_pipeline( + client_id=client_id, + client_secret=client_secret, + tenant_id=tenant, + csv_path=temp_csv_path, + max_rows=5000, + on_page_callback=handle_page, + is_cancelled_callback=lambda: self.is_cancelled or request_id != self.current_request_id + ) + + if not self.is_cancelled and request_id == self.current_request_id: + if os.path.exists(temp_csv_path): + if os.path.exists(self.csv_path): + os.remove(self.csv_path) + os.rename(temp_csv_path, self.csv_path) + import asyncio + db_dir = os.path.dirname(self.csv_path) + db_path = os.path.join(db_dir, "telemetry_cache.db") + asyncio.run(import_csv_to_sqlite(self.csv_path, db_path, "app_signins", "appDisplayName")) + + if self.is_cancelled or request_id != self.current_request_id: + return + self.after(0, self._render_success, app_signins, request_id) + except Exception as e: + usage_logger.error(f"Error fetching app sign-ins: {e}", exc_info=True) + if not self.is_cancelled and request_id == self.current_request_id: + self.after(0, self._render_error, str(e), request_id) + finally: + if 'temp_csv_path' in locals() and os.path.exists(temp_csv_path): + try: + os.remove(temp_csv_path) + except Exception: + pass + if self.semaphore: + self.semaphore.release() + + def _render_partial(self, data, request_id): + if self.is_cancelled or request_id != self.current_request_id: + return + self._update_ui_paginated(data, is_partial=True) + + def _render_success(self, data, request_id): + if self.is_cancelled or request_id != self.current_request_id: + return + self.status = "success" + self._cached_app_signins = data + self._update_ui_paginated(data=None, is_partial=False) + self.on_status_change() + if hasattr(self, "btn_refresh") and self.btn_refresh.winfo_exists(): + self.btn_refresh.configure(state="normal") + + def _render_error(self, err_msg, request_id): + if self.is_cancelled or request_id != self.current_request_id: + return + self.status = "error" + self._set_state_error(err_msg) + self.on_status_change() + if hasattr(self, "btn_refresh") and self.btn_refresh.winfo_exists(): + self.btn_refresh.configure(state="normal") + + def _load_page_from_csv(self, page): + if not self.csv_path or not os.path.exists(self.csv_path): + return [], 0 + + db_dir = os.path.dirname(self.csv_path) + db_path = os.path.join(db_dir, "telemetry_cache.db") + try: + page_data, total_count = query_page_sync(db_path, "app_signins", page, self.ITEMS_PER_PAGE) + mapped_data = [] + for r in page_data: + mapped_data.append(( + r.get("appDisplayName", ""), + r.get("successSignInCount", "") + )) + return mapped_data, total_count + except Exception as e: + usage_logger.error(f"Error loading page from SQLite cache: {e}") + return [], 0 + + def _update_ui_paginated(self, data=None, is_partial=False): + for w in self.body_frame.winfo_children(): + w.destroy() + + if is_partial: + progress_frame = ctk.CTkFrame(self.body_frame, fg_color=COLOR_TONAL_BG, height=26, corner_radius=6) + progress_frame.pack(fill="x", pady=(0, 6)) + ctk.CTkLabel( + progress_frame, + text="⏳ Querying App Sign Ins in the background... UI will auto-refresh.", + font=FONT_BODY_SMALL, + text_color=COLOR_TONAL_TEXT + ).pack(padx=10, pady=2, anchor="w") + + # Get the page slice + if data is not None: + total_count = len(data) + start_idx = self.current_page * self.ITEMS_PER_PAGE + end_idx = start_idx + self.ITEMS_PER_PAGE + page_data = data[start_idx:end_idx] + else: + page_data, total_count = self._load_page_from_csv(self.current_page) + + # Draw the table grid + metrics_grid = ctk.CTkFrame(self.body_frame, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + metrics_grid.pack(fill="x", pady=(5, 10)) + + headers = ["App Name", "Successful Sign Ins"] + for i in range(2): + metrics_grid.grid_columnconfigure(i, weight=1) + + for col_idx, head_text in enumerate(headers): + cell = ctk.CTkFrame(metrics_grid, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + if not page_data: + c = ctk.CTkFrame(metrics_grid, fg_color=COLOR_SURFACE, corner_radius=0) + c.grid(row=1, column=0, columnspan=2, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c, text="No app sign-ins detected.", font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN, justify="center").pack(padx=10, pady=12) + else: + for r_idx, (app, success) in enumerate(page_data, start=1): + bg_style = COLOR_SURFACE if r_idx % 2 != 0 else COLOR_SURFACE_VARIANT + vals = [app, success] + for c_idx, val in enumerate(vals): + c = ctk.CTkFrame(metrics_grid, fg_color=bg_style, corner_radius=0) + c.grid(row=r_idx, column=c_idx, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c, text=val, font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN, justify="left", wraplength=350).pack(padx=10, pady=8, anchor="nw") + + # Draw the pagination controls if we have items + if total_count > 0: + self._draw_pagination_controls(total_count, data, is_partial) + + def _draw_pagination_controls(self, total_count, data, is_partial): + total_pages = (total_count + self.ITEMS_PER_PAGE - 1) // self.ITEMS_PER_PAGE + if total_pages <= 1: + return + + control_frame = ctk.CTkFrame(self.body_frame, fg_color=COLOR_SURFACE) + control_frame.pack(fill="x", pady=0) + + center_container = ctk.CTkFrame(control_frame, fg_color="transparent") + center_container.pack(pady=(5, 10)) + + prev_state = "normal" if self.current_page > 0 else "disabled" + btn_prev = ctk.CTkButton( + center_container, text="◀ Prev", width=70, height=22, corner_radius=6, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, + state=prev_state, + command=lambda: self._change_page(-1, data, is_partial) + ) + btn_prev.pack(side="left", padx=5) + + page_lbl = ctk.CTkLabel( + center_container, + text=f"Page {self.current_page + 1} of {total_pages} ({total_count} items)", + font=FONT_BODY_MEDIUM, + text_color=COLOR_TEXT_SUB + ) + page_lbl.pack(side="left", padx=15) + + next_state = "normal" if self.current_page < total_pages - 1 else "disabled" + btn_next = ctk.CTkButton( + center_container, text="Next ▶", width=70, height=22, corner_radius=6, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, + state=next_state, + command=lambda: self._change_page(1, data, is_partial) + ) + btn_next.pack(side="left", padx=5) + + def _change_page(self, delta, data, is_partial): + self.current_page += delta + self._update_ui_paginated(data, is_partial) + + def cancel(self): + self.is_cancelled = True + self.current_request_id += 1 + if self.status == "loading": + self.status = "cancelled" + self._update_ui_paginated(self.last_data) + if hasattr(self, "btn_refresh") and self.btn_refresh.winfo_exists(): + self.btn_refresh.configure(state="normal") + + def _load_data_from_csv(self): + if not self.csv_path or not os.path.exists(self.csv_path): + tenant, clients, secrets = self.get_credentials() + if tenant and clients: + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + if os.path.basename(script_dir) == "entra": + script_dir = os.path.dirname(script_dir) + self.csv_path = os.path.join(script_dir, "reports", f"{tenant}_{clients[0]}", "entra_app_signins.csv") + else: + return [] + + db_dir = os.path.dirname(self.csv_path) + db_path = os.path.join(db_dir, "telemetry_cache.db") + if not os.path.exists(db_path): + return [] + + import sqlite3 + items = [] + try: + conn = sqlite3.connect(db_path) + cursor = conn.cursor() + cursor.execute("SELECT appDisplayName, successSignInCount FROM app_signins") + for r in cursor.fetchall(): + items.append((r[0], r[1])) + conn.close() + except Exception as e: + usage_logger.error(f"Error reading SQLite cache for AppSigninsSubFrame: {e}", exc_info=True) + return items + + @property + def last_data(self): + if hasattr(self, "_cached_app_signins") and self._cached_app_signins: + return self._cached_app_signins + return self._load_data_from_csv() diff --git a/telemetry/entra/auth_methods.py b/telemetry/entra/auth_methods.py new file mode 100644 index 00000000..a0f95eca --- /dev/null +++ b/telemetry/entra/auth_methods.py @@ -0,0 +1,342 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""UI frame for Microsoft Entra Authentication Methods telemetry.""" + +import os +import csv +import logging +import threading +import asyncio +import sqlite3 +import customtkinter as ctk + +from core.graph.entra.auth_methods import run_auth_methods_pipeline +from core.graph.db import import_csv_to_sqlite +from telemetry.styles import * + +usage_logger = logging.getLogger("M365TelemetryAsyncLogger.AuthMethodsUI") + +class AuthMethodsSubFrame(ctk.CTkFrame): + """Sub-frame for Microsoft Entra Authentication Methods.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, semaphore=None, **kwargs): + super().__init__(master, fg_color="transparent", **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.semaphore = semaphore + self.status = None + self._cached_auth_methods = [] + self.is_cancelled = False + self.current_request_id = 0 + self.current_page = 0 + self.ITEMS_PER_PAGE = 5 + self.csv_path = None + self.period = "D7" + + self.build_ui() + + def build_ui(self): + self.header_frame = ctk.CTkFrame(self, fg_color="transparent") + self.header_frame.pack(fill="x", pady=(0, 5)) + + ctk.CTkLabel(self.header_frame, text="Authentication Methods", font=FONT_SUBSECTION_HEADER, text_color=COLOR_PRIMARY).pack(side="left") + + self.btn_refresh = ctk.CTkButton( + self.header_frame, state="disabled", text="↻ Reload", width=80, height=26, corner_radius=13, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, + command=self.trigger_fetch_individual + ) + self.btn_refresh.pack(side="right", padx=(10, 0)) + + self.body_frame = ctk.CTkFrame(self, fg_color="transparent") + self.body_frame.pack(fill="x") + + self.reset_view() + + def reset_view(self): + self.status = None + self._cached_auth_methods = [] + self.is_cancelled = False + self.csv_path = None + for w in self.body_frame.winfo_children(): + w.destroy() + + def _set_state_loading(self, msg="Loading..."): + for w in self.body_frame.winfo_children(): + w.destroy() + ctk.CTkLabel(self.body_frame, text=f"⏳ {msg}", text_color=COLOR_TEXT_SUB, font=FONT_BODY_MEDIUM).pack(pady=10) + + def _set_state_error(self, error_msg): + for w in self.body_frame.winfo_children(): + w.destroy() + display_msg = error_msg + if "401" in error_msg or "403" in error_msg or "unauthorized" in error_msg.lower() or "forbidden" in error_msg.lower() or "auditlog.read.all" in error_msg.lower(): + display_msg = "AuditLog.Read.All application permission required.\nPlease grant 'AuditLog.Read.All' to your App Registration in Microsoft Entra ID." + ctk.CTkLabel(self.body_frame, text=f"✖ {display_msg}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center").pack(pady=(10, 5)) + ctk.CTkButton(self.body_frame, text="Try Again", command=self.trigger_fetch_individual, width=120, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 10)) + + def trigger_fetch_individual(self): + tenant, clients, secrets = self.get_credentials() + if tenant: + self.trigger_fetch(tenant, clients[0], secrets[0]) + else: + self._set_state_error("Missing credentials. Please submit the credentials above.") + + def trigger_fetch(self, tenant, client_id, client_secret): + self.status = "loading" + self.is_cancelled = False + self.current_page = 0 + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + if os.path.basename(script_dir) == "entra": + script_dir = os.path.dirname(script_dir) + reports_dir = os.path.join(script_dir, "reports", f"{tenant}_{client_id}") + self.csv_path = os.path.join(reports_dir, "entra_auth_methods.csv") + + self._set_state_loading("Downloading and parsing Authentication Methods...") + self.on_status_change() + if hasattr(self, "btn_refresh") and self.btn_refresh.winfo_exists(): + self.btn_refresh.configure(state="disabled") + + threading.Thread( + target=self._execute_worker, + args=(tenant, client_id, client_secret, self.current_request_id), + daemon=True + ).start() + + def _execute_worker(self, tenant, client_id, client_secret, request_id): + if self.semaphore: + self.semaphore.acquire() + try: + if self.is_cancelled or request_id != self.current_request_id: + return + + auth_methods = [] + page_count = 0 + def handle_page(value_list): + nonlocal auth_methods, page_count + if self.is_cancelled or request_id != self.current_request_id: + return + for item in value_list: + method = item.get("authenticationMethod") or "" + activity = str(item.get("successActivityCount") or 0) + auth_methods.append((method, activity)) + page_count += 1 + if page_count % 3 == 0: + self.after(0, self._render_partial, list(auth_methods), request_id) + + temp_csv_path = self.csv_path + ".tmp" + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + if os.path.basename(script_dir) == "entra": + script_dir = os.path.dirname(script_dir) + reports_dir = os.path.join(script_dir, "reports", f"{tenant}_{client_id}") + os.makedirs(reports_dir, exist_ok=True) + + with open(temp_csv_path, 'w', encoding='utf-8', newline='') as f: + writer = csv.writer(f) + writer.writerow(["authenticationMethod", "successActivityCount"]) + + run_auth_methods_pipeline( + client_id=client_id, + client_secret=client_secret, + tenant_id=tenant, + csv_path=temp_csv_path, + period=self.period, + max_rows=5000, + on_page_callback=handle_page, + is_cancelled_callback=lambda: self.is_cancelled or request_id != self.current_request_id + ) + + if not self.is_cancelled and request_id == self.current_request_id: + if os.path.exists(temp_csv_path): + if os.path.exists(self.csv_path): + os.remove(self.csv_path) + os.rename(temp_csv_path, self.csv_path) + + db_path = os.path.join(reports_dir, "telemetry_cache.db") + asyncio.run(import_csv_to_sqlite(self.csv_path, db_path, "entra_auth_methods")) + + self.after(0, self._render_success, auth_methods, request_id) + except Exception as e: + usage_logger.error(f"Error fetching auth methods: {e}", exc_info=True) + if not self.is_cancelled and request_id == self.current_request_id: + self.after(0, self._render_error, str(e), request_id) + finally: + if 'temp_csv_path' in locals() and os.path.exists(temp_csv_path): + try: + os.remove(temp_csv_path) + except Exception: + pass + if self.semaphore: + self.semaphore.release() + + def _render_partial(self, data, request_id): + if self.is_cancelled or request_id != self.current_request_id: + return + self._update_ui_paginated(data, is_partial=True) + + def _render_success(self, data, request_id): + if self.is_cancelled or request_id != self.current_request_id: + return + self.status = "success" + self._cached_auth_methods = data + self._update_ui_paginated(data, is_partial=False) + self.on_status_change() + if hasattr(self, "btn_refresh") and self.btn_refresh.winfo_exists(): + self.btn_refresh.configure(state="normal") + + def _render_error(self, err_msg, request_id): + if self.is_cancelled or request_id != self.current_request_id: + return + self.status = "error" + self._set_state_error(err_msg) + self.on_status_change() + if hasattr(self, "btn_refresh") and self.btn_refresh.winfo_exists(): + self.btn_refresh.configure(state="normal") + + def _update_ui_paginated(self, data=None, is_partial=False): + for w in self.body_frame.winfo_children(): + w.destroy() + + if is_partial: + progress_frame = ctk.CTkFrame(self.body_frame, fg_color=COLOR_TONAL_BG, height=26, corner_radius=6) + progress_frame.pack(fill="x", pady=(0, 6)) + ctk.CTkLabel( + progress_frame, + text="⏳ Querying Authentication Methods in the background... UI will auto-refresh.", + font=FONT_BODY_SMALL, + text_color=COLOR_TONAL_TEXT + ).pack(padx=10, pady=2, anchor="w") + + if data is None: + data = self._load_data_from_sqlite() + + metrics_grid = ctk.CTkFrame(self.body_frame, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + metrics_grid.pack(fill="x", pady=(5, 10)) + + period_str = self.period + if period_str.startswith("D"): + period_str = f"{period_str[1:]} days" + headers = ["Authentication Method", f"Success Activity Count ({period_str})"] + for i in range(2): + metrics_grid.grid_columnconfigure(i, weight=1) + + for col_idx, head_text in enumerate(headers): + cell = ctk.CTkFrame(metrics_grid, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + if not data: + c = ctk.CTkFrame(metrics_grid, fg_color=COLOR_SURFACE, corner_radius=0) + c.grid(row=1, column=0, columnspan=2, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c, text="No authentication activity detected.", font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN, justify="center").pack(padx=10, pady=12) + else: + total_count = len(data) + start_idx = self.current_page * self.ITEMS_PER_PAGE + end_idx = start_idx + self.ITEMS_PER_PAGE + page_data = data[start_idx:end_idx] + + for r_idx, (method, activity) in enumerate(page_data, start=1): + bg_style = COLOR_SURFACE if r_idx % 2 != 0 else COLOR_SURFACE_VARIANT + vals = [method, activity] + for c_idx, val in enumerate(vals): + c = ctk.CTkFrame(metrics_grid, fg_color=bg_style, corner_radius=0) + c.grid(row=r_idx, column=c_idx, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c, text=val, font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN, justify="left").pack(padx=10, pady=8, anchor="nw") + + self._draw_pagination_controls(total_count, data, is_partial) + + def _draw_pagination_controls(self, total_count, data, is_partial): + total_pages = max(1, (total_count + self.ITEMS_PER_PAGE - 1) // self.ITEMS_PER_PAGE) + + control_frame = ctk.CTkFrame(self.body_frame, fg_color=COLOR_SURFACE) + control_frame.pack(fill="x", pady=0) + + center_container = ctk.CTkFrame(control_frame, fg_color="transparent") + center_container.pack(pady=(5, 10)) + + prev_state = "normal" if self.current_page > 0 else "disabled" + btn_prev = ctk.CTkButton( + center_container, text="◀ Prev", width=70, height=22, corner_radius=6, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, state=prev_state, + command=lambda: self._change_page(-1, data, is_partial) + ) + btn_prev.pack(side="left", padx=5) + + page_lbl = ctk.CTkLabel( + center_container, text=f"Page {self.current_page + 1} of {total_pages}", + font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_SUB + ) + page_lbl.pack(side="left", padx=15) + + next_state = "normal" if self.current_page < total_pages - 1 else "disabled" + btn_next = ctk.CTkButton( + center_container, text="Next ▶", width=70, height=22, corner_radius=6, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, state=next_state, + command=lambda: self._change_page(1, data, is_partial) + ) + btn_next.pack(side="left", padx=5) + + def _change_page(self, delta, data, is_partial): + self.current_page += delta + self._update_ui_paginated(data, is_partial) + + def cancel(self): + self.is_cancelled = True + self.current_request_id += 1 + if self.status == "loading": + self.status = "cancelled" + self._update_ui_paginated(self.last_data) + if hasattr(self, "btn_refresh") and self.btn_refresh.winfo_exists(): + self.btn_refresh.configure(state="normal") + + def _load_data_from_sqlite(self): + if not self.csv_path or not os.path.exists(self.csv_path): + tenant, clients, secrets = self.get_credentials() + if tenant and clients: + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + if os.path.basename(script_dir) == "entra": + script_dir = os.path.dirname(script_dir) + self.csv_path = os.path.join(script_dir, "reports", f"{tenant}_{clients[0]}", "entra_auth_methods.csv") + else: + return [] + + reports_dir = os.path.dirname(self.csv_path) + db_path = os.path.join(reports_dir, "telemetry_cache.db") + if not os.path.exists(db_path): + return [] + + items = [] + try: + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + cursor = conn.cursor() + cursor.execute("SELECT authenticationMethod, successActivityCount FROM entra_auth_methods") + for row in cursor.fetchall(): + items.append((row["authenticationMethod"], str(row["successActivityCount"]))) + conn.close() + except Exception as e: + usage_logger.error(f"Error reading SQLite for AuthMethodsSubFrame: {e}", exc_info=True) + return items + + @property + def last_data(self): + if hasattr(self, "_cached_auth_methods") and self._cached_auth_methods: + return self._cached_auth_methods + return self._load_data_from_sqlite() diff --git a/telemetry/entra/user_signins.py b/telemetry/entra/user_signins.py new file mode 100644 index 00000000..6c9983b6 --- /dev/null +++ b/telemetry/entra/user_signins.py @@ -0,0 +1,342 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""UI frame for Microsoft Entra User Sign-ins telemetry.""" + +import os +import csv +import logging +import threading +import customtkinter as ctk + +from core.graph.entra.user_signins import run_user_signins_pipeline +from core.graph.db import import_csv_to_sqlite +from telemetry.styles import * + +usage_logger = logging.getLogger("M365TelemetryAsyncLogger.UserSigninsUI") + +class UserSigninsSubFrame(ctk.CTkFrame): + """Sub-frame for Microsoft Entra User Sign Ins with unique set displays.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, semaphore=None, **kwargs): + super().__init__(master, fg_color="transparent", **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.semaphore = semaphore + self.status = None + self.is_cancelled = False + self.current_request_id = 0 + self.csv_path = None + + self.build_ui() + + def build_ui(self): + self.header_frame = ctk.CTkFrame(self, fg_color="transparent") + self.header_frame.pack(fill="x", pady=(0, 5)) + + ctk.CTkLabel(self.header_frame, text="User Sign-Ins", font=FONT_SUBSECTION_HEADER, text_color=COLOR_PRIMARY).pack(side="left") + + self.btn_refresh = ctk.CTkButton( + self.header_frame, state="disabled", text="↻ Reload", width=80, height=26, corner_radius=13, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, + command=self.trigger_fetch_individual + ) + self.btn_refresh.pack(side="right", padx=(10, 0)) + + self.body_frame = ctk.CTkFrame(self, fg_color="transparent") + self.body_frame.pack(fill="x") + + self.reset_view() + + def reset_view(self): + self.status = None + self.is_cancelled = False + self.csv_path = None + for w in self.body_frame.winfo_children(): + w.destroy() + + def _set_state_loading(self, msg="Loading..."): + for w in self.body_frame.winfo_children(): + w.destroy() + ctk.CTkLabel(self.body_frame, text=f"⏳ {msg}", text_color=COLOR_TEXT_SUB, font=FONT_BODY_MEDIUM).pack(pady=10) + + def _set_state_error(self, error_msg): + for w in self.body_frame.winfo_children(): + w.destroy() + display_msg = error_msg + if "401" in error_msg or "403" in error_msg or "unauthorized" in error_msg.lower() or "forbidden" in error_msg.lower() or "auditlog.read.all" in error_msg.lower(): + display_msg = "AuditLog.Read.All application permission required.\nPlease grant 'AuditLog.Read.All' to your App Registration in Microsoft Entra ID." + ctk.CTkLabel(self.body_frame, text=f"✖ {display_msg}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center").pack(pady=(10, 5)) + ctk.CTkButton(self.body_frame, text="Try Again", command=self.trigger_fetch_individual, width=120, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 10)) + + def trigger_fetch_individual(self): + tenant, clients, secrets = self.get_credentials() + if tenant: + self.trigger_fetch(tenant, clients[0], secrets[0]) + else: + self._set_state_error("Missing credentials. Please submit the credentials above.") + + def trigger_fetch(self, tenant, client_id, client_secret): + self.status = "loading" + self.is_cancelled = False + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + if os.path.basename(script_dir) == "entra": + script_dir = os.path.dirname(script_dir) + reports_dir = os.path.join(script_dir, "reports", f"{tenant}_{client_id}") + self.csv_path = os.path.join(reports_dir, "entra_user_signins.csv") + + self._set_state_loading("Downloading and parsing User Sign-Ins...") + self.on_status_change() + if hasattr(self, "btn_refresh") and self.btn_refresh.winfo_exists(): + self.btn_refresh.configure(state="disabled") + + threading.Thread( + target=self._execute_worker, + args=(tenant, client_id, client_secret, self.current_request_id), + daemon=True + ).start() + + def _execute_worker(self, tenant, client_id, client_secret, request_id): + if self.semaphore: + self.semaphore.acquire() + try: + if self.is_cancelled or request_id != self.current_request_id: + return + + unique_apps = set() + unique_os = set() + unique_browsers = set() + page_count = 0 + + def handle_page(filtered_list): + nonlocal unique_apps, unique_os, unique_browsers, page_count + if self.is_cancelled or request_id != self.current_request_id: + return + for item in filtered_list: + app_name = item.get("appDisplayName") or "" + device = item.get("deviceDetail") or {} + os_name = device.get("operatingSystem") or "" + browser_name = device.get("browser") or "" + + if app_name: unique_apps.add(app_name) + if os_name: unique_os.add(os_name) + if browser_name: unique_browsers.add(browser_name) + + page_count += 1 + if page_count % 3 == 0: + current_data = { + "apps": sorted(list(unique_apps)), + "os": sorted(list(unique_os)), + "browsers": sorted(list(unique_browsers)) + } + self.after(0, self._render_partial, current_data, request_id) + + temp_csv_path = self.csv_path + ".tmp" + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + if os.path.basename(script_dir) == "entra": + script_dir = os.path.dirname(script_dir) + reports_dir = os.path.join(script_dir, "reports", f"{tenant}_{client_id}") + os.makedirs(reports_dir, exist_ok=True) + + with open(temp_csv_path, 'w', encoding='utf-8', newline='') as f: + writer = csv.writer(f) + writer.writerow(["appDisplayName", "operatingSystem", "browser", "isInteractive"]) + + run_user_signins_pipeline( + client_id=client_id, + client_secret=client_secret, + tenant_id=tenant, + csv_path=temp_csv_path, + max_rows=20000, + on_page_callback=handle_page, + is_cancelled_callback=lambda: self.is_cancelled or request_id != self.current_request_id + ) + + # Compile final list + final_data = { + "apps": sorted(list(unique_apps)), + "os": sorted(list(unique_os)), + "browsers": sorted(list(unique_browsers)) + } + + if not self.is_cancelled and request_id == self.current_request_id: + if os.path.exists(temp_csv_path): + if os.path.exists(self.csv_path): + os.remove(self.csv_path) + os.rename(temp_csv_path, self.csv_path) + import asyncio + db_dir = os.path.dirname(self.csv_path) + db_path = os.path.join(db_dir, "telemetry_cache.db") + asyncio.run(import_csv_to_sqlite(self.csv_path, db_path, "user_signins", "appDisplayName")) + + if self.is_cancelled or request_id != self.current_request_id: + return + self.after(0, self._render_success, final_data, request_id) + except Exception as e: + usage_logger.error(f"Error fetching user sign-ins: {e}", exc_info=True) + if not self.is_cancelled and request_id == self.current_request_id: + self.after(0, self._render_error, str(e), request_id) + finally: + if 'temp_csv_path' in locals() and os.path.exists(temp_csv_path): + try: + os.remove(temp_csv_path) + except Exception: + pass + if self.semaphore: + self.semaphore.release() + + def _render_partial(self, data, request_id): + if self.is_cancelled or request_id != self.current_request_id: + return + self._update_ui(data, is_partial=True) + + def _render_success(self, data, request_id): + if self.is_cancelled or request_id != self.current_request_id: + return + self.status = "success" + self._update_ui(data=None, is_partial=False) + self.on_status_change() + if hasattr(self, "btn_refresh") and self.btn_refresh.winfo_exists(): + self.btn_refresh.configure(state="normal") + + def _render_error(self, err_msg, request_id): + if self.is_cancelled or request_id != self.current_request_id: + return + self.status = "error" + self._set_state_error(err_msg) + self.on_status_change() + if hasattr(self, "btn_refresh") and self.btn_refresh.winfo_exists(): + self.btn_refresh.configure(state="normal") + + def _load_data_from_csv(self): + if not self.csv_path or not os.path.exists(self.csv_path): + tenant, clients, secrets = self.get_credentials() + if tenant and clients: + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + if os.path.basename(script_dir) == "entra": + script_dir = os.path.dirname(script_dir) + self.csv_path = os.path.join(script_dir, "reports", f"{tenant}_{clients[0]}", "entra_user_signins.csv") + else: + return {"apps": [], "os": [], "browsers": []} + + db_dir = os.path.dirname(self.csv_path) + db_path = os.path.join(db_dir, "telemetry_cache.db") + if not os.path.exists(db_path): + return {"apps": [], "os": [], "browsers": []} + + unique_apps = set() + unique_os = set() + unique_browsers = set() + + import sqlite3 + try: + conn = sqlite3.connect(db_path) + cursor = conn.cursor() + + cursor.execute("SELECT DISTINCT appDisplayName FROM user_signins WHERE appDisplayName IS NOT NULL AND appDisplayName != ''") + for r in cursor.fetchall(): + unique_apps.add(r[0]) + + cursor.execute("SELECT DISTINCT operatingSystem FROM user_signins WHERE operatingSystem IS NOT NULL AND operatingSystem != ''") + for r in cursor.fetchall(): + unique_os.add(r[0]) + + cursor.execute("SELECT DISTINCT browser FROM user_signins WHERE browser IS NOT NULL AND browser != ''") + for r in cursor.fetchall(): + unique_browsers.add(r[0]) + + conn.close() + except Exception as e: + usage_logger.error(f"Error reading SQLite cache for User Sign-ins: {e}", exc_info=True) + + return { + "apps": sorted(list(unique_apps)), + "os": sorted(list(unique_os)), + "browsers": sorted(list(unique_browsers)) + } + + def _update_ui(self, data=None, is_partial=False): + for w in self.body_frame.winfo_children(): + w.destroy() + + if is_partial: + progress_frame = ctk.CTkFrame(self.body_frame, fg_color=COLOR_TONAL_BG, height=26, corner_radius=6) + progress_frame.pack(fill="x", pady=(0, 6)) + ctk.CTkLabel( + progress_frame, + text="⏳ Querying User Sign-Ins in the background... UI will auto-refresh.", + font=FONT_BODY_SMALL, + text_color=COLOR_TONAL_TEXT + ).pack(padx=10, pady=2, anchor="w") + + if data is None: + data = self._load_data_from_csv() + + metrics_grid = ctk.CTkFrame(self.body_frame, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + metrics_grid.pack(fill="x", pady=(5, 10)) + + headers = ["Sign-in Attribute", "Successful Unique Values"] + for i in range(2): + metrics_grid.grid_columnconfigure(i, weight=1 if i == 0 else 3) + + for col_idx, head_text in enumerate(headers): + cell = ctk.CTkFrame(metrics_grid, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + apps_str = ", ".join(data.get("apps", [])) or "None" + os_str = ", ".join(data.get("os", [])) or "None" + browsers_str = ", ".join(data.get("browsers", [])) or "None" + + rows = [ + ("Successful App Sign-ins", apps_str), + ("Successful Client OS", os_str), + ("Successful Browsers", browsers_str) + ] + + for r_idx, (attribute, val_str) in enumerate(rows, start=1): + bg_style = COLOR_SURFACE if r_idx % 2 != 0 else COLOR_SURFACE_VARIANT + + c0 = ctk.CTkFrame(metrics_grid, fg_color=bg_style, corner_radius=0) + c0.grid(row=r_idx, column=0, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c0, text=attribute, font=FONT_BODY_BOLD, text_color=COLOR_TEXT_MAIN, justify="left").pack(padx=10, pady=8, anchor="nw") + + c1 = ctk.CTkFrame(metrics_grid, fg_color=bg_style, corner_radius=0) + c1.grid(row=r_idx, column=1, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c1, text=val_str, font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN, justify="left", wraplength=650).pack(padx=10, pady=8, anchor="nw") + + # Footnote disclaimer + ctk.CTkLabel( + self.body_frame, + text="* Based on sample data collected from signins.", + font=FONT_BODY_SMALL, + text_color=COLOR_TEXT_SUB, + justify="left" + ).pack(anchor="w", padx=10, pady=(5, 10)) + + @property + def last_data(self): + return self._load_data_from_csv() + + def cancel(self): + self.is_cancelled = True + self.current_request_id += 1 + if self.status == "loading": + self.status = "cancelled" + self._update_ui(data=None, is_partial=False) + if hasattr(self, "btn_refresh") and self.btn_refresh.winfo_exists(): + self.btn_refresh.configure(state="normal") diff --git a/telemetry/exchange/__init__.py b/telemetry/exchange/__init__.py new file mode 100644 index 00000000..d68482ec --- /dev/null +++ b/telemetry/exchange/__init__.py @@ -0,0 +1,238 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Consolidated Exchange Online Telemetry Orchestrator Container.""" + +import logging +import customtkinter as ctk + +from telemetry.exchange.mailbox import MailboxUsageFrame +from telemetry.exchange.calendar import CalendarTelemetryFrame +from telemetry.exchange.integrated_apps import ExchangeAppsFrame +from telemetry.exchange.mail_security import MailSecurityFrame +from telemetry.exchange.transport_rules import TransportRulesFrame +from telemetry.exchange.connectors import ExchangeConnectorsFrame +from telemetry.exchange.email_clients import EmailClientSupportFrame +from telemetry.exchange.pst_files import PstFilesFrame +from telemetry.styles import * + +usage_logger = logging.getLogger("M365TelemetryAsyncLogger.ExchangeUI") + +class ExchangeOnlineFrame(ctk.CTkFrame): + """Uber section container hosting Mailbox, Calendar, Apps, Mail Security, Rules, Connectors, Email Clients, and PST files UI frames vertically stacked.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, **kwargs): + self.semaphore = kwargs.pop("concurrency_semaphore", None) + super().__init__(master, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=12, **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.status = None + + self.build_ui() + + def build_ui(self): + """Instantiates and stacks the 8 decoupled Exchange telemetry sub-frames.""" + self.pack(fill="x", expand=True, pady=10) + + self.inner_pad = ctk.CTkFrame(self, fg_color="transparent") + self.inner_pad.pack(fill="both", expand=True, padx=20, pady=20) + + # Main Header Title + ctk.CTkLabel( + self.inner_pad, + text="Email, Calendar and Contacts", + font=FONT_HEADER_SMALL, + text_color=COLOR_TEXT_MAIN + ).pack(anchor="w", pady=(0, 5)) + + # 1. Mailbox Usage Sub-frame + self.mailbox_view = MailboxUsageFrame( + master=self.inner_pad, + log_callback=self.log_msg, + credentials_callback=self.get_credentials, + status_change_callback=self._subframe_status_changed, + concurrency_semaphore=self.semaphore + ) + self.mailbox_view.configure(fg_color="transparent", border_width=0) + self.mailbox_view.pack(fill="x", expand=True, pady=(0, 5)) + + # Divider 1 + self.divider1 = ctk.CTkFrame(self.inner_pad, height=1, fg_color=COLOR_OUTLINE_LIGHT) + self.divider1.pack(fill="x", pady=10) + + # 2. Calendar Telemetry Sub-frame + self.calendar_view = CalendarTelemetryFrame( + master=self.inner_pad, + log_callback=self.log_msg, + credentials_callback=self.get_credentials, + status_change_callback=self._subframe_status_changed, + concurrency_semaphore=self.semaphore + ) + self.calendar_view.configure(fg_color="transparent", border_width=0) + self.calendar_view.pack(fill="x", expand=True, pady=(0, 5)) + + # Divider 2 + self.divider2 = ctk.CTkFrame(self.inner_pad, height=1, fg_color=COLOR_OUTLINE_LIGHT) + self.divider2.pack(fill="x", pady=10) + + # 3. Organization-wide Apps Sub-frame + self.apps_view = ExchangeAppsFrame( + master=self.inner_pad, + log_callback=self.log_msg, + credentials_callback=self.get_credentials, + status_change_callback=self._subframe_status_changed, + concurrency_semaphore=self.semaphore + ) + self.apps_view.configure(fg_color="transparent", border_width=0) + self.apps_view.pack(fill="x", expand=True, pady=(0, 5)) + + # Divider 3 + self.divider3 = ctk.CTkFrame(self.inner_pad, height=1, fg_color=COLOR_OUTLINE_LIGHT) + self.divider3.pack(fill="x", pady=10) + + # 4. Mail Security Sub-frame + self.mail_security_view = MailSecurityFrame( + master=self.inner_pad, + log_callback=self.log_msg, + credentials_callback=self.get_credentials, + status_change_callback=self._subframe_status_changed, + concurrency_semaphore=self.semaphore + ) + self.mail_security_view.configure(fg_color="transparent", border_width=0) + self.mail_security_view.pack(fill="x", expand=True, pady=(0, 5)) + + # Divider 4 + self.divider4 = ctk.CTkFrame(self.inner_pad, height=1, fg_color=COLOR_OUTLINE_LIGHT) + self.divider4.pack(fill="x", pady=10) + + # 5. Transport Rules Sub-frame + self.transport_rules_view = TransportRulesFrame( + master=self.inner_pad, + log_callback=self.log_msg, + credentials_callback=self.get_credentials, + status_change_callback=self._subframe_status_changed, + concurrency_semaphore=self.semaphore + ) + self.transport_rules_view.configure(fg_color="transparent", border_width=0) + self.transport_rules_view.pack(fill="x", expand=True, pady=(0, 5)) + + # Divider 5 + self.divider5 = ctk.CTkFrame(self.inner_pad, height=1, fg_color=COLOR_OUTLINE_LIGHT) + self.divider5.pack(fill="x", pady=10) + + # 6. Exchange Connectors Sub-frame + self.connectors_view = ExchangeConnectorsFrame( + master=self.inner_pad, + log_callback=self.log_msg, + credentials_callback=self.get_credentials, + status_change_callback=self._subframe_status_changed, + concurrency_semaphore=self.semaphore + ) + self.connectors_view.configure(fg_color="transparent", border_width=0) + self.connectors_view.pack(fill="x", expand=True, pady=(0, 5)) + + # Divider 6 + self.divider6 = ctk.CTkFrame(self.inner_pad, height=1, fg_color=COLOR_OUTLINE_LIGHT) + self.divider6.pack(fill="x", pady=10) + + # 7. Email Client Support Sub-frame + self.email_clients_view = EmailClientSupportFrame( + master=self.inner_pad, + log_callback=self.log_msg, + credentials_callback=self.get_credentials, + status_change_callback=self._subframe_status_changed, + concurrency_semaphore=self.semaphore + ) + self.email_clients_view.configure(fg_color="transparent", border_width=0) + self.email_clients_view.pack(fill="x", expand=True, pady=(0, 5)) + + # Divider 7 + self.divider7 = ctk.CTkFrame(self.inner_pad, height=1, fg_color=COLOR_OUTLINE_LIGHT) + self.divider7.pack(fill="x", pady=10) + + # 8. PST Files Discovery Sub-frame + self.pst_files_view = PstFilesFrame( + master=self.inner_pad, + log_callback=self.log_msg, + credentials_callback=self.get_credentials, + status_change_callback=self._subframe_status_changed, + concurrency_semaphore=self.semaphore + ) + self.pst_files_view.configure(fg_color="transparent", border_width=0) + self.pst_files_view.pack(fill="x", expand=True, pady=(0, 5)) + + self.reset_view() + + def _subframe_status_changed(self): + statuses = [ + self.mailbox_view.status, + self.calendar_view.status, + self.apps_view.status, + self.mail_security_view.status, + self.transport_rules_view.status, + self.connectors_view.status, + self.email_clients_view.status, + self.pst_files_view.status + ] + if "loading" in statuses: + self.status = "loading" + elif "error" in statuses: + self.status = "error" + elif "success" in statuses: + self.status = "success" + else: + self.status = None + self.on_status_change() + + def reset_view(self): + """Resets all sub-views and hides container.""" + self.pack_forget() + self.status = None + self.mailbox_view.reset_view() + self.calendar_view.reset_view() + self.apps_view.reset_view() + self.mail_security_view.reset_view() + self.transport_rules_view.reset_view() + self.connectors_view.reset_view() + self.email_clients_view.reset_view() + self.pst_files_view.reset_view() + + def trigger_fetch(self, tenant, client_id, client_secret): + """Displays container and delegates fetches to all sub-views.""" + self.status = "loading" + self.on_status_change() + + self.pack(fill="x", expand=True, pady=10) + + self.mailbox_view.trigger_fetch(tenant, client_id, client_secret) + self.calendar_view.trigger_fetch(tenant, client_id, client_secret) + self.apps_view.trigger_fetch(tenant, client_id, client_secret) + self.mail_security_view.trigger_fetch(tenant, client_id, client_secret) + self.transport_rules_view.trigger_fetch(tenant, client_id, client_secret) + self.connectors_view.trigger_fetch(tenant, client_id, client_secret) + self.email_clients_view.trigger_fetch(tenant, client_id, client_secret) + self.pst_files_view.trigger_fetch(tenant, client_id, client_secret) + + def cancel(self): + """Cancels all child views in this container.""" + self.mailbox_view.cancel() + self.calendar_view.cancel() + self.apps_view.cancel() + self.mail_security_view.cancel() + self.transport_rules_view.cancel() + self.connectors_view.cancel() + self.email_clients_view.cancel() + self.pst_files_view.cancel() + self.status = None diff --git a/telemetry/exchange/calendar.py b/telemetry/exchange/calendar.py new file mode 100644 index 00000000..4fbe04aa --- /dev/null +++ b/telemetry/exchange/calendar.py @@ -0,0 +1,245 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""UI frame for Exchange Online Calendar Environment Telemetry.""" + +import os +import csv +import logging +import threading +import customtkinter as ctk + +from core.graph.exchange.calendar import run_calendar_telemetry_pipeline +from telemetry.styles import * + +calendar_logger = logging.getLogger("M365TelemetryAsyncLogger.CalendarTelemetryUI") + +class CalendarTelemetryFrame(ctk.CTkFrame): + """Self-contained customtkinter component wrapping Exchange Online Calendar Environment Telemetry UI.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, **kwargs): + self.semaphore = kwargs.pop("concurrency_semaphore", None) + super().__init__(master, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=12, **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.status = None + self.last_data = {} + + self.build_ui() + + def build_ui(self): + """Creates card container for the tab.""" + self.pack(fill="x", expand=True, pady=10) + + self.inner_pad = ctk.CTkFrame(self, fg_color="transparent") + self.inner_pad.pack(fill="both", expand=True, padx=20, pady=20) + + self.header = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.header.pack(fill="x", pady=(0, 10)) + ctk.CTkLabel(self.header, text="Exchange Online Calendar Environment", font=FONT_HEADER_SMALL, text_color=COLOR_TEXT_MAIN).pack(side="left") + self.reload_btn = ctk.CTkButton( + self.header, + state="disabled", text="↻ Reload", + width=80, + height=24, + font=ctk.CTkFont(family="Segoe UI", size=12), + fg_color="transparent", + border_width=1, + text_color="#2563EB", + hover_color="#DBEAFE", + command=self._retry_fetch + ) + self.reload_btn.pack(side="right") + + self.state_frame = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.warning_label = ctk.CTkLabel(self.inner_pad, text="", font=FONT_BODY_MEDIUM, text_color=COLOR_ERROR, justify="left", anchor="w", wraplength=750) + self.grid_frame = ctk.CTkFrame(self.inner_pad, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + + self.reset_view() + + def reset_view(self): + """Resets and hides grids.""" + self.state_frame.pack_forget() + self.grid_frame.pack_forget() + self.last_data = {} + if hasattr(self, "warning_label"): + self.warning_label.pack_forget() + + for w in self.state_frame.winfo_children(): + w.destroy() + for w in self.grid_frame.winfo_children(): + w.destroy() + + def _set_state_loading(self, msg="Loading..."): + for w in self.state_frame.winfo_children(): + w.destroy() + ctk.CTkLabel(self.state_frame, text=f"⏳ {msg}", text_color=COLOR_TEXT_SUB, font=FONT_BODY_MEDIUM).pack(pady=(20, 5)) + self.progress = ctk.CTkProgressBar(self.state_frame, mode="indeterminate", width=250, fg_color="#F3F4F6", progress_color="#2563EB") + self.progress.pack(pady=(0, 20)) + self.progress.start() + self.state_frame.pack(fill="x", expand=True) + + def _set_state_error(self, error_msg): + for w in self.state_frame.winfo_children(): + w.destroy() + + ctk.CTkLabel(self.state_frame, text=f"✖ {error_msg}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center").pack(pady=(20, 10)) + ctk.CTkButton(self.state_frame, text="Try Again", command=self._retry_fetch, width=120, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 20)) + self.state_frame.pack(fill="x", expand=True) + + def _retry_fetch(self): + if hasattr(self, 'reload_btn') and self.reload_btn.winfo_exists(): + self.reload_btn.configure(state="disabled") + tenant, clients, secrets = self.get_credentials() + if tenant: + self.trigger_fetch(tenant, clients[0], secrets[0]) + + def trigger_fetch(self, tenant, client_id, client_secret): + """Triggers background fetch thread.""" + calendar_logger.info("Calendar Telemetry trigger_fetch called. Spawning background thread...") + self.status = "loading" + self.on_status_change() + + self.grid_frame.pack_forget() + self.warning_label.pack_forget() + + self._set_state_loading("Downloading and auditing Exchange Calendar configurations...") + + threading.Thread( + target=self._execute_worker, + args=(tenant, client_id, client_secret), + daemon=True + ).start() + + def _execute_worker(self, tenant: str, client_id: str, client_secret: str): + if self.semaphore: + self.semaphore.acquire() + try: + data = run_calendar_telemetry_pipeline(client_id, client_secret, tenant) + + # Stream to CSV in reports_dir + if not data.get("powershell_error"): + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + reports_dir = os.path.join(script_dir, "reports", f"{tenant}_{client_id}") + os.makedirs(reports_dir, exist_ok=True) + csv_path = os.path.join(reports_dir, "calendar_configurations.csv") + + with open(csv_path, 'w', encoding='utf-8', newline='') as f: + writer = csv.writer(f) + writer.writerow(["Configuration", "Value"]) + for k, v in data.items(): + if k != "RoomsList": + writer.writerow([k, str(v)]) + + rooms_list = data.get("RoomsList", []) + rooms_csv = os.path.join(reports_dir, "room_mailboxes.csv") + with open(rooms_csv, 'w', encoding='utf-8', newline='') as f: + writer = csv.writer(f) + writer.writerow(["PrimarySmtpAddress"]) + for room in rooms_list: + writer.writerow([room]) + + calendar_logger.info(f"Successfully streamed Calendar data to {csv_path} and room list to {rooms_csv}") + + calendar_logger.info("Successfully completed Calendar telemetry data fetch.") + self.after(0, self._render_success, data) + except Exception as e: + calendar_logger.error("Exception caught in Calendar worker.", exc_info=True) + self.after(0, self._render_error, str(e)) + finally: + if self.semaphore: + self.semaphore.release() + + def _render_success(self, data: dict): + if hasattr(self, 'reload_btn') and self.reload_btn.winfo_exists(): + self.reload_btn.configure(state="normal") + self.last_data = data + calendar_logger.info("Calendar data successfully retrieved. Rendering UI grid.") + self.state_frame.pack_forget() + for w in self.grid_frame.winfo_children(): + w.destroy() + + if data.get("powershell_error"): + friendly_msg = f"Exchange PowerShell query failed: {data['powershell_error']}" + self.warning_label.configure(text=f"⚠️ Warning: {friendly_msg}") + self.warning_label.pack(anchor="w", pady=(0, 10)) + else: + self.warning_label.pack_forget() + + self.grid_frame.pack(fill="x", expand=True) + + self.grid_frame.grid_columnconfigure(0, weight=3) + self.grid_frame.grid_columnconfigure(1, weight=2) + + headers_sp = ["Calendar Configuration / Metric", "Value / Configuration"] + for col_idx, head_text in enumerate(headers_sp): + cell = ctk.CTkFrame(self.grid_frame, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=1, pady=1) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + rooms_err = data.get("RoomsError") + devs_err = data.get("DevicesError") + rooms_count = data.get("RoomsCount", 0) + equip_count = data.get("EquipmentCount", 0) + + if rooms_err and devs_err: + res_val = rooms_err + else: + r_str = "Error" if rooms_err else str(rooms_count) + e_str = "Error" if devs_err else str(equip_count) + tot = "Error" if (rooms_err or devs_err) else str(rooms_count + equip_count) + res_val = f"Total: {tot} ({r_str} Rooms, {e_str} Equipment)" + + reserve_val = data.get("CanUsersReserveRooms") + if isinstance(reserve_val, bool): + reserve_val = "Yes" if reserve_val else "No" + + att_val = data.get("CanShareAttachments") + if isinstance(att_val, bool): + attachments_val = "Yes" if att_val else "No" + else: + attachments_val = att_val + + rows_data = [ + ("Room & Resource Reservation", reserve_val), + ("Calendar Resources", res_val), + ("Resource Naming Convention", data.get("NamingConvention") or "None found"), + ("Calendar Attachments Enabled", attachments_val), + ] + + for r_idx, (metric_name, val) in enumerate(rows_data, start=1): + bg_style = "transparent" if r_idx % 2 != 0 else COLOR_SURFACE_VARIANT + + c0 = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + c0.grid(row=r_idx, column=0, sticky="nsew", padx=1, pady=1) + ctk.CTkLabel(c0, text=metric_name, font=FONT_BODY_BOLD, text_color=COLOR_TEXT_MAIN, wraplength=450, justify="left", anchor="w").pack(padx=10, pady=6, anchor="w") + + c1 = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + c1.grid(row=r_idx, column=1, sticky="nsew", padx=1, pady=1) + ctk.CTkLabel(c1, text=str(val), font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN, wraplength=300, justify="left", anchor="w").pack(padx=10, pady=6, anchor="w") + + self.status = "success" + self.on_status_change() + + def _render_error(self, err_msg): + calendar_logger.warning(f"Calendar Telemetry fetch failed: {err_msg}") + if hasattr(self, 'reload_btn') and self.reload_btn.winfo_exists(): + self.reload_btn.configure(state="normal") + self._set_state_error(err_msg) + self.status = "error" + self.on_status_change() + + def cancel(self): + pass diff --git a/telemetry/exchange/connectors.py b/telemetry/exchange/connectors.py new file mode 100644 index 00000000..64fe2f39 --- /dev/null +++ b/telemetry/exchange/connectors.py @@ -0,0 +1,385 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""UI frame for Exchange Online Inbound and Outbound Connectors.""" + +import os +import csv +import logging +import threading +import webbrowser +import asyncio +import sqlite3 +import customtkinter as ctk + +from core.graph.exchange.connectors import fetch_exchange_connectors_data +from core.graph.db import import_csv_to_sqlite, query_page_sync +from telemetry.styles import * + +usage_logger = logging.getLogger("M365TelemetryAsyncLogger.ExchangeConnectorsUI") + +class ExchangeConnectorsFrame(ctk.CTkFrame): + """Self-contained component for rendering Exchange Online Connectors routing logic.""" + + def update_loading_text(self, text_msg): + if hasattr(self, 'loading_label') and self.loading_label.winfo_exists(): + self.loading_label.configure(text=f"⏳ {text_msg}") + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, **kwargs): + self.semaphore = kwargs.pop("concurrency_semaphore", None) + super().__init__(master, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=12, **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.status = None + self.current_page = 0 + self.ITEMS_PER_PAGE = 5 + self._cached_connectors_data = [] + + self.build_ui() + + def build_ui(self): + """Creates card container for the tab.""" + self.pack(fill="x", expand=True, pady=10) + + self.inner_pad = ctk.CTkFrame(self, fg_color="transparent") + self.inner_pad.pack(fill="both", expand=True, padx=20, pady=20) + + self.header_frame = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.header_frame.pack(fill="x", pady=(0, 10)) + + self.title_lbl = ctk.CTkLabel( + self.header_frame, + text="Exchange Connectors (Inbound & Outbound Routing)", + font=FONT_HEADER_SMALL, + text_color=COLOR_TEXT_MAIN + ) + self.title_lbl.pack(side="left", anchor="w") + + self.link_lbl = ctk.CTkLabel( + self.header_frame, + text="Open Exchange Admin Center ↗", + font=FONT_BODY_BOLD, + text_color=COLOR_PRIMARY, + cursor="hand2" + ) + self.link_lbl.pack(side="left", anchor="w", padx=(15, 0)) + self.link_lbl.bind("", lambda e: webbrowser.open("https://admin.cloud.microsoft/exchange?#/connectors")) + self.link_lbl.bind("", lambda e: self.link_lbl.configure(text_color=COLOR_PRIMARY_HOVER)) + self.link_lbl.bind("", lambda e: self.link_lbl.configure(text_color=COLOR_PRIMARY)) + + self.reload_btn = ctk.CTkButton( + self.header_frame, + state="disabled", text="↻ Reload", + width=80, + height=24, + font=ctk.CTkFont(family="Segoe UI", size=12), + fg_color="transparent", + border_width=1, + text_color="#2563EB", + hover_color="#DBEAFE", + command=self._retry_fetch + ) + self.reload_btn.pack(side="right") + + self.state_frame = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.grid_frame = ctk.CTkFrame(self.inner_pad, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + + self.reset_view() + + def reset_view(self): + self.state_frame.pack_forget() + self.grid_frame.pack_forget() + self._cached_connectors_data = [] + + for w in self.state_frame.winfo_children(): + w.destroy() + for w in self.grid_frame.winfo_children(): + w.destroy() + + def _set_state_loading(self, msg="Loading..."): + for w in self.state_frame.winfo_children(): + w.destroy() + self.loading_label = ctk.CTkLabel(self.state_frame, text=f"⏳ {msg}", text_color="#6b7280", font=ctk.CTkFont(family="Segoe UI", size=13)) + self.loading_label.pack(pady=(20, 5)) + pb = ctk.CTkProgressBar(self.state_frame, mode="indeterminate", width=250, fg_color="#F3F4F6", progress_color="#2563EB") + pb.pack(pady=(0, 20)) + pb.start() + self.state_frame.pack(fill="x", expand=True) + + def _set_state_error(self, error_msg): + for w in self.state_frame.winfo_children(): + w.destroy() + + display_msg = error_msg + if "is not installed or not in PATH" in error_msg.lower() or "pwsh" in error_msg.lower(): + display_msg = "PowerShell Core ('pwsh') is not installed or configured on this machine." + elif "exchangeonlinemanagement" in error_msg.lower(): + display_msg = "ExchangeOnlineManagement PowerShell module is missing." + + ctk.CTkLabel(self.state_frame, text=f"✖ {display_msg}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center").pack(pady=(20, 5)) + ctk.CTkButton(self.state_frame, text="Try Again", command=self._retry_fetch, width=120, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 20)) + self.state_frame.pack(fill="x", expand=True) + + def _retry_fetch(self): + if hasattr(self, 'reload_btn') and self.reload_btn.winfo_exists(): + self.reload_btn.configure(state="disabled") + tenant, clients, secrets = self.get_credentials() + if tenant: + self.trigger_fetch(tenant, clients[0], secrets[0]) + + def trigger_fetch(self, tenant, client_id, client_secret): + usage_logger.info("Exchange Connectors trigger_fetch called. Spawning background thread...") + self.status = "loading" + self.on_status_change() + + self.grid_frame.pack_forget() + + self._set_state_loading("Retrieving Exchange Connectors routing configurations...") + + threading.Thread( + target=self._execute_worker, + args=(tenant, client_id, client_secret), + daemon=True + ).start() + + def _execute_worker(self, tenant: str, client_id: str, client_secret: str): + if self.semaphore: + self.semaphore.acquire() + try: + res = fetch_exchange_connectors_data(client_id, client_secret, tenant) + + connectors_data = res.get("connectors") + if connectors_data and not connectors_data.get("Errors"): + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + if os.path.basename(script_dir) == "exchange": + script_dir = os.path.dirname(script_dir) + reports_dir = os.path.join(script_dir, "reports", f"{tenant}_{client_id}") + os.makedirs(reports_dir, exist_ok=True) + + inbound_path = os.path.join(reports_dir, "exchange_inbound_connectors.csv") + outbound_path = os.path.join(reports_dir, "exchange_outbound_connectors.csv") + + inbound = connectors_data.get("InboundConnectors", []) + outbound = connectors_data.get("OutboundConnectors", []) + + with open(inbound_path, 'w', encoding='utf-8', newline='') as f: + writer = csv.writer(f) + writer.writerow(["Name", "Enabled", "SenderDomains", "ConnectorType", "RequireTls"]) + for c in inbound: + writer.writerow([c.get("Name"), c.get("Enabled"), c.get("SenderDomains"), c.get("ConnectorType"), c.get("RequireTls")]) + + with open(outbound_path, 'w', encoding='utf-8', newline='') as f: + writer = csv.writer(f) + writer.writerow(["Name", "Enabled", "RecipientDomains", "SmartHosts", "UseMxRecord"]) + for c in outbound: + writer.writerow([c.get("Name"), c.get("Enabled"), c.get("RecipientDomains"), c.get("SmartHosts"), c.get("UseMxRecord")]) + + db_path = os.path.join(reports_dir, "telemetry_cache.db") + asyncio.run(import_csv_to_sqlite(inbound_path, db_path, "inbound_connectors")) + asyncio.run(import_csv_to_sqlite(outbound_path, db_path, "outbound_connectors")) + + usage_logger.info("Successfully streamed inbound and outbound connectors to CSV.") + + self.after(0, self._handle_result, res) + finally: + if self.semaphore: + self.semaphore.release() + + def _handle_result(self, result: dict): + if hasattr(self, 'reload_btn') and self.reload_btn.winfo_exists(): + self.reload_btn.configure(state="normal") + self.state_frame.pack_forget() + for w in self.grid_frame.winfo_children(): + w.destroy() + + connectors_data = result.get("connectors") + err = result.get("error") + + if err: + self.status = "error" + self._set_state_error(err) + else: + ps_errs = connectors_data.get("Errors", {}) if connectors_data else {} + if ps_errs: + err_msg = "\n".join([f"{k}: {v}" for k,v in ps_errs.items()]) + self.status = "error" + self._set_state_error(f"PowerShell Execution Error:\n{err_msg}") + else: + self.status = "success" + self.grid_frame.pack(fill="x") + + unified_data = [] + inbound = connectors_data.get("InboundConnectors", []) + outbound = connectors_data.get("OutboundConnectors", []) + + for conn in inbound: + unified_data.append({ + "Direction": "📥 Inbound", + "Name": conn.get("Name", "N/A"), + "Status": "🟢 Enabled" if conn.get("Enabled") else "🔴 Disabled", + "Domains": conn.get("SenderDomains", "N/A") or "N/A", + "Routing": f"Type: {conn.get('ConnectorType', 'N/A')}\nRequire TLS: {'Yes' if conn.get('RequireTls') else 'No'}" + }) + + for conn in outbound: + unified_data.append({ + "Direction": "📤 Outbound", + "Name": conn.get("Name", "N/A"), + "Status": "🟢 Enabled" if conn.get("Enabled") else "🔴 Disabled", + "Domains": conn.get("RecipientDomains", "N/A") or "N/A", + "Routing": f"SmartHosts: {conn.get('SmartHosts', 'N/A')}\nUse MX: {'Yes' if conn.get('UseMxRecord') else 'No'}" + }) + + self._cached_connectors_data = unified_data + self.current_page = 0 + self._update_ui_paginated(self.last_data) + + self.on_status_change() + + @property + def last_data(self): + if hasattr(self, "_cached_connectors_data") and self._cached_connectors_data: + return self._cached_connectors_data + + tenant, clients, secrets = self.get_credentials() + if not tenant or not clients: + return [] + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + if os.path.basename(script_dir) == "exchange": + script_dir = os.path.dirname(script_dir) + reports_dir = os.path.join(script_dir, "reports", f"{tenant}_{clients[0]}") + db_path = os.path.join(reports_dir, "telemetry_cache.db") + + unified_data = [] + if os.path.exists(db_path): + try: + conn = sqlite3.connect(db_path) + cursor = conn.cursor() + + # Check if inbound_connectors table exists + cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='inbound_connectors'") + if cursor.fetchone(): + cursor.execute("SELECT * FROM inbound_connectors") + for row in cursor.fetchall(): + if len(row) >= 5: + enabled = str(row[1]) + unified_data.append({ + "Direction": "📥 Inbound", + "Name": row[0] or "N/A", + "Status": "🟢 Enabled" if enabled in ("True", "1") else "🔴 Disabled", + "Domains": row[2] or "N/A", + "Routing": f"Type: {row[3]}\nRequire TLS: {'Yes' if str(row[4]) in ('True', '1') else 'No'}" + }) + + # Check if outbound_connectors table exists + cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='outbound_connectors'") + if cursor.fetchone(): + cursor.execute("SELECT * FROM outbound_connectors") + for row in cursor.fetchall(): + if len(row) >= 5: + enabled = str(row[1]) + unified_data.append({ + "Direction": "📤 Outbound", + "Name": row[0] or "N/A", + "Status": "🟢 Enabled" if enabled in ("True", "1") else "🔴 Disabled", + "Domains": row[2] or "N/A", + "Routing": f"SmartHosts: {row[3]}\nUse MX: {'Yes' if str(row[4]) in ('True', '1') else 'No'}" + }) + + conn.close() + except Exception as e: + usage_logger.error(f"Error loading connectors from DB: {e}") + + return unified_data + + def _update_ui_paginated(self, data): + for w in self.grid_frame.winfo_children(): + w.destroy() + + if not data: + self.grid_frame.configure(fg_color=COLOR_SURFACE, border_width=1, border_color=COLOR_OUTLINE_LIGHT, corner_radius=8) + ctk.CTkLabel(self.grid_frame, text="N/A (No Exchange Connectors configured)", font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_SUB).pack(padx=20, pady=20, anchor="w") + return + + headers_in = ["Direction", "Connector Name", "Status", "Domains", "Routing Config"] + for i in range(5): + self.grid_frame.grid_columnconfigure(i, weight=1) + + for col_idx, head_text in enumerate(headers_in): + cell = ctk.CTkFrame(self.grid_frame, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + total_count = len(data) + start_idx = self.current_page * self.ITEMS_PER_PAGE + end_idx = start_idx + self.ITEMS_PER_PAGE + page_data = data[start_idx:end_idx] + + for r_idx, conn in enumerate(page_data, start=1): + bg_style = COLOR_SURFACE if r_idx % 2 != 0 else COLOR_SURFACE_VARIANT + vals = [ + conn["Direction"], + conn["Name"], + conn["Status"], + conn["Domains"], + conn["Routing"] + ] + for c_idx, val in enumerate(vals): + c = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + c.grid(row=r_idx, column=c_idx, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c, text=val, font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN, justify="left", wraplength=180).pack(padx=10, pady=12, anchor="nw") + + self._draw_pagination_controls(total_count, data) + + def _draw_pagination_controls(self, total_count, data): + total_pages = max(1, (total_count + self.ITEMS_PER_PAGE - 1) // self.ITEMS_PER_PAGE) + + control_frame = ctk.CTkFrame(self.grid_frame, fg_color=COLOR_SURFACE) + control_frame.grid(row=self.ITEMS_PER_PAGE + 2, column=0, columnspan=5, pady=0, sticky="ew") + + center_container = ctk.CTkFrame(control_frame, fg_color="transparent") + center_container.pack(pady=(5, 10)) + + prev_state = "normal" if self.current_page > 0 else "disabled" + btn_prev = ctk.CTkButton( + center_container, text="◀ Prev", width=70, height=22, corner_radius=6, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, state=prev_state, + command=lambda: self._change_page(-1, data) + ) + btn_prev.pack(side="left", padx=5) + + page_lbl = ctk.CTkLabel( + center_container, text=f"Page {self.current_page + 1} of {total_pages}", + font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_SUB + ) + page_lbl.pack(side="left", padx=15) + + next_state = "normal" if self.current_page < total_pages - 1 else "disabled" + btn_next = ctk.CTkButton( + center_container, text="Next ▶", width=70, height=22, corner_radius=6, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, state=next_state, + command=lambda: self._change_page(1, data) + ) + btn_next.pack(side="left", padx=5) + + def _change_page(self, delta, data): + self.current_page += delta + self._update_ui_paginated(data) + + def cancel(self): + pass diff --git a/telemetry/exchange/email_clients.py b/telemetry/exchange/email_clients.py new file mode 100644 index 00000000..9bd14136 --- /dev/null +++ b/telemetry/exchange/email_clients.py @@ -0,0 +1,253 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""UI frame for Exchange Online Supported Email Clients telemetry.""" + +import os +import csv +import logging +import threading +import asyncio +import sqlite3 +import customtkinter as ctk + +from core.graph.exchange.email_clients import run_email_client_usage_pipeline +from core.graph.db import import_csv_to_sqlite, query_page_sync +from telemetry.styles import * + +usage_logger = logging.getLogger("M365TelemetryAsyncLogger.EmailClientSupportUI") + +class EmailClientSupportFrame(ctk.CTkFrame): + """Self-contained component wrapping Exchange Online Supported Email Clients UI.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, **kwargs): + self.semaphore = kwargs.pop("concurrency_semaphore", None) + super().__init__(master, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=12, **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + + self.status = None + self._cached_client_data = {} + + self.build_ui() + + def build_ui(self): + self.pack(fill="x", expand=True, pady=10) + + self.inner_pad = ctk.CTkFrame(self, fg_color="transparent") + self.inner_pad.pack(fill="both", expand=True, padx=20, pady=20) + + self.client_header_frame = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.client_header_frame.pack(fill="x", pady=(0, 10)) + ctk.CTkLabel(self.client_header_frame, text="Email Client Classification", font=FONT_HEADER_SMALL, text_color=COLOR_TEXT_MAIN).pack(side="left") + self.client_reload_btn = ctk.CTkButton( + self.client_header_frame, + state="disabled", text="↻ Reload", width=80, height=24, + font=ctk.CTkFont(family="Segoe UI", size=12), + fg_color="transparent", border_width=1, text_color="#2563EB", hover_color="#DBEAFE", + command=self._retry_client_fetch + ) + self.client_reload_btn.pack(side="right") + + self.client_grid_frame = ctk.CTkFrame(self.inner_pad, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + self.client_grid_frame.pack(fill="x", expand=True) + + self.reset_view() + + def reset_view(self): + self.status = None + self._cached_client_data = {} + for w in self.client_grid_frame.winfo_children(): w.destroy() + + def _retry_client_fetch(self): + tenant, clients, secrets = self.get_credentials() + if tenant and clients and secrets: + self.client_reload_btn.configure(state="disabled") + self.trigger_fetch(tenant, clients[0], secrets[0]) + + def trigger_fetch(self, tenant, client_id, client_secret): + self.status = "loading" + self.on_status_change() + + for w in self.client_grid_frame.winfo_children(): w.destroy() + f = ctk.CTkFrame(self.client_grid_frame, fg_color="transparent") + ctk.CTkLabel(f, text="⏳ Analyzing Email Clients...", text_color=COLOR_TEXT_SUB, font=FONT_BODY_MEDIUM).pack(pady=(20, 5)) + pb = ctk.CTkProgressBar(f, mode="indeterminate", width=250, fg_color=COLOR_SURFACE_VARIANT, progress_color=COLOR_PRIMARY) + pb.pack(pady=(0, 20)) + pb.start() + f.pack(fill="x", expand=True) + + threading.Thread(target=self._execute_client_worker, args=(tenant, client_id, client_secret), daemon=True).start() + + def _execute_client_worker(self, tenant, client_id, client_secret): + if self.semaphore: self.semaphore.acquire() + try: + data = run_email_client_usage_pipeline(client_id, client_secret, tenant) + if not data.get("client_error"): + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + if os.path.basename(script_dir) == "exchange": + script_dir = os.path.dirname(script_dir) + reports_dir = os.path.join(script_dir, "reports", f"{tenant}_{client_id}") + os.makedirs(reports_dir, exist_ok=True) + csv_path = os.path.join(reports_dir, "email_client_support_metrics.csv") + + with open(csv_path, 'w', encoding='utf-8', newline='') as f: + writer = csv.writer(f) + writer.writerow(["Client Environment", "Active User Count"]) + adop = data.get("client_adoption", {}) + writer.writerow(["Browser Users (Web)", adop.get("browser_users", 0)]) + writer.writerow(["Desktop Windows (Outlook)", adop.get("desktop_win", 0)]) + writer.writerow(["Desktop Mac (Outlook)", adop.get("desktop_mac", 0)]) + writer.writerow(["Desktop Mac (Mail)", adop.get("desktop_mail_mac", 0)]) + writer.writerow(["Mobile Outlook", adop.get("mobile_outlook", 0)]) + writer.writerow(["Mobile Native (Exchange ActiveSync)", adop.get("mobile_native", 0)]) + writer.writerow(["IMAP Users", adop.get("imap_users", 0)]) + writer.writerow(["POP Users", adop.get("pop_users", 0)]) + writer.writerow(["SMTP Users", adop.get("smtp_users", 0)]) + + db_path = os.path.join(reports_dir, "telemetry_cache.db") + asyncio.run(import_csv_to_sqlite(csv_path, db_path, "email_clients")) + + self.after(0, self._render_client_success, data) + except Exception as e: + self.after(0, self._render_client_error, str(e)) + finally: + if self.semaphore: self.semaphore.release() + + def _render_client_error(self, error_msg): + self._cached_client_data = {"client_error": error_msg} + self.client_reload_btn.configure(state="normal") + for w in self.client_grid_frame.winfo_children(): w.destroy() + f = ctk.CTkFrame(self.client_grid_frame, fg_color="transparent") + display_msg = error_msg + if "401" in error_msg or "403" in error_msg or "unauthorized" in error_msg.lower(): + display_msg = "Reports permission required. Please grant 'Reports.Read.All'." + ctk.CTkLabel(f, text=f"✖ {display_msg}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center").pack(pady=(20, 5)) + ctk.CTkButton(f, text="Try Again", command=self._retry_client_fetch, width=120, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 20)) + f.pack(fill="x", expand=True) + self.status = "error" + self.on_status_change() + + def _render_client_success(self, data: dict): + self._cached_client_data = data + self.client_reload_btn.configure(state="normal") + for w in self.client_grid_frame.winfo_children(): w.destroy() + + for i in range(2): + self.client_grid_frame.grid_columnconfigure(i, weight=1) + + headers_client = ["Email Client Classification", "Active User Counts (180-Day Telemetry)"] + for col_idx, head_text in enumerate(headers_client): + cell = ctk.CTkFrame(self.client_grid_frame, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=1, pady=1) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + client_err = data.get("client_error") + if client_err: + rows_client = [ + ("Supported Browser-Based Clients", f"✖ Error: {client_err}"), + ("Supported Non-Browser (Desktop)", f"✖ Error: {client_err}"), + ("Supported Non-Browser (Mobile)", f"✖ Error: {client_err}"), + ("Supported Non-Browser (Protocols)", f"✖ Error: {client_err}") + ] + else: + c_adop = data.get("client_adoption", {}) + b_users = c_adop.get("browser_users", 0) if c_adop else 0 + d_win = c_adop.get("desktop_win", 0) if c_adop else 0 + d_mac = c_adop.get("desktop_mac", 0) if c_adop else 0 + m_mac = c_adop.get("desktop_mail_mac", 0) if c_adop else 0 + m_out = c_adop.get("mobile_outlook", 0) if c_adop else 0 + m_oth = c_adop.get("mobile_other", 0) if c_adop else 0 + p_imap = c_adop.get("protocol_imap4", 0) if c_adop else 0 + p_smtp = c_adop.get("protocol_smtp", 0) if c_adop else 0 + p_pop = c_adop.get("protocol_pop3", 0) if c_adop else 0 + + d_str = (f"• Outlook for Windows: {d_win:,} Users\n" + f"• Outlook for Mac: {d_mac:,} Users\n" + f"• Apple Mail (macOS): {m_mac:,} Users") + + m_str = (f"• Outlook Mobile (iOS/Android): {m_out:,} Users\n" + f"• Native / Other Mobile Apps: {m_oth:,} Users") + + p_str = (f"• IMAP4 App: {p_imap:,} Users\n" + f"• POP3 App: {p_pop:,} Users\n" + f"• SMTP App: {p_smtp:,} Accounts") + + rows_client = [ + ("Supported Browser-Based Clients", f"• Outlook on the Web (OWA): {b_users:,} Users"), + ("Supported Non-Browser (Desktop)", d_str), + ("Supported Non-Browser (Mobile)", m_str), + ("Supported Non-Browser (Protocols)", p_str) + ] + + for cr_idx, (c_name, c_val) in enumerate(rows_client, start=1): + bg_c = "transparent" if cr_idx % 2 != 0 else COLOR_SURFACE_VARIANT + + cc0 = ctk.CTkFrame(self.client_grid_frame, fg_color=bg_c, corner_radius=0) + cc0.grid(row=cr_idx, column=0, sticky="nsew", padx=1, pady=1) + ctk.CTkLabel(cc0, text=c_name, font=FONT_BODY_BOLD, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=10, anchor="nw") + + cc1 = ctk.CTkFrame(self.client_grid_frame, fg_color=bg_c, corner_radius=0) + cc1.grid(row=cr_idx, column=1, sticky="nsew", padx=1, pady=1) + ctk.CTkLabel(cc1, text=c_val, font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN, justify="left").pack(padx=10, pady=10, anchor="nw") + + self.status = "success" + self.on_status_change() + + def cancel(self): + pass + + def _load_client_data_from_csv(self): + tenant, clients, secrets = self.get_credentials() + if not tenant or not clients: + return {} + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + if os.path.basename(script_dir) == "exchange": + script_dir = os.path.dirname(script_dir) + db_path = os.path.join(script_dir, "reports", f"{tenant}_{clients[0]}", "telemetry_cache.db") + + if not os.path.exists(db_path): + return {} + + adop = {} + try: + conn = sqlite3.connect(db_path) + cursor = conn.cursor() + cursor.execute("SELECT * FROM email_clients") + rows = cursor.fetchall() + conn.close() + for row in rows: + if len(row) >= 2: + name, val = row[0], int(row[1]) + if "Browser Users" in name: adop["browser_users"] = val + elif "Desktop Windows" in name: adop["desktop_win"] = val + elif "Desktop Mac (Outlook)" in name: adop["desktop_mac"] = val + elif "Desktop Mac (Mail)" in name: adop["desktop_mail_mac"] = val + elif "Mobile Outlook" in name: adop["mobile_outlook"] = val + elif "Mobile Native" in name: adop["mobile_other"] = val + elif "IMAP" in name: adop["protocol_imap4"] = val + elif "POP" in name: adop["protocol_pop3"] = val + elif "SMTP" in name: adop["protocol_smtp"] = val + return {"client_adoption": adop, "client_error": None} + except Exception as e: + usage_logger.error(f"Error loading client data from DB: {e}") + return {"client_error": str(e)} + + @property + def last_data(self): + if hasattr(self, "_cached_client_data") and self._cached_client_data: + return self._cached_client_data + return self._load_client_data_from_csv() diff --git a/telemetry/exchange/integrated_apps.py b/telemetry/exchange/integrated_apps.py new file mode 100644 index 00000000..485d0e23 --- /dev/null +++ b/telemetry/exchange/integrated_apps.py @@ -0,0 +1,247 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""UI frame for Exchange Online Integrated Apps (Organization-wide Apps).""" + +import os +import csv +import logging +import threading +import customtkinter as ctk + +from core.graph.exchange.integrated_apps import run_exchange_apps_pipeline +from telemetry.styles import * + +usage_logger = logging.getLogger("M365TelemetryAsyncLogger.ExchangeAppsUI") + +class ExchangeAppsFrame(ctk.CTkFrame): + """Self-contained customtkinter component wrapping Exchange Online Org-wide Apps UI.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, **kwargs): + self.semaphore = kwargs.pop("concurrency_semaphore", None) + super().__init__(master, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=12, **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.status = None + self.last_apps = [] + + self.build_ui() + + def build_ui(self): + """Creates card container for the tab.""" + self.pack(fill="x", expand=True, pady=10) + + self.inner_pad = ctk.CTkFrame(self, fg_color="transparent") + self.inner_pad.pack(fill="both", expand=True, padx=20, pady=20) + + self.header = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.header.pack(fill="x", pady=(0, 10)) + ctk.CTkLabel(self.header, text="Integrated Apps", font=FONT_HEADER_SMALL, text_color=COLOR_TEXT_MAIN).pack(side="left") + self.reload_btn = ctk.CTkButton( + self.header, + state="disabled", text="↻ Reload", + width=80, + height=24, + font=ctk.CTkFont(family="Segoe UI", size=12), + fg_color="transparent", + border_width=1, + text_color="#2563EB", + hover_color="#DBEAFE", + command=self._retry_fetch + ) + self.reload_btn.pack(side="right") + + self.state_frame = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.warning_label = ctk.CTkLabel(self.inner_pad, text="", font=FONT_BODY_MEDIUM, text_color=COLOR_ERROR, justify="left", anchor="w", wraplength=750) + self.grid_frame = ctk.CTkFrame(self.inner_pad, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + + self.reset_view() + + def reset_view(self): + """Resets and hides grids.""" + self.state_frame.pack_forget() + self.grid_frame.pack_forget() + self.last_apps = [] + if hasattr(self, "warning_label"): + self.warning_label.pack_forget() + + for w in self.state_frame.winfo_children(): + w.destroy() + for w in self.grid_frame.winfo_children(): + w.destroy() + + def _set_state_loading(self, msg="Loading..."): + for w in self.state_frame.winfo_children(): + w.destroy() + self.loading_label = ctk.CTkLabel(self.state_frame, text=f"⏳ {msg}", text_color="#6b7280", font=ctk.CTkFont(family="Segoe UI", size=13)) + self.loading_label.pack(pady=(20, 5)) + pb = ctk.CTkProgressBar(self.state_frame, mode="indeterminate", width=250, fg_color="#F3F4F6", progress_color="#2563EB") + pb.pack(pady=(0, 20)) + pb.start() + self.state_frame.pack(fill="x", expand=True) + + def _set_state_error(self, error_msg): + for w in self.state_frame.winfo_children(): + w.destroy() + + ctk.CTkLabel(self.state_frame, text=f"✖ {error_msg}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center").pack(pady=(20, 5)) + ctk.CTkButton(self.state_frame, text="Try Again", command=self._retry_fetch, width=120, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 20)) + self.state_frame.pack(fill="x", expand=True) + + def _retry_fetch(self): + if hasattr(self, 'reload_btn') and self.reload_btn.winfo_exists(): + self.reload_btn.configure(state="disabled") + tenant, clients, secrets = self.get_credentials() + if tenant: + self.trigger_fetch(tenant, clients[0], secrets[0]) + + def trigger_fetch(self, tenant, client_id, client_secret): + """Triggers background fetch thread.""" + usage_logger.info("Exchange Apps trigger_fetch called. Spawning background thread...") + self.status = "loading" + self.on_status_change() + + self.grid_frame.pack_forget() + self.warning_label.pack_forget() + + self._set_state_loading("Downloading Exchange organization-wide apps configurations...") + + threading.Thread( + target=self._execute_worker, + args=(tenant, client_id, client_secret), + daemon=True + ).start() + + def _execute_worker(self, tenant: str, client_id: str, client_secret: str): + if self.semaphore: + self.semaphore.acquire() + try: + data = run_exchange_apps_pipeline(client_id, client_secret, tenant) + usage_logger.info("Successfully completed Exchange Apps telemetry data fetch.") + self.after(0, self._render_success, data) + except Exception as e: + usage_logger.error("Exception caught in Exchange Apps worker.", exc_info=True) + self.after(0, self._render_error, str(e)) + finally: + if self.semaphore: + self.semaphore.release() + + def _render_success(self, data: dict): + usage_logger.info("Exchange Apps data successfully retrieved. Rendering UI grid.") + if hasattr(self, 'reload_btn') and self.reload_btn.winfo_exists(): + self.reload_btn.configure(state="normal") + self.state_frame.pack_forget() + for w in self.grid_frame.winfo_children(): + w.destroy() + + if data.get("powershell_error"): + friendly_msg = f"Exchange PowerShell query failed: {data['powershell_error']}" + self.warning_label.configure(text=f"⚠️ Warning: {friendly_msg}") + self.warning_label.pack(anchor="w", pady=(0, 10)) + else: + self.warning_label.pack_forget() + + self.grid_frame.pack(fill="x", expand=True) + + for i in range(4): + self.grid_frame.grid_columnconfigure(i, weight=1) + + headers = ["App Display Name", "Status", "App Display Name", "Status"] + for col_idx, head_text in enumerate(headers): + cell = ctk.CTkFrame(self.grid_frame, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=1, pady=1) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + apps_list = data.get("OrganizationApps", []) + apps_err = data.get("AppsError") + self.last_apps = apps_list + + if apps_list and not apps_err: + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + if os.path.basename(script_dir) == "exchange": + script_dir = os.path.dirname(script_dir) + tenant, clients, _ = self.get_credentials() + reports_dir = os.path.join(script_dir, "reports", f"{tenant}_{clients[0]}") + os.makedirs(reports_dir, exist_ok=True) + csv_path = os.path.join(reports_dir, "exchange_organization_apps.csv") + + try: + with open(csv_path, 'w', encoding='utf-8', newline='') as f: + writer = csv.writer(f) + writer.writerow(["DisplayName", "Enabled"]) + for app in apps_list: + writer.writerow([app.get("DisplayName", "-"), app.get("Enabled")]) + usage_logger.info(f"Successfully streamed Exchange Apps to {csv_path}") + except Exception as e: + usage_logger.error(f"Failed to stream Exchange Apps to CSV: {e}") + + if apps_err: + usage_logger.error(f"Exchange PowerShell query failed for organization apps: {apps_err}") + empty_cell = ctk.CTkFrame(self.grid_frame, fg_color="transparent") + empty_cell.grid(row=1, column=0, columnspan=4, sticky="nsew", pady=15) + ctk.CTkLabel(empty_cell, text=f"Error retrieving apps: {apps_err}", text_color=COLOR_ERROR).pack() + elif not apps_list: + empty_cell = ctk.CTkFrame(self.grid_frame, fg_color="transparent") + empty_cell.grid(row=1, column=0, columnspan=4, sticky="nsew", pady=15) + ctk.CTkLabel(empty_cell, text="No organization-wide apps found.", text_color=COLOR_TEXT_SUB).pack() + else: + half = (len(apps_list) + 1) // 2 + left_col = apps_list[:half] + right_col = apps_list[half:] + + for r_idx in range(half): + bg_style = "transparent" if r_idx % 2 != 0 else COLOR_SURFACE_VARIANT + row_items = [] + + if r_idx < len(left_col): + app = left_col[r_idx] + status_str = "Enabled" if app.get("Enabled") else "Disabled" + row_items.extend([app.get("DisplayName", "-"), status_str]) + else: + row_items.extend(["", ""]) + + if r_idx < len(right_col): + app = right_col[r_idx] + status_str = "Enabled" if app.get("Enabled") else "Disabled" + row_items.extend([app.get("DisplayName", "-"), status_str]) + else: + row_items.extend(["", ""]) + + for c_idx, val in enumerate(row_items): + cell = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + cell.grid(row=r_idx + 1, column=c_idx, sticky="nsew", padx=1, pady=1) + + is_enabled_col = c_idx in [1, 3] + fnt = FONT_BODY_MEDIUM if is_enabled_col else FONT_BODY_BOLD + + text_color = COLOR_TEXT_MAIN + if is_enabled_col and val == "Disabled": + text_color = COLOR_TEXT_SUB + + ctk.CTkLabel(cell, text=val, font=fnt, text_color=text_color, wraplength=200, justify="left", anchor="w").pack(padx=10, pady=6, anchor="w") + + self.status = "success" + self.on_status_change() + + def _render_error(self, err_msg): + usage_logger.warning(f"Exchange Apps Telemetry fetch failed: {err_msg}") + if hasattr(self, 'reload_btn') and self.reload_btn.winfo_exists(): + self.reload_btn.configure(state="normal") + self._set_state_error(err_msg) + self.status = "error" + self.on_status_change() + + def cancel(self): + pass diff --git a/telemetry/exchange/mail_security.py b/telemetry/exchange/mail_security.py new file mode 100644 index 00000000..e3528fcb --- /dev/null +++ b/telemetry/exchange/mail_security.py @@ -0,0 +1,295 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""UI frame for Exchange Online Mail Security.""" + +import os +import csv +import logging +import threading +import concurrent.futures +import customtkinter as ctk + +from core.graph.exchange.mail_security import run_mail_security_pipeline +from core.powershell.client import PowerShellClient +from core.powershell.encryption import get_encryption_policies +from telemetry.styles import * + +usage_logger = logging.getLogger("M365TelemetryAsyncLogger.MailSecurityUI") + +class MailSecurityFrame(ctk.CTkFrame): + """Self-contained customtkinter component wrapping Exchange Online Mail Security UI.""" + + def update_loading_text(self, text_msg): + if hasattr(self, 'loading_label') and self.loading_label.winfo_exists(): + self.loading_label.configure(text=f"⏳ {text_msg}") + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, **kwargs): + self.semaphore = kwargs.pop("concurrency_semaphore", None) + super().__init__(master, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=12, **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change_cb = status_change_callback + + self.status = None + self.loading = True + self.error_msg = None + + self.total_eop_users = 0 + self.eop_skus = [] + + self.total_defender_users = 0 + self.defender_skus = [] + + self.encryption_data = {"m365_policies": [], "exchange_deps": [], "error": None} + self.last_data = {} + + self.inner_pad = ctk.CTkFrame(self, fg_color="transparent") + self.inner_pad.pack(fill="both", expand=True, padx=20, pady=20) + + self.header = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.header.pack(fill="x", pady=(0, 10)) + ctk.CTkLabel(self.header, text="Mail Security", font=FONT_HEADER_SMALL, text_color=COLOR_TEXT_MAIN).pack(side="left") + self.reload_btn = ctk.CTkButton( + self.header, + state="disabled", text="↻ Reload", + width=80, + height=24, + font=ctk.CTkFont(family="Segoe UI", size=12), + fg_color="transparent", + border_width=1, + text_color="#2563EB", + hover_color="#DBEAFE", + command=self._retry_fetch + ) + self.reload_btn.pack(side="right") + + self.grid_frame = ctk.CTkFrame(self.inner_pad, fg_color=COLOR_SURFACE, corner_radius=12, border_width=1, border_color=COLOR_OUTLINE_LIGHT) + self.grid_frame.pack(fill="x", expand=True, pady=(0, 10)) + + self.loading_label = None + self.progress = None + self.render_ui_state() + + def trigger_fetch(self, tenant, client_id, client_secret): + self.status = "loading" + self.loading = True + self.render_ui_state() + threading.Thread(target=self._fetch_data, args=(tenant, client_id, client_secret), daemon=True).start() + + def _fetch_data(self, tenant, c_id, c_secret): + if self.semaphore: + self.semaphore.acquire() + try: + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor: + skus_future = executor.submit(run_mail_security_pipeline, c_id, c_secret, tenant) + + # Fetch encryption policies + def fetch_encryption(): + try: + from core.graph.client import GraphClient + from core.graph.directory import DirectoryService + + tenant_domain = tenant + client = None + try: + client = GraphClient( + tenant_id=tenant, + client_ids=c_id, + client_secrets=c_secret, + concurrency=1, + retries=3, + backoff=2 + ) + client.authenticate() + dir_svc = DirectoryService(client) + tenant_domain = dir_svc.get_tenant_primary_domain() + except Exception as e: + usage_logger.warning(f"Could not retrieve tenant domain via Graph. Falling back to Tenant ID Guid: {e}") + finally: + if client: + client.close() + + ps_client = PowerShellClient( + tenant_id=tenant_domain, + client_id=c_id, + client_secret=c_secret, + cert_tenant_id=tenant + ) + return get_encryption_policies(ps_client) + except Exception as e: + usage_logger.warning(f"Failed to fetch encryption policies: {e}") + return {"m365_policies": [], "exchange_deps": [], "error": str(e)} + + enc_future = executor.submit(fetch_encryption) + + result_data = skus_future.result() + self.encryption_data = enc_future.result() + + self.defender_skus = result_data["defender"]["skus"] + self.total_defender_users = result_data["defender"]["users"] + + self.eop_skus = result_data["eop"]["skus"] + self.total_eop_users = result_data["eop"]["users"] + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + if os.path.basename(script_dir) == "exchange": + script_dir = os.path.dirname(script_dir) + reports_dir = os.path.join(script_dir, "reports", f"{tenant}_{c_id}") + os.makedirs(reports_dir, exist_ok=True) + csv_path = os.path.join(reports_dir, "mail_security_licensing.csv") + + with open(csv_path, 'w', encoding='utf-8', newline='') as f: + writer = csv.writer(f) + writer.writerow(["Protection Type", "Associated SKUs", "Covered User Count"]) + writer.writerow(["Defender for Office 365", " | ".join(self.defender_skus) if self.defender_skus else "None", self.total_defender_users]) + writer.writerow(["Exchange Online Protection (EOP)", " | ".join(self.eop_skus) if self.eop_skus else "None", self.total_eop_users]) + + usage_logger.info(f"Successfully streamed Mail Security data to {csv_path}") + self.after(0, self._render_success, result_data) + except Exception as e: + usage_logger.error(f"Error fetching mail security SKUs: {e}", exc_info=True) + self.after(0, self._render_error, str(e)) + finally: + if self.semaphore: + self.semaphore.release() + + def _render_success(self, data): + self.last_data = data + self.status = "success" + self.loading = False + self.on_status_change() + + def _render_error(self, err_msg): + self._set_state_error(err_msg) + + def _set_state_error(self, error_msg): + self.error_msg = error_msg + self.status = "error" + self.loading = False + self.on_status_change() + + def reset_view(self): + self.status = None + self.error_msg = None + + def _retry_fetch(self): + tenant, clients, secrets = self.get_credentials() + if tenant and clients and secrets: + if hasattr(self, 'reload_btn') and self.reload_btn.winfo_exists(): + self.reload_btn.configure(state="disabled") + self.trigger_fetch(tenant, clients[0], secrets[0]) + + def cancel(self): + self.status = None + self.loading = False + self.error_msg = None + self.reset_view() + + def on_status_change(self): + self.render_ui_state() + if hasattr(self, "on_status_change_cb") and self.on_status_change_cb: + self.on_status_change_cb() + + def render_ui_state(self): + for widget in self.grid_frame.winfo_children(): + widget.destroy() + + if not self.loading and hasattr(self, 'reload_btn') and self.reload_btn.winfo_exists(): + self.reload_btn.configure(state="normal") + + if self.loading: + self.loading_label = ctk.CTkLabel(self.grid_frame, text="⏳ Loading Mail Security Data...", text_color="#6b7280", font=ctk.CTkFont(family="Segoe UI", size=13)) + self.loading_label.pack(pady=(20, 5)) + self.progress = ctk.CTkProgressBar(self.grid_frame, mode="indeterminate", width=250, fg_color=COLOR_SURFACE_VARIANT, progress_color=COLOR_PRIMARY) + self.progress.pack(pady=(0, 20)) + self.progress.start() + return + if self.error_msg: + f = ctk.CTkFrame(self.grid_frame, fg_color="transparent") + ctk.CTkLabel(f, text=f"✖ {self.error_msg}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center").pack(pady=(20, 5)) + ctk.CTkButton(f, text="Try Again", command=self._retry_fetch, width=120, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 20)) + f.pack(fill="x", expand=True) + return + + metrics_grid = ctk.CTkFrame(self.grid_frame, fg_color=COLOR_SURFACE, corner_radius=0) + metrics_grid.pack(fill="x", padx=10, pady=(5, 5)) + + headers = ["Mail Security Configuration", "Detected SKUs", "Affected Users"] + for i in range(3): + metrics_grid.grid_columnconfigure(i, weight=1 if i == 2 else 2) + + for col_idx, head_text in enumerate(headers): + cell = ctk.CTkFrame(metrics_grid, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + rows_data = [] + if self.defender_skus: + rows_data.append(("Microsoft Defender for Office 365", ", ".join(self.defender_skus), str(self.total_defender_users))) + if self.eop_skus: + rows_data.append(("Exchange Online Protection (Baseline)", ", ".join(self.eop_skus), str(self.total_eop_users))) + + if not rows_data: + c = ctk.CTkFrame(metrics_grid, fg_color=COLOR_SURFACE, corner_radius=0) + c.grid(row=1, column=0, columnspan=3, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c, text="No Mail Security SKUs detected.", font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN, justify="center").pack(padx=10, pady=12) + else: + for r_idx, vals in enumerate(rows_data, start=1): + bg_style = COLOR_SURFACE if r_idx % 2 != 0 else COLOR_SURFACE_VARIANT + for c_idx, val in enumerate(vals): + c = ctk.CTkFrame(metrics_grid, fg_color=bg_style, corner_radius=0) + c.grid(row=r_idx, column=c_idx, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c, text=val, font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN, justify="left", wraplength=450).pack(padx=10, pady=12, anchor="nw") + + # --- ENCRYPTION UI --- + ctk.CTkLabel(self.grid_frame, text="Encryption Key Management", font=FONT_HEADER_SMALL, text_color=COLOR_TEXT_MAIN).pack(fill="x", padx=15, pady=(20, 5)) + + enc_grid = ctk.CTkFrame(self.grid_frame, fg_color=COLOR_SURFACE, corner_radius=0) + enc_grid.pack(fill="x", padx=10, pady=(0, 10)) + + enc_headers = ["Key Management Model", "M365DataAtRestEncryptionPolicy", "DataEncryptionPolicy"] + for i in range(3): + enc_grid.grid_columnconfigure(i, weight=1) + + for col_idx, head_text in enumerate(enc_headers): + cell = ctk.CTkFrame(enc_grid, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + if self.encryption_data.get("error"): + # Error fetching policies (e.g., missing permissions or script error) + c = ctk.CTkFrame(enc_grid, fg_color=COLOR_SURFACE, corner_radius=0) + c.grid(row=1, column=0, columnspan=3, sticky="nsew", padx=0, pady=(0, 1)) + err_lbl = f"Failed to fetch encryption policies: {self.encryption_data['error']}\nNote: Requires Exchange Online PowerShell certificate auth." + ctk.CTkLabel(c, text=err_lbl, font=FONT_BODY_MEDIUM, text_color=COLOR_ERROR, justify="center").pack(padx=10, pady=12) + else: + m365_pols = self.encryption_data.get("m365_policies", []) + exc_deps = self.encryption_data.get("exchange_deps", []) + + posture = "Customer Key (Customer-Managed)" if (m365_pols or exc_deps) else "Microsoft-Managed Keys (Default)" + + m365_text = str(len(m365_pols)) if m365_pols else "None detected" + exc_text = str(len(exc_deps)) if exc_deps else "None detected" + + bg_style = COLOR_SURFACE + vals = [posture, m365_text, exc_text] + for c_idx, val in enumerate(vals): + c = ctk.CTkFrame(enc_grid, fg_color=bg_style, corner_radius=0) + c.grid(row=1, column=c_idx, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c, text=val, font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN, justify="left", wraplength=450).pack(padx=10, pady=12, anchor="nw") + + # --- DISCLAIMER --- + disclaimer = ctk.CTkLabel(self.grid_frame, text="Note: Users can track inbound connectors displayed below to identify 3rd-party security apps.", font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_SUB, anchor="w") + disclaimer.pack(fill="x", padx=15, pady=(5, 15)) diff --git a/telemetry/exchange/mailbox.py b/telemetry/exchange/mailbox.py new file mode 100644 index 00000000..3a494a4d --- /dev/null +++ b/telemetry/exchange/mailbox.py @@ -0,0 +1,227 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""UI frame for Exchange Online Mailbox Usage telemetry.""" + +import os +import logging +import threading +import customtkinter as ctk + +from core.graph.exchange.mailbox import run_mailbox_usage_pipeline +from telemetry.styles import * + +usage_logger = logging.getLogger("M365TelemetryAsyncLogger.MailboxUsageUI") + +class MailboxUsageFrame(ctk.CTkFrame): + """Self-contained customtkinter component wrapping Exchange Online Mailbox Usage UI.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, **kwargs): + self.semaphore = kwargs.pop("concurrency_semaphore", None) + super().__init__(master, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=12, **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.status = None # 'loading', 'success', 'error', None + self.last_data = {} + + self.build_ui() + + def build_ui(self): + """Creates card container for the tab.""" + self.pack(fill="x", expand=True, pady=10) + + self.inner_pad = ctk.CTkFrame(self, fg_color="transparent") + self.inner_pad.pack(fill="both", expand=True, padx=20, pady=20) + + self.header = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.header.pack(fill="x", pady=(0, 10)) + ctk.CTkLabel(self.header, text="Exchange Online Mailbox Usage", font=FONT_HEADER_SMALL, text_color=COLOR_TEXT_MAIN).pack(side="left") + self.reload_btn = ctk.CTkButton( + self.header, + state="disabled", text="↻ Reload", + width=80, + height=24, + font=ctk.CTkFont(family="Segoe UI", size=12), + fg_color="transparent", + border_width=1, + text_color="#2563EB", + hover_color="#DBEAFE", + command=self._retry_fetch + ) + self.reload_btn.pack(side="right") + + self.state_frame = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.warning_label = ctk.CTkLabel(self.inner_pad, text="", font=FONT_BODY_MEDIUM, text_color=COLOR_ERROR, justify="left", anchor="w", wraplength=750) + self.grid_frame = ctk.CTkFrame(self.inner_pad, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + + self.reset_view() + + def reset_view(self): + """Resets and hides grids.""" + self.state_frame.pack_forget() + self.grid_frame.pack_forget() + self.last_data = {} + if hasattr(self, "warning_label"): + self.warning_label.pack_forget() + + for w in self.state_frame.winfo_children(): + w.destroy() + for w in self.grid_frame.winfo_children(): + w.destroy() + + def _set_state_loading(self, msg="Loading..."): + for w in self.state_frame.winfo_children(): + w.destroy() + ctk.CTkLabel(self.state_frame, text=f"⏳ {msg}", text_color=COLOR_TEXT_SUB, font=FONT_BODY_MEDIUM).pack(pady=(20, 5)) + self.progress = ctk.CTkProgressBar(self.state_frame, mode="indeterminate", width=250, fg_color="#F3F4F6", progress_color="#2563EB") + self.progress.pack(pady=(0, 20)) + self.progress.start() + self.state_frame.pack(fill="x", expand=True) + + def _set_state_error(self, error_msg): + for w in self.state_frame.winfo_children(): + w.destroy() + + display_msg = error_msg + if "401" in error_msg or "403" in error_msg or "unauthorized" in error_msg.lower() or "forbidden" in error_msg.lower(): + display_msg = "Reports telemetry permission required.\nPlease grant the 'Reports.Read.All' application permission to your App Registration in Microsoft Entra ID." + + ctk.CTkLabel(self.state_frame, text=f"✖ {display_msg}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center").pack(pady=(20, 10)) + ctk.CTkButton(self.state_frame, text="Try Again", command=self._retry_fetch, width=120, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 20)) + self.state_frame.pack(fill="x", expand=True) + + def _retry_fetch(self): + if hasattr(self, 'reload_btn') and self.reload_btn.winfo_exists(): + self.reload_btn.configure(state="disabled") + tenant, clients, secrets = self.get_credentials() + if tenant: + self.trigger_fetch(tenant, clients[0], secrets[0]) + + def trigger_fetch(self, tenant, client_id, client_secret): + """Triggers parallel fetches inside isolated background threads.""" + usage_logger.info("Mailbox Usage trigger_fetch called. Spawning background worker thread...") + self.status = "loading" + self.on_status_change() + + self.grid_frame.pack_forget() + + self._set_state_loading("Downloading and parsing Mailbox Usage reports...") + + threading.Thread( + target=self._execute_worker, + args=(tenant, client_id, client_secret), + daemon=True + ).start() + + def _execute_worker(self, tenant: str, client_id: str, client_secret: str): + if self.semaphore: + self.semaphore.acquire() + try: + data = run_mailbox_usage_pipeline(client_id, client_secret, tenant) + usage_logger.info("Successfully completed Mailbox Usage telemetry data fetch.") + self.after(0, self._render_success, data) + except Exception as e: + usage_logger.error("Exception caught in Mailbox Usage worker.", exc_info=True) + self.after(0, self._render_error, str(e)) + finally: + if self.semaphore: + self.semaphore.release() + + def _render_success(self, data: dict): + if hasattr(self, 'reload_btn') and self.reload_btn.winfo_exists(): + self.reload_btn.configure(state="normal") + self.last_data = data + usage_logger.info("Mailbox Usage data successfully retrieved. Rendering UI grid.") + self.state_frame.pack_forget() + for w in self.grid_frame.winfo_children(): + w.destroy() + + if data.get("powershell_error"): + err_msg = data["powershell_error"] + if "powershell" in err_msg.lower() or "pwsh" in err_msg.lower(): + friendly_msg = "PowerShell Core ('pwsh') not installed/configured. Cannot retrieve shared mailbox or public folder statistics." + elif "exchangeonlinemanagement" in err_msg.lower(): + friendly_msg = "ExchangeOnlineManagement PowerShell module is missing. Run: Install-Module -Name ExchangeOnlineManagement -Scope CurrentUser" + else: + friendly_msg = f"Failed to retrieve shared mailbox or public folder statistics: {err_msg}" + + self.warning_label.configure(text=f"⚠️ Warning: {friendly_msg}") + self.warning_label.pack(anchor="w", pady=(0, 10)) + else: + self.warning_label.pack_forget() + + self.grid_frame.pack(fill="x", expand=True) + + self.grid_frame.grid_columnconfigure(0, weight=3) + self.grid_frame.grid_columnconfigure(1, weight=2) + + headers_sp = ["Mailbox Metric Description", "Value / Measurement"] + for col_idx, head_text in enumerate(headers_sp): + cell = ctk.CTkFrame(self.grid_frame, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=1, pady=1) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + rows_data = [ + ("Total Mailboxes Analyzed", f"{data.get('total_mailboxes', 0):,} Mailboxes"), + ("Total Size of All Mailboxes", data.get("total_storage_formatted", "0.00 Bytes")), + ("Average Mailbox Size", data.get("average_mailbox_size_formatted", "0.00 Bytes")), + ("Total Number of Emails", f"{data.get('total_emails', 0):,} Emails"), + ("Average Emails per User", f"{data.get('average_emails', 0.0):,.0f} Emails") + ] + + s_count = data.get("shared_mailboxes_count") + s_count_str = f"{s_count:,} Shared Mailboxes" if s_count is not None else "Error/Unavailable" + s_size_str = data.get("shared_mailboxes_total_formatted", "Error/Unavailable") + + pf_count = data.get("public_folders_count") + pf_count_str = f"{pf_count:,} Public Folders" if pf_count is not None else "Error/Unavailable" + + mail_pf_count = data.get("mail_public_folders_count") + mail_pf_count_str = f"{mail_pf_count:,} Public Folders" if mail_pf_count is not None else "Error/Unavailable" + + pf_size_str = data.get("public_folders_total_formatted", "Error/Unavailable") + + rows_data += [ + ("Shared Mailboxes Count", s_count_str), + ("Total Shared Mailbox Size", s_size_str), + ("Public Folders Count", pf_count_str), + ("Mail-enabled Public Folders Count", mail_pf_count_str), + ("Total Public Folder Size", pf_size_str) + ] + + for r_idx, (metric_name, val) in enumerate(rows_data, start=1): + bg_style = "transparent" if r_idx % 2 != 0 else COLOR_SURFACE_VARIANT + + c0 = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + c0.grid(row=r_idx, column=0, sticky="nsew", padx=1, pady=1) + ctk.CTkLabel(c0, text=metric_name, font=FONT_BODY_BOLD, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=6, anchor="w") + + c1 = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + c1.grid(row=r_idx, column=1, sticky="nsew", padx=1, pady=1) + ctk.CTkLabel(c1, text=val, font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=6, anchor="w") + + self.status = "success" + self.on_status_change() + + def _render_error(self, err_msg): + usage_logger.warning(f"Mailbox Usage fetch failed: {err_msg}") + if hasattr(self, 'reload_btn') and self.reload_btn.winfo_exists(): + self.reload_btn.configure(state="normal") + self._set_state_error(err_msg) + self.status = "error" + self.on_status_change() + + def cancel(self): + pass diff --git a/telemetry/exchange/pst_files.py b/telemetry/exchange/pst_files.py new file mode 100644 index 00000000..ea43d3e5 --- /dev/null +++ b/telemetry/exchange/pst_files.py @@ -0,0 +1,266 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""UI frame for Exchange Online PST Files discovery telemetry.""" + +import os +import csv +import logging +import threading +import asyncio +import sqlite3 +import customtkinter as ctk + +from core.graph.exchange.pst_files import run_pst_discovery_pipeline +from core.graph.exchange.mailbox import format_bytes +from core.graph.db import import_csv_to_sqlite, query_page_sync +from telemetry.styles import * + +usage_logger = logging.getLogger("M365TelemetryAsyncLogger.PstFilesUI") + +class PstFilesFrame(ctk.CTkFrame): + """Self-contained component wrapping Exchange Online PST Files discovery UI.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, **kwargs): + self.semaphore = kwargs.pop("concurrency_semaphore", None) + super().__init__(master, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=12, **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + + self.status = None + self._cached_pst_data = {} + self.pst_disclaimer_lbl = None + + self.build_ui() + + def build_ui(self): + self.pack(fill="x", expand=True, pady=10) + + self.inner_pad = ctk.CTkFrame(self, fg_color="transparent") + self.inner_pad.pack(fill="both", expand=True, padx=20, pady=20) + + self.pst_header_frame = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.pst_header_frame.pack(fill="x", pady=(0, 10)) + ctk.CTkLabel(self.pst_header_frame, text="PST Files", font=FONT_HEADER_SMALL, text_color=COLOR_TEXT_MAIN).pack(side="left") + self.pst_reload_btn = ctk.CTkButton( + self.pst_header_frame, + state="disabled", text="↻ Reload", width=80, height=24, + font=ctk.CTkFont(family="Segoe UI", size=12), + fg_color="transparent", border_width=1, text_color="#2563EB", hover_color="#DBEAFE", + command=self._retry_pst_fetch + ) + self.pst_reload_btn.pack(side="right") + + self.pst_grid_frame = ctk.CTkFrame(self.inner_pad, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + self.pst_grid_frame.pack(fill="x", expand=True) + + self.reset_view() + + def reset_view(self): + self.status = None + self._cached_pst_data = {} + for w in self.pst_grid_frame.winfo_children(): w.destroy() + if hasattr(self, 'pst_disclaimer_lbl') and self.pst_disclaimer_lbl: + self.pst_disclaimer_lbl.destroy() + self.pst_disclaimer_lbl = None + + def _retry_pst_fetch(self): + tenant, clients, secrets = self.get_credentials() + if tenant and clients and secrets: + self.pst_reload_btn.configure(state="disabled") + self.trigger_fetch(tenant, clients[0], secrets[0]) + + def trigger_fetch(self, tenant, client_id, client_secret): + self.status = "loading" + self.on_status_change() + + for w in self.pst_grid_frame.winfo_children(): w.destroy() + f = ctk.CTkFrame(self.pst_grid_frame, fg_color="transparent") + ctk.CTkLabel(f, text="⏳ Discovering PST Files...", text_color=COLOR_TEXT_SUB, font=FONT_BODY_MEDIUM).pack(pady=(20, 5)) + pb = ctk.CTkProgressBar(f, mode="indeterminate", width=250, fg_color=COLOR_SURFACE_VARIANT, progress_color=COLOR_PRIMARY) + pb.pack(pady=(0, 20)) + pb.start() + f.pack(fill="x", expand=True) + + threading.Thread(target=self._execute_pst_worker, args=(tenant, client_id, client_secret), daemon=True).start() + + def _execute_pst_worker(self, tenant, client_id, client_secret): + if self.semaphore: self.semaphore.acquire() + try: + data = run_pst_discovery_pipeline(client_id, client_secret, tenant) + if not data.get("pst_error"): + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + if os.path.basename(script_dir) == "exchange": + script_dir = os.path.dirname(script_dir) + reports_dir = os.path.join(script_dir, "reports", f"{tenant}_{client_id}") + os.makedirs(reports_dir, exist_ok=True) + csv_path = os.path.join(reports_dir, "pst_discovery.csv") + + pst_cloud = data.get("pst_cloud_data", {}) + cloud_count = 0 + cloud_bytes = 0 + if pst_cloud and "value" in pst_cloud: + for item in pst_cloud.get("value", []): + for hc in item.get("hitsContainers", []): + cloud_count += hc.get("total", 0) + for hit in hc.get("hits", []): + cloud_bytes += int(hit.get("resource", {}).get("size", 0)) + + with open(csv_path, 'w', encoding='utf-8', newline='') as f: + writer = csv.writer(f) + writer.writerow(["Location", "Discovered File Count", "Total Size (Bytes)"]) + writer.writerow(["Cloud (SharePoint & OneDrive)", cloud_count, cloud_bytes]) + + db_path = os.path.join(reports_dir, "telemetry_cache.db") + asyncio.run(import_csv_to_sqlite(csv_path, db_path, "pst_files")) + + self.after(0, self._render_pst_success, data) + except Exception as e: + self.after(0, self._render_pst_error, str(e)) + finally: + if self.semaphore: self.semaphore.release() + + def _render_pst_error(self, error_msg): + self._cached_pst_data = {"pst_error": error_msg} + self.pst_reload_btn.configure(state="normal") + for w in self.pst_grid_frame.winfo_children(): w.destroy() + f = ctk.CTkFrame(self.pst_grid_frame, fg_color="transparent") + display_msg = error_msg + if "401" in error_msg or "403" in error_msg or "unauthorized" in error_msg.lower(): + display_msg = "Search permission required. Please grant 'Files.Read.All'." + ctk.CTkLabel(f, text=f"✖ {display_msg}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center").pack(pady=(20, 5)) + ctk.CTkButton(f, text="Try Again", command=self._retry_pst_fetch, width=120, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 20)) + f.pack(fill="x", expand=True) + self.status = "error" + self.on_status_change() + + def _render_pst_success(self, data: dict): + self._cached_pst_data = data + self.pst_reload_btn.configure(state="normal") + for w in self.pst_grid_frame.winfo_children(): w.destroy() + + self.pst_grid_frame.grid_columnconfigure(0, weight=2) + self.pst_grid_frame.grid_columnconfigure(1, weight=5) + + headers_pst = ["PST Storage Location", "Discovered File Count & Size"] + for col_idx, head_text in enumerate(headers_pst): + cell = ctk.CTkFrame(self.pst_grid_frame, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=1, pady=1) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + pst_err = data.get("pst_error") + cloud_count = 0 + if pst_err: + cloud_str = f"✖ Error: {pst_err}" + else: + pst_cloud = data.get("pst_cloud_data", {}) + cloud_bytes = 0 + if pst_cloud and "value" in pst_cloud: + for item in pst_cloud.get("value", []): + for hc in item.get("hitsContainers", []): + cloud_count += hc.get("total", 0) + for hit in hc.get("hits", []): + cloud_bytes += int(hit.get("resource", {}).get("size", 0)) + + cloud_size_str = f" ({format_bytes(cloud_bytes)})" if cloud_bytes > 0 else "" + cloud_str = f"{cloud_count:,} Files{cloud_size_str}" if cloud_count > 0 else "None Detected" + + rows_pst = [ + ("Cloud (SharePoint & OneDrive)", cloud_str) + ] + + for p_idx, (p_name, p_val) in enumerate(rows_pst, start=1): + bg_p = "transparent" if p_idx % 2 != 0 else COLOR_SURFACE_VARIANT + + pp0 = ctk.CTkFrame(self.pst_grid_frame, fg_color=bg_p, corner_radius=0) + pp0.grid(row=p_idx, column=0, sticky="nsew", padx=1, pady=1) + ctk.CTkLabel(pp0, text=p_name, font=FONT_BODY_BOLD, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=10, anchor="nw") + + pp1 = ctk.CTkFrame(self.pst_grid_frame, fg_color=bg_p, corner_radius=0) + pp1.grid(row=p_idx, column=1, sticky="nsew", padx=1, pady=1) + ctk.CTkLabel(pp1, text=p_val, font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN, justify="left").pack(padx=10, pady=10, anchor="nw") + + if not pst_err and cloud_count > 0: + if hasattr(self, 'pst_disclaimer_lbl') and self.pst_disclaimer_lbl: + self.pst_disclaimer_lbl.destroy() + self.pst_disclaimer_lbl = ctk.CTkLabel( + self.inner_pad, + text="* Note: There may be more than 2,000 files in the tenant; this tool only checks up to 2,000 files.", + font=FONT_BODY_SMALL, + text_color=COLOR_TEXT_SUB, + justify="left" + ) + self.pst_disclaimer_lbl.pack(anchor="w", pady=(10, 0)) + + self.status = "success" + self.on_status_change() + + def cancel(self): + pass + + def _load_pst_data_from_csv(self): + tenant, clients, secrets = self.get_credentials() + if not tenant or not clients: + return {} + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + if os.path.basename(script_dir) == "exchange": + script_dir = os.path.dirname(script_dir) + db_path = os.path.join(script_dir, "reports", f"{tenant}_{clients[0]}", "telemetry_cache.db") + + if not os.path.exists(db_path): + return {} + + try: + conn = sqlite3.connect(db_path) + cursor = conn.cursor() + cursor.execute("SELECT * FROM pst_files") + rows = cursor.fetchall() + conn.close() + if rows: + row = rows[0] + if len(row) >= 3: + count, size = int(row[1]), int(row[2]) + return { + "pst_cloud_data": { + "value": [ + { + "hitsContainers": [ + { + "total": count, + "hits": [ + { + "resource": { + "size": size + } + } + ] + } + ] + } + ] + }, + "pst_error": None + } + return {} + except Exception as e: + usage_logger.error(f"Error loading PST data from DB: {e}") + return {"pst_error": str(e)} + + @property + def last_data(self): + if hasattr(self, "_cached_pst_data") and self._cached_pst_data: + return self._cached_pst_data + return self._load_pst_data_from_csv() diff --git a/telemetry/exchange/transport_rules.py b/telemetry/exchange/transport_rules.py new file mode 100644 index 00000000..dd5952fd --- /dev/null +++ b/telemetry/exchange/transport_rules.py @@ -0,0 +1,330 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""UI frame for Exchange Online Transport Rules.""" + +import os +import csv +import logging +import threading +import customtkinter as ctk + +from core.graph.exchange.transport_rules import run_transport_rules_pipeline +from telemetry.styles import * + +usage_logger = logging.getLogger("M365TelemetryAsyncLogger.TransportRulesUI") + +class CsvLazyList: + """A lazy list-like wrapper around a CSV file to provide O(1) memory slicing and length.""" + def __init__(self, csv_path): + self.csv_path = csv_path + self._len = None + + def __len__(self): + if not os.path.exists(self.csv_path): + return 0 + if self._len is None: + with open(self.csv_path, 'r', encoding='utf-8') as f: + reader = csv.DictReader(f) + self._len = max(0, sum(1 for _ in reader)) + return self._len + + def __bool__(self): + return len(self) > 0 + + def __getitem__(self, key): + if not os.path.exists(self.csv_path): + raise IndexError("File does not exist") + if isinstance(key, slice): + start = key.start or 0 + stop = key.stop or len(self) + res = [] + with open(self.csv_path, 'r', encoding='utf-8') as f: + reader = csv.DictReader(f) + for i, row in enumerate(reader): + if start <= i < stop: + res.append(row) + elif i >= stop: + break + return res + else: + with open(self.csv_path, 'r', encoding='utf-8') as f: + reader = csv.DictReader(f) + for i, row in enumerate(reader): + if i == key: + return row + raise IndexError("list index out of range") + +class TransportRulesFrame(ctk.CTkFrame): + """Self-contained component for Exchange Transport Rules UI.""" + + def update_loading_text(self, text_msg): + if hasattr(self, 'loading_label') and self.loading_label.winfo_exists(): + self.loading_label.configure(text=f"⏳ {text_msg}") + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, **kwargs): + self.semaphore = kwargs.pop("concurrency_semaphore", None) + super().__init__(master, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=12, **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change_cb = status_change_callback + + self.status = None + self.loading = True + self.error_msg = None + + self.last_data = [] + self.page_index = 0 + self.page_size = 5 + self.csv_path = None + + self.inner_pad = ctk.CTkFrame(self, fg_color="transparent") + self.inner_pad.pack(fill="both", expand=True, padx=20, pady=20) + + self.header = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.header.pack(fill="x", pady=(0, 10)) + ctk.CTkLabel(self.header, text="Exchange Transport Rules", font=FONT_HEADER_SMALL, text_color=COLOR_TEXT_MAIN).pack(side="left") + + self.reload_btn = ctk.CTkButton( + self.header, + state="disabled", text="↻ Reload", + width=80, + height=24, + font=ctk.CTkFont(family="Segoe UI", size=12), + fg_color="transparent", + border_width=1, + text_color="#2563EB", + hover_color="#DBEAFE", + command=self._retry_fetch + ) + self.reload_btn.pack(side="right") + + self.state_frame = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + + self.grid_frame = ctk.CTkFrame(self.inner_pad, fg_color=COLOR_SURFACE, corner_radius=12, border_width=1, border_color=COLOR_OUTLINE_LIGHT) + self.grid_frame.pack_forget() + + self.pagination_frame = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.pagination_frame.pack_forget() + + self.loading_label = None + self.progress = None + self.render_ui_state() + + def trigger_fetch(self, tenant, client_id, client_secret): + self.status = "loading" + self.loading = True + self.render_ui_state() + threading.Thread(target=self._fetch_data, args=(tenant, client_id, client_secret), daemon=True).start() + + def _fetch_data(self, tenant, c_id, c_secret): + if self.semaphore: + self.semaphore.acquire() + try: + res = run_transport_rules_pipeline(c_id, c_secret, tenant) + self.csv_path = res["csv_path"] + self.after(0, self._render_success) + except Exception as e: + usage_logger.error(f"Error fetching transport rules: {e}", exc_info=True) + self.after(0, self._render_error, str(e)) + finally: + if self.semaphore: + self.semaphore.release() + + def _render_success(self): + self.last_data = CsvLazyList(self.csv_path) + self.status = "success" + self.loading = False + self.page_index = 0 + self.on_status_change() + + def _render_error(self, err_msg): + self._set_state_error(err_msg) + + def _set_state_error(self, error_msg): + self.error_msg = error_msg + self.status = "error" + self.loading = False + self.on_status_change() + + def reset_view(self): + self.state_frame.pack_forget() + self.grid_frame.pack_forget() + self.pagination_frame.pack_forget() + self.status = None + self.error_msg = None + + def _retry_fetch(self): + tenant, clients, secrets = self.get_credentials() + if tenant and clients and secrets: + if hasattr(self, 'reload_btn') and self.reload_btn.winfo_exists(): + self.reload_btn.configure(state="disabled") + self.trigger_fetch(tenant, clients[0], secrets[0]) + + def cancel(self): + self.status = None + self.loading = False + self.error_msg = None + self.reset_view() + + def on_status_change(self): + self.render_ui_state() + if hasattr(self, "on_status_change_cb") and self.on_status_change_cb: + self.on_status_change_cb() + + def render_ui_state(self): + self.state_frame.pack_forget() + self.grid_frame.pack_forget() + self.pagination_frame.pack_forget() + + for widget in self.state_frame.winfo_children(): + widget.destroy() + for widget in self.grid_frame.winfo_children(): + widget.destroy() + for widget in self.pagination_frame.winfo_children(): + widget.destroy() + + if not self.loading and hasattr(self, 'reload_btn') and self.reload_btn.winfo_exists(): + self.reload_btn.configure(state="normal") + + if self.loading: + self.loading_label = ctk.CTkLabel(self.state_frame, text="⏳ Initializing...", text_color="#6b7280", font=ctk.CTkFont(family="Segoe UI", size=13)) + self.loading_label.pack(pady=(20, 5)) + self.progress = ctk.CTkProgressBar(self.state_frame, mode="indeterminate", width=250, fg_color=COLOR_SURFACE_VARIANT, progress_color=COLOR_PRIMARY) + self.progress.pack(pady=(0, 20)) + self.progress.start() + self.state_frame.pack(fill="x", expand=True) + return + + if self.error_msg: + ctk.CTkLabel(self.state_frame, text=f"✖ {self.error_msg}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center").pack(pady=(20, 5)) + ctk.CTkButton(self.state_frame, text="Try Again", command=self._retry_fetch, width=120, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 20)) + self.state_frame.pack(fill="x", expand=True) + return + + self.grid_frame.pack(fill="x", pady=(0, 10)) + self.pagination_frame.pack(fill="x", pady=(5, 0)) + self._update_ui_paginated() + + def _update_ui_paginated(self): + for widget in self.grid_frame.winfo_children(): + widget.destroy() + for widget in self.pagination_frame.winfo_children(): + widget.destroy() + + if not self.csv_path or not os.path.exists(self.csv_path): + c = ctk.CTkFrame(self.grid_frame, fg_color=COLOR_SURFACE, corner_radius=0) + c.pack(fill="x", expand=True) + ctk.CTkLabel(c, text="No Transport Rules discovered.", font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN, justify="center").pack(padx=10, pady=20) + return + + total_items = len(self.last_data) + + if total_items <= 0: + c = ctk.CTkFrame(self.grid_frame, fg_color=COLOR_SURFACE, corner_radius=0) + c.pack(fill="x", expand=True) + ctk.CTkLabel(c, text="No Transport Rules discovered.", font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN, justify="center").pack(padx=10, pady=20) + return + + metrics_grid = ctk.CTkFrame(self.grid_frame, fg_color=COLOR_SURFACE, corner_radius=0) + metrics_grid.pack(fill="x", padx=10, pady=(5, 5)) + + headers = ["Rule Name", "State", "Priority", "Mode", "Rule Logic"] + for i in range(5): + if i == 0: + weight = 2 + elif i == 4: + weight = 4 + else: + weight = 1 + metrics_grid.grid_columnconfigure(i, weight=weight) + + for col_idx, head_text in enumerate(headers): + cell = ctk.CTkFrame(metrics_grid, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + start_idx = self.page_index * self.page_size + end_idx = min(start_idx + self.page_size, total_items) + + page_items = self.last_data[start_idx:end_idx] + + for r_idx, rule in enumerate(page_items, start=1): + bg_style = COLOR_SURFACE if r_idx % 2 != 0 else COLOR_SURFACE_VARIANT + + c0 = ctk.CTkFrame(metrics_grid, fg_color=bg_style, corner_radius=0) + c0.grid(row=r_idx, column=0, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c0, text=str(rule.get("Name", "-")), font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN, justify="left", wraplength=350).pack(padx=10, pady=12, anchor="w") + + c1 = ctk.CTkFrame(metrics_grid, fg_color=bg_style, corner_radius=0) + c1.grid(row=r_idx, column=1, sticky="nsew", padx=0, pady=(0, 1)) + status_text = rule.get("State", "-") + status_color = "#16a34a" if status_text == "Enabled" else COLOR_TEXT_SUB + ctk.CTkLabel(c1, text=status_text, font=FONT_BODY_MEDIUM, text_color=status_color).pack(padx=10, pady=12, anchor="w") + + c2 = ctk.CTkFrame(metrics_grid, fg_color=bg_style, corner_radius=0) + c2.grid(row=r_idx, column=2, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c2, text=str(rule.get("Priority", "-")), font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=12, anchor="w") + + c3 = ctk.CTkFrame(metrics_grid, fg_color=bg_style, corner_radius=0) + c3.grid(row=r_idx, column=3, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c3, text=str(rule.get("Mode", "-")), font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=12, anchor="w") + + c4 = ctk.CTkFrame(metrics_grid, fg_color=bg_style, corner_radius=0) + c4.grid(row=r_idx, column=4, sticky="nsew", padx=0, pady=(0, 1)) + + desc_text = rule.get("Description", "") + if desc_text: + desc_text = desc_text.strip() + else: + desc_text = "N/A" + + textbox = ctk.CTkTextbox(c4, height=85, fg_color="transparent", text_color=COLOR_TEXT_MAIN, font=FONT_BODY_MEDIUM, wrap="word") + textbox.pack(fill="both", expand=True, padx=5, pady=5) + textbox.insert("0.0", desc_text) + textbox.configure(state="disabled") + + if total_items >= 0: + total_pages = max(1, (total_items + self.page_size - 1) // self.page_size) + + center_container = ctk.CTkFrame(self.pagination_frame, fg_color="transparent") + center_container.pack(pady=(5, 10)) + + prev_btn = ctk.CTkButton(center_container, text="◀ Prev", width=70, height=22, corner_radius=6, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, + state="normal" if self.page_index > 0 else "disabled", + command=self._prev_page) + prev_btn.pack(side="left", padx=5) + + ctk.CTkLabel(center_container, text=f"Page {self.page_index + 1} of {total_pages}", + font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_SUB).pack(side="left", padx=15) + + next_btn = ctk.CTkButton(center_container, text="Next ▶", width=70, height=22, corner_radius=6, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, + state="normal" if self.page_index < total_pages - 1 else "disabled", + command=self._next_page) + next_btn.pack(side="left", padx=5) + + def _prev_page(self): + if self.page_index > 0: + self.page_index -= 1 + self._update_ui_paginated() + + def _next_page(self): + total_pages = (len(self.last_data) + self.page_size - 1) // self.page_size + if self.page_index < total_pages - 1: + self.page_index += 1 + self._update_ui_paginated() diff --git a/telemetry/exchange_apps.py b/telemetry/exchange_apps.py new file mode 100644 index 00000000..e68e4595 --- /dev/null +++ b/telemetry/exchange_apps.py @@ -0,0 +1,21 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Backward compatibility facade for Exchange Online Integrated Apps telemetry.""" + +# Re-export pipeline from core backend +from core.graph.exchange.integrated_apps import run_exchange_apps_pipeline + +# Re-export UI subframe from telemetry package +from telemetry.exchange.integrated_apps import ExchangeAppsFrame diff --git a/telemetry/exchange_connectors_ui.py b/telemetry/exchange_connectors_ui.py new file mode 100644 index 00000000..4e13753c --- /dev/null +++ b/telemetry/exchange_connectors_ui.py @@ -0,0 +1,21 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Backward compatibility facade for Exchange Online Connectors telemetry.""" + +# Re-export pipeline from core backend +from core.graph.exchange.connectors import fetch_exchange_connectors_data + +# Re-export UI subframe from telemetry package +from telemetry.exchange.connectors import ExchangeConnectorsFrame diff --git a/telemetry/files/__init__.py b/telemetry/files/__init__.py new file mode 100644 index 00000000..b8ef6026 --- /dev/null +++ b/telemetry/files/__init__.py @@ -0,0 +1,110 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Consolidated Files (SharePoint & OneDrive) Telemetry Orchestrator Container.""" + +import customtkinter as ctk +from telemetry.styles import * +from telemetry.files.sharepoint import SharePointUsageFrame +from telemetry.files.onedrive import OneDriveUsageFrame + +class FilesTelemetryFrame(ctk.CTkFrame): + """Uber section container hosting SharePoint and OneDrive telemetry frames vertically stacked.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, **kwargs): + self.semaphore = kwargs.pop("concurrency_semaphore", None) + super().__init__(master, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=12, **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.status = None + + self.build_ui() + + def build_ui(self): + """Instantiates and stacks SharePoint and OneDrive frames.""" + self.pack(fill="x", expand=True, pady=10) + + self.inner_pad = ctk.CTkFrame(self, fg_color="transparent") + self.inner_pad.pack(fill="both", expand=True, padx=20, pady=20) + + # Uber Title Heading + ctk.CTkLabel( + self.inner_pad, + text="Files", + font=FONT_HEADER_SMALL, + text_color=COLOR_TEXT_MAIN + ).pack(anchor="w", pady=(0, 5)) + + # SharePoint Usage Sub-frame + self.sharepoint_view = SharePointUsageFrame( + master=self.inner_pad, + log_callback=self.log_msg, + credentials_callback=self.get_credentials, + status_change_callback=self._check_overall_status, + concurrency_semaphore=self.semaphore + ) + self.sharepoint_view.configure(fg_color="transparent", border_width=0) + self.sharepoint_view.pack(fill="x", expand=True, pady=(0, 5)) + + # Separator line + self.divider = ctk.CTkFrame(self.inner_pad, height=1, fg_color=COLOR_OUTLINE_LIGHT) + self.divider.pack(fill="x", pady=10) + + # OneDrive Usage Sub-frame + self.onedrive_view = OneDriveUsageFrame( + master=self.inner_pad, + log_callback=self.log_msg, + credentials_callback=self.get_credentials, + status_change_callback=self._check_overall_status, + concurrency_semaphore=self.semaphore + ) + self.onedrive_view.configure(fg_color="transparent", border_width=0) + self.onedrive_view.pack(fill="x", expand=True, pady=(0, 5)) + + self.reset_view() + + def reset_view(self): + """Resets both sub-views and hides container.""" + self.pack_forget() + self.sharepoint_view.reset_view() + self.onedrive_view.reset_view() + self.divider.pack_forget() + + def trigger_fetch(self, tenant, client_id, client_secret): + """Displays container and delegates fetches to both sub-views.""" + self.status = "loading" + self.on_status_change() + + self.pack(fill="x", expand=True, pady=10) + self.divider.pack(fill="x", pady=10) + + self.sharepoint_view.trigger_fetch(tenant, client_id, client_secret) + self.onedrive_view.trigger_fetch(tenant, client_id, client_secret) + + def _check_overall_status(self): + """Updates main container status based on sub-frame statuses.""" + if self.sharepoint_view.status == "loading" or self.onedrive_view.status == "loading": + self.status = "loading" + elif self.sharepoint_view.status == "success" or self.onedrive_view.status == "success": + self.status = "success" + else: + self.status = "error" + self.on_status_change() + + def cancel(self): + """Cancels all child views in this container.""" + self.sharepoint_view.cancel() + self.onedrive_view.cancel() + self.status = None diff --git a/telemetry/files/msteams_overview.py b/telemetry/files/msteams_overview.py new file mode 100644 index 00000000..2c4c92fd --- /dev/null +++ b/telemetry/files/msteams_overview.py @@ -0,0 +1,229 @@ +import os +import logging +import threading +import sqlite3 +import asyncio +import customtkinter as ctk + +from core.graph.files.msteams_overview import run_msteams_pipeline +from core.graph.db import import_csv_to_sqlite +from telemetry.styles import * + +logger = logging.getLogger("M365TelemetryAsyncLogger.MsTeamsOverviewUI") + +class MsTeamsOverviewFrame(ctk.CTkFrame): + def __init__(self, master, log_callback, credentials_callback, status_change_callback, semaphore=None, **kwargs): + self.semaphore = semaphore + super().__init__(master, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=12, **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + + self.status = None + self.is_cancelled = False + self.current_request_id = 0 + self.csv_path = None + + self.build_ui() + + def build_ui(self): + self.pack(fill="x", expand=True, pady=10) + + self.inner_pad = ctk.CTkFrame(self, fg_color="transparent") + self.inner_pad.pack(fill="both", expand=True, padx=20, pady=20) + + self.header = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.header.pack(fill="x", pady=(0, 10)) + ctk.CTkLabel(self.header, text="Microsoft Teams Overview", font=FONT_HEADER_SMALL, text_color=COLOR_TEXT_MAIN).pack(side="left") + self.reload_btn = ctk.CTkButton( + self.header, + state="disabled", text="↻ Reload", + width=80, + height=24, + font=ctk.CTkFont(family="Segoe UI", size=12), + fg_color="transparent", + border_width=1, + text_color="#2563EB", + hover_color="#DBEAFE", + command=self._retry_fetch + ) + self.reload_btn.pack(side="right") + + self.state_frame = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.grid_frame = ctk.CTkFrame(self.inner_pad, fg_color=COLOR_OUTLINE_LIGHT, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + + self.reset_view() + + def reset_view(self): + self.pack_forget() + self.state_frame.pack_forget() + self.grid_frame.pack_forget() + + for w in self.state_frame.winfo_children(): + w.destroy() + for w in self.grid_frame.winfo_children(): + w.destroy() + + def _set_state_loading(self, msg="Loading..."): + for w in self.state_frame.winfo_children(): + w.destroy() + self.loading_label = ctk.CTkLabel(self.state_frame, text=f"⏳ {msg}", text_color="#6b7280", font=ctk.CTkFont(family="Segoe UI", size=13)) + self.loading_label.pack(pady=(20, 5)) + pb = ctk.CTkProgressBar(self.state_frame, mode="indeterminate", width=250, fg_color="#F3F4F6", progress_color="#2563EB") + pb.pack(pady=(0, 20)) + pb.start() + self.state_frame.pack(fill="x", expand=True) + + def _set_state_error(self, error_msg): + for w in self.state_frame.winfo_children(): + w.destroy() + + display_msg = error_msg + if "401" in error_msg or "403" in error_msg or "unauthorized" in error_msg.lower() or "forbidden" in error_msg.lower(): + display_msg = "Reports telemetry permission required.\nPlease grant the 'Reports.Read.All' application permission to your App Registration in Microsoft Entra ID." + + ctk.CTkLabel(self.state_frame, text=f"✖ {display_msg}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center").pack(pady=(20, 5)) + ctk.CTkButton(self.state_frame, text="Try Again", command=self._retry_fetch, width=120, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 20)) + self.state_frame.pack(fill="x", expand=True) + + def _retry_fetch(self): + if hasattr(self, 'reload_btn') and self.reload_btn.winfo_exists(): + self.reload_btn.configure(state="disabled") + tenant, clients, secrets = self.get_credentials() + if tenant: + self.trigger_fetch(tenant, clients[0], secrets[0]) + + def trigger_fetch(self, tenant, client_id, client_secret): + logger.info("MsTeams Overview trigger_fetch called.") + self.status = "loading" + self.is_cancelled = False + self.on_status_change() + + self.pack(fill="x", expand=True, pady=(20, 5)) + self.grid_frame.pack_forget() + + self._set_state_loading("Scanning MsTeams details...") + + threading.Thread( + target=self._execute_worker, + args=(tenant, client_id, client_secret, self.current_request_id), + daemon=True + ).start() + + def _execute_worker(self, tenant: str, client_id: str, client_secret: str, request_id: int): + if self.semaphore: + self.semaphore.acquire() + try: + if self.is_cancelled or request_id != self.current_request_id: return + + self.csv_path = run_msteams_pipeline(client_id, client_secret, tenant) + + if self.is_cancelled or request_id != self.current_request_id: return + + db_path = os.path.join(os.path.dirname(self.csv_path), "telemetry_cache.db") + asyncio.run(import_csv_to_sqlite(self.csv_path, db_path, "msteams_activity")) + + if self.is_cancelled or request_id != self.current_request_id: return + + self.status = "success" + self.after(0, self._render_success, request_id) + except Exception as e: + logger.error(f"Error fetching feature telemetry: {e}", exc_info=True) + self.status = "error" + self.after(0, self._render_error, str(e), request_id) + finally: + if self.semaphore: + self.semaphore.release() + self.after(0, self.on_status_change) + + def _load_metrics_from_sqlite(self): + if not self.csv_path or not os.path.exists(self.csv_path): return {} + db_path = os.path.join(os.path.dirname(self.csv_path), "telemetry_cache.db") + if not os.path.exists(db_path): return {} + + conn = sqlite3.connect(db_path) + metrics = {} + try: + conn.row_factory = sqlite3.Row + cursor = conn.cursor() + + cursor.execute("SELECT COUNT(*) FROM msteams_activity WHERE Team_Name IS NOT NULL AND Team_Name != ''") + row = cursor.fetchone() + metrics["total_teams"] = row[0] if row else 0 + + cursor.execute("SELECT SUM(Active_Users), SUM(Guests), SUM(Active_Channels), SUM(Channel_Messages), SUM(Meetings_Organized) FROM msteams_activity WHERE Team_Name IS NOT NULL AND Team_Name != ''") + row = cursor.fetchone() + metrics["active_users"] = int(row[0] or 0) if row else 0 + metrics["guests"] = int(row[1] or 0) if row else 0 + metrics["active_channels"] = int(row[2] or 0) if row else 0 + metrics["channel_messages"] = int(row[3] or 0) if row else 0 + metrics["meetings_organized"] = int(row[4] or 0) if row else 0 + + return metrics + except Exception as e: + logger.error(f"Error reading SQLite: {e}") + return {} + finally: + conn.close() + + def _render_success(self, request_id): + if self.is_cancelled or request_id != self.current_request_id: return + if hasattr(self, 'reload_btn') and self.reload_btn.winfo_exists(): + self.reload_btn.configure(state="normal") + + self.state_frame.pack_forget() + self._update_grid() + self.grid_frame.pack(fill="x", expand=True) + + def _render_error(self, err_msg, request_id): + if self.is_cancelled or request_id != self.current_request_id: return + logger.warning(f"MsTeams Overview fetch failed: {err_msg}") + if hasattr(self, 'reload_btn') and self.reload_btn.winfo_exists(): + self.reload_btn.configure(state="normal") + self._set_state_error(err_msg) + + def _update_grid(self): + for w in self.grid_frame.winfo_children(): + w.destroy() + + metrics = self._load_metrics_from_sqlite() + + self.grid_frame.grid_columnconfigure(0, weight=3) + self.grid_frame.grid_columnconfigure(1, weight=2) + + headers = ["MsTeams Metric Description", "Value / Measurement"] + for col_idx, head_text in enumerate(headers): + cell = ctk.CTkFrame(self.grid_frame, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + if not metrics: + c0 = ctk.CTkFrame(self.grid_frame, fg_color=COLOR_SURFACE, corner_radius=0) + c0.grid(row=1, column=0, columnspan=len(headers), sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c0, text="No activity data found.", font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=6) + return + + users = metrics.get('active_users', 0) + channels = metrics.get('active_channels', 0) + avg_users_per_channel = f"{(users / channels):.1f}" if channels > 0 else "0" + + rows_data = [ + ("Total Teams Count", f"{metrics.get('total_teams', 0):,} Teams"), + ("Total Active Channels (180 days)", f"{channels:,} Channels"), + ("Total Channel Messages", f"{metrics.get('channel_messages', 0):,} Messages"), + ("Total Active Users(180 days)", f"{users:,} Users"), + ("Average Users per Channel", avg_users_per_channel), + ("Total Meetings Organized", f"{metrics.get('meetings_organized', 0):,} Meetings"), + ("Total Guests", f"{metrics.get('guests', 0):,} Guests") + ] + + for r_idx, (metric_name, val) in enumerate(rows_data, start=1): + bg_style = COLOR_SURFACE if r_idx % 2 == 0 else COLOR_SURFACE_VARIANT + + c0 = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + c0.grid(row=r_idx, column=0, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c0, text=metric_name, font=FONT_BODY_BOLD, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=6, anchor="w") + + c1 = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + c1.grid(row=r_idx, column=1, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c1, text=val, font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=6, anchor="w") diff --git a/telemetry/files/onedrive.py b/telemetry/files/onedrive.py new file mode 100644 index 00000000..10bbac1c --- /dev/null +++ b/telemetry/files/onedrive.py @@ -0,0 +1,193 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""UI frame for OneDrive for Business Usage telemetry.""" + +import logging +import threading +import customtkinter as ctk + +from core.graph.files.onedrive import run_onedrive_pipeline +from telemetry.styles import * + +usage_logger = logging.getLogger("M365TelemetryAsyncLogger.OneDriveUsageUI") + +class OneDriveUsageFrame(ctk.CTkFrame): + """Self-contained customtkinter component wrapping OneDrive Telemetry UI.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, **kwargs): + self.semaphore = kwargs.pop("concurrency_semaphore", None) + super().__init__(master, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=12, **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.status = None # 'loading', 'success', 'error', None + self.last_data = {} + + self.build_ui() + + def build_ui(self): + """Creates card container for the tab.""" + self.pack(fill="x", expand=True, pady=10) + + self.inner_pad = ctk.CTkFrame(self, fg_color="transparent") + self.inner_pad.pack(fill="both", expand=True, padx=20, pady=20) + + self.header = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.header.pack(fill="x", pady=(0, 10)) + ctk.CTkLabel(self.header, text="OneDrive Usage (180 Days)", font=FONT_HEADER_SMALL, text_color=COLOR_TEXT_MAIN).pack(side="left") + self.reload_btn = ctk.CTkButton( + self.header, + state="disabled", text="↻ Reload", + width=80, + height=24, + font=ctk.CTkFont(family="Segoe UI", size=12), + fg_color="transparent", + border_width=1, + text_color="#2563EB", + hover_color="#DBEAFE", + command=self._retry_fetch + ) + self.reload_btn.pack(side="right") + + self.state_frame = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.grid_frame = ctk.CTkFrame(self.inner_pad, fg_color=COLOR_OUTLINE_LIGHT, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + + self.reset_view() + + def reset_view(self): + """Resets and hides grids.""" + self.pack_forget() + self.state_frame.pack_forget() + self.grid_frame.pack_forget() + self.last_data = {} + + for w in self.state_frame.winfo_children(): + w.destroy() + for w in self.grid_frame.winfo_children(): + w.destroy() + + def _set_state_loading(self, msg="Loading..."): + for w in self.state_frame.winfo_children(): + w.destroy() + self.loading_label = ctk.CTkLabel(self.state_frame, text=f"⏳ {msg}", text_color="#6b7280", font=ctk.CTkFont(family="Segoe UI", size=13)) + self.loading_label.pack(pady=(20, 5)) + pb = ctk.CTkProgressBar(self.state_frame, mode="indeterminate", width=250, fg_color="#F3F4F6", progress_color="#2563EB") + pb.pack(pady=(0, 20)) + pb.start() + self.state_frame.pack(fill="x", expand=True) + + def _set_state_error(self, error_msg): + for w in self.state_frame.winfo_children(): + w.destroy() + + display_msg = error_msg + if "401" in error_msg or "403" in error_msg or "unauthorized" in error_msg.lower() or "forbidden" in error_msg.lower(): + display_msg = "Reports telemetry permission required.\nPlease grant the 'Reports.Read.All' application permission to your App Registration in Microsoft Entra ID." + + ctk.CTkLabel(self.state_frame, text=f"✖ {display_msg}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center").pack(pady=(20, 5)) + ctk.CTkButton(self.state_frame, text="Try Again", command=self._retry_fetch, width=120, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 20)) + self.state_frame.pack(fill="x", expand=True) + + def _retry_fetch(self): + if hasattr(self, 'reload_btn') and self.reload_btn.winfo_exists(): + self.reload_btn.configure(state="disabled") + tenant, clients, secrets = self.get_credentials() + if tenant: + self.trigger_fetch(tenant, clients[0], secrets[0]) + + def trigger_fetch(self, tenant, client_id, client_secret): + """Triggers parallel fetches inside isolated background threads.""" + usage_logger.info("OneDrive Usage trigger_fetch called. Spawning background worker thread...") + self.status = "loading" + self.on_status_change() + + self.pack(fill="x", expand=True, pady=(20, 5)) + self.grid_frame.pack_forget() + + self._set_state_loading("Downloading and parsing OneDrive reports...") + + threading.Thread( + target=self._execute_worker, + args=(tenant, client_id, client_secret), + daemon=True + ).start() + + def _execute_worker(self, tenant: str, client_id: str, client_secret: str): + if self.semaphore: + self.semaphore.acquire() + try: + data = run_onedrive_pipeline(client_id, client_secret, tenant) + usage_logger.info("Successfully completed OneDrive telemetry data fetch.") + self.after(0, self._render_success, data) + except Exception as e: + usage_logger.error("Exception caught in OneDrive worker.", exc_info=True) + self.after(0, self._render_error, str(e)) + finally: + if self.semaphore: + self.semaphore.release() + + def _render_success(self, data: dict): + if hasattr(self, 'reload_btn') and self.reload_btn.winfo_exists(): + self.reload_btn.configure(state="normal") + self.last_data = data + usage_logger.info("OneDrive Usage data successfully retrieved. Rendering UI grid.") + self.state_frame.pack_forget() + for w in self.grid_frame.winfo_children(): + w.destroy() + + self.grid_frame.pack(fill="x", expand=True) + + self.grid_frame.grid_columnconfigure(0, weight=3) + self.grid_frame.grid_columnconfigure(1, weight=2) + + headers_od = ["OneDrive Metric Description", "Value / Measurement"] + for col_idx, head_text in enumerate(headers_od): + cell = ctk.CTkFrame(self.grid_frame, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + rows_data = [ + ("Total Accounts Count", f"{data.get('total_accounts', 0):,} Accounts"), + ("Total Storage Used", data.get("total_storage_formatted", "0.00 Bytes")), + ("Total Files Stored", f"{data.get('total_files', 0):,} Files"), + ("Active Files Count", f"{data.get('active_files', 0):,} Files ({data.get('active_files_pct', 0.0):.1f}%)"), + ("Users with Synced Files", f"{data.get('sync_users', 0):,} Users ({data.get('sync_users_pct', 0.0):.1f}%)"), + ("OneNote Active Users", f"{data.get('onenote_users', 0):,} Users") + ] + + for r_idx, (metric_name, val) in enumerate(rows_data, start=1): + bg_style = COLOR_SURFACE if r_idx % 2 == 0 else COLOR_SURFACE_VARIANT + + c0 = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + c0.grid(row=r_idx, column=0, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c0, text=metric_name, font=FONT_BODY_BOLD, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=6, anchor="w") + + c1 = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + c1.grid(row=r_idx, column=1, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c1, text=val, font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=6, anchor="w") + + self.status = "success" + self.on_status_change() + + def _render_error(self, err_msg): + usage_logger.warning(f"OneDrive Usage fetch failed: {err_msg}") + if hasattr(self, 'reload_btn') and self.reload_btn.winfo_exists(): + self.reload_btn.configure(state="normal") + self._set_state_error(err_msg) + self.status = "error" + self.on_status_change() + + def cancel(self): + pass diff --git a/telemetry/files/sharepoint.py b/telemetry/files/sharepoint.py new file mode 100644 index 00000000..135ac6b5 --- /dev/null +++ b/telemetry/files/sharepoint.py @@ -0,0 +1,203 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""UI frame for SharePoint Online Site Usage telemetry.""" + +import logging +import threading +import customtkinter as ctk + +from core.graph.files.sharepoint import run_sharepoint_pipeline +from core.graph.files.sharepoint_data_types import run_sharepoint_data_types_pipeline +import concurrent.futures +from telemetry.styles import * + +usage_logger = logging.getLogger("M365TelemetryAsyncLogger.SharePointUsageUI") + +class SharePointUsageFrame(ctk.CTkFrame): + """Self-contained customtkinter component wrapping SharePoint Telemetry UI.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, **kwargs): + self.semaphore = kwargs.pop("concurrency_semaphore", None) + super().__init__(master, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=12, **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.status = None # 'loading', 'success', 'error', None + self.last_data = {} + + self.build_ui() + + def build_ui(self): + """Creates card container for the tab.""" + self.pack(fill="x", expand=True, pady=10) + + self.inner_pad = ctk.CTkFrame(self, fg_color="transparent") + self.inner_pad.pack(fill="both", expand=True, padx=20, pady=20) + + self.header = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.header.pack(fill="x", pady=(0, 10)) + ctk.CTkLabel(self.header, text="SharePoint Overview", font=FONT_HEADER_SMALL, text_color=COLOR_TEXT_MAIN).pack(side="left") + self.reload_btn = ctk.CTkButton( + self.header, + state="disabled", text="↻ Reload", + width=80, + height=24, + font=ctk.CTkFont(family="Segoe UI", size=12), + fg_color="transparent", + border_width=1, + text_color="#2563EB", + hover_color="#DBEAFE", + command=self._retry_fetch + ) + self.reload_btn.pack(side="right") + + self.state_frame = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.grid_frame = ctk.CTkFrame(self.inner_pad, fg_color=COLOR_OUTLINE_LIGHT, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + + self.reset_view() + + def reset_view(self): + """Resets and hides grids.""" + self.pack_forget() + self.state_frame.pack_forget() + self.grid_frame.pack_forget() + self.last_data = {} + + for w in self.state_frame.winfo_children(): + w.destroy() + for w in self.grid_frame.winfo_children(): + w.destroy() + + def _set_state_loading(self, msg="Loading..."): + for w in self.state_frame.winfo_children(): + w.destroy() + self.loading_label = ctk.CTkLabel(self.state_frame, text=f"⏳ {msg}", text_color="#6b7280", font=ctk.CTkFont(family="Segoe UI", size=13)) + self.loading_label.pack(pady=(20, 5)) + pb = ctk.CTkProgressBar(self.state_frame, mode="indeterminate", width=250, fg_color="#F3F4F6", progress_color="#2563EB") + pb.pack(pady=(0, 20)) + pb.start() + self.state_frame.pack(fill="x", expand=True) + + def _set_state_error(self, error_msg): + for w in self.state_frame.winfo_children(): + w.destroy() + + display_msg = error_msg + if "401" in error_msg or "403" in error_msg or "unauthorized" in error_msg.lower() or "forbidden" in error_msg.lower(): + display_msg = "Reports telemetry permission required.\nPlease grant the 'Reports.Read.All' application permission to your App Registration in Microsoft Entra ID." + + ctk.CTkLabel(self.state_frame, text=f"✖ {display_msg}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center").pack(pady=(20, 5)) + ctk.CTkButton(self.state_frame, text="Try Again", command=self._retry_fetch, width=120, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 20)) + self.state_frame.pack(fill="x", expand=True) + + def _retry_fetch(self): + if hasattr(self, 'reload_btn') and self.reload_btn.winfo_exists(): + self.reload_btn.configure(state="disabled") + tenant, clients, secrets = self.get_credentials() + if tenant: + self.trigger_fetch(tenant, clients[0], secrets[0]) + + def trigger_fetch(self, tenant, client_id, client_secret): + """Triggers background fetch thread.""" + usage_logger.info("SharePoint Site Usage trigger_fetch called. Spawning background worker thread...") + self.status = "loading" + self.on_status_change() + + self.pack(fill="x", expand=True, pady=(20, 5)) + self.grid_frame.pack_forget() + + self._set_state_loading("Downloading and parsing SharePoint Site Usage and Data Types...") + + threading.Thread( + target=self._execute_worker, + args=(tenant, client_id, client_secret), + daemon=True + ).start() + + def _execute_worker(self, tenant: str, client_id: str, client_secret: str): + if self.semaphore: + self.semaphore.acquire() + try: + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor: + usage_future = executor.submit(run_sharepoint_pipeline, client_id, client_secret, tenant) + datatypes_future = executor.submit(run_sharepoint_data_types_pipeline, client_id, client_secret, tenant) + + usage_data = usage_future.result() + datatypes_data = datatypes_future.result() + + combined_data = {**usage_data, **datatypes_data} + usage_logger.info("Successfully completed SharePoint telemetry data fetch.") + self.after(0, self._render_success, combined_data) + except Exception as e: + usage_logger.error("Exception caught in SharePoint worker.", exc_info=True) + self.after(0, self._render_error, str(e)) + finally: + if self.semaphore: + self.semaphore.release() + + def _render_success(self, data: dict): + if hasattr(self, 'reload_btn') and self.reload_btn.winfo_exists(): + self.reload_btn.configure(state="normal") + self.last_data = data + usage_logger.info("SharePoint Site Usage data successfully retrieved. Rendering UI grid.") + self.state_frame.pack_forget() + for w in self.grid_frame.winfo_children(): + w.destroy() + + self.grid_frame.pack(fill="x", expand=True) + + self.grid_frame.grid_columnconfigure(0, weight=3) + self.grid_frame.grid_columnconfigure(1, weight=2) + + headers_sp = ["SharePoint Metric Description", "Value / Measurement"] + for col_idx, head_text in enumerate(headers_sp): + cell = ctk.CTkFrame(self.grid_frame, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + rows_data = [ + ("Total Sites Count", f"{data.get('total_sites', 0):,} Sites"), + ("Total Storage Used", data.get("total_storage_formatted", "0.00 Bytes")), + ("Total Files Stored", f"{data.get('total_files', 0):,} Files"), + ("Active Files Count (180 days)", f"{data.get('active_files', 0):,} Files ({data.get('active_files_pct', 0.0):.1f}%)"), + ("Document Libraries", f"{data.get('Document Libraries', 0):,}"), + ("Lists", f"{data.get('Lists', 0):,}"), + ("Web Pages", f"{data.get('Web Pages', 0):,}") + ] + + for r_idx, (metric_name, val) in enumerate(rows_data, start=1): + bg_style = COLOR_SURFACE if r_idx % 2 == 0 else COLOR_SURFACE_VARIANT + + c0 = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + c0.grid(row=r_idx, column=0, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c0, text=metric_name, font=FONT_BODY_BOLD, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=6, anchor="w") + + c1 = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + c1.grid(row=r_idx, column=1, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c1, text=val, font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=6, anchor="w") + + self.status = "success" + self.on_status_change() + + def _render_error(self, err_msg): + usage_logger.warning(f"SharePoint Site Usage fetch failed: {err_msg}") + if hasattr(self, 'reload_btn') and self.reload_btn.winfo_exists(): + self.reload_btn.configure(state="normal") + self._set_state_error(err_msg) + self.status = "error" + self.on_status_change() + + def cancel(self): + pass diff --git a/telemetry/files_telemetry.py b/telemetry/files_telemetry.py new file mode 100644 index 00000000..4d29e38e --- /dev/null +++ b/telemetry/files_telemetry.py @@ -0,0 +1,18 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Backward compatibility facade for consolidated Files telemetry.""" + +# Re-export UI container from telemetry package +from telemetry.files import FilesTelemetryFrame diff --git a/telemetry/intune/__init__.py b/telemetry/intune/__init__.py new file mode 100644 index 00000000..1d7596d6 --- /dev/null +++ b/telemetry/intune/__init__.py @@ -0,0 +1,291 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Consolidated Microsoft Intune Telemetry Orchestrator Container.""" + +import logging +import webbrowser +import customtkinter as ctk + +from telemetry.styles import * +from telemetry.intune.mobile_apps import MobileAppsSubFrame +from telemetry.intune.detected_apps import DetectedAppsSubFrame +from telemetry.intune.managed_devices import ManagedDevicesSubFrame +from telemetry.intune.vc_devices import VCDevicesSubFrame +from telemetry.intune.device_configs import DeviceConfigsSubFrame +from telemetry.intune.device_compliance import DeviceComplianceSubFrame +from telemetry.intune.mdm_policies import MdmPoliciesSubFrame +from telemetry.intune.byod_configs import ByodConfigsSubFrame + +usage_logger = logging.getLogger("M365TelemetryAsyncLogger.IntuneUI") + +class IntunePoliciesFrame(ctk.CTkFrame): + """Component for rendering Intune Policies and Detected Apps data inside modular subframes.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, **kwargs): + self.semaphore = kwargs.pop("concurrency_semaphore", None) + self.get_delegated_auth = kwargs.pop("delegated_auth_callback", None) + super().__init__(master, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=12, **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.status = None + + self.build_ui() + + def build_ui(self): + self.pack(fill="x", expand=True, pady=10) + + self.inner_pad = ctk.CTkFrame(self, fg_color="transparent") + self.inner_pad.pack(fill="both", expand=True, padx=20, pady=20) + + self.header_frame = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.header_frame.pack(fill="x", pady=(0, 10)) + + self.title_lbl = ctk.CTkLabel( + self.header_frame, + text="Microsoft Intune Data", + font=FONT_HEADER_SMALL, + text_color=COLOR_TEXT_MAIN + ) + self.title_lbl.pack(side="left", anchor="w") + + self.link_lbl = ctk.CTkLabel( + self.header_frame, + text="Open Intune Admin Center ↗", + font=FONT_BODY_BOLD, + text_color=COLOR_PRIMARY, + cursor="hand2" + ) + self.link_lbl.pack(side="left", anchor="w", padx=(15, 0)) + self.link_lbl.bind("", lambda e: webbrowser.open("https://intune.microsoft.com/#view/Microsoft_Intune_DeviceSettings/DevicesMenu/~/configuration")) + self.link_lbl.bind("", lambda e: self.link_lbl.configure(text_color=COLOR_PRIMARY_HOVER)) + self.link_lbl.bind("", lambda e: self.link_lbl.configure(text_color=COLOR_PRIMARY)) + + self.body_frame = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.body_frame.pack(fill="x", expand=True) + + # 1. Managed Mobile Apps + self.mobile_apps_view = MobileAppsSubFrame( + self.body_frame, + log_callback=self.log_msg, + credentials_callback=self.get_credentials, + status_change_callback=self._subframe_status_changed, + semaphore=self.semaphore + ) + self.mobile_apps_view.pack(fill="x", pady=(10, 15)) + + # Divider 1 + self.divider1 = ctk.CTkFrame(self.body_frame, fg_color=COLOR_OUTLINE_LIGHT, height=1) + self.divider1.pack(fill="x", pady=15) + + # 2. Detected Apps + self.detected_apps_view = DetectedAppsSubFrame( + self.body_frame, + log_callback=self.log_msg, + credentials_callback=self.get_credentials, + status_change_callback=self._subframe_status_changed, + semaphore=self.semaphore + ) + self.detected_apps_view.pack(fill="x", pady=(0, 15)) + + # Divider 2 + self.divider2 = ctk.CTkFrame(self.body_frame, fg_color=COLOR_OUTLINE_LIGHT, height=1) + self.divider2.pack(fill="x", pady=15) + + # 3. Managed Devices + self.managed_devices_view = ManagedDevicesSubFrame( + self.body_frame, + log_callback=self.log_msg, + credentials_callback=self.get_credentials, + status_change_callback=self._subframe_status_changed, + semaphore=self.semaphore + ) + self.managed_devices_view.pack(fill="x", pady=(0, 15)) + + # Divider 3 + self.divider3 = ctk.CTkFrame(self.body_frame, fg_color=COLOR_OUTLINE_LIGHT, height=1) + self.divider3.pack(fill="x", pady=15) + + # 4. VC Devices (Filtered meeting devices) + self.vc_devices_view = VCDevicesSubFrame( + self.body_frame, + log_callback=self.log_msg, + credentials_callback=self.get_credentials, + status_change_callback=self._subframe_status_changed, + semaphore=self.semaphore + ) + self.vc_devices_view.pack(fill="x", pady=(0, 15)) + + # Divider VC + self.divider_vc = ctk.CTkFrame(self.body_frame, fg_color=COLOR_OUTLINE_LIGHT, height=1) + self.divider_vc.pack(fill="x", pady=15) + + # 5. Device Configurations + self.device_configs_view = DeviceConfigsSubFrame( + self.body_frame, + log_callback=self.log_msg, + credentials_callback=self.get_credentials, + status_change_callback=self._subframe_status_changed, + semaphore=self.semaphore + ) + self.device_configs_view.pack(fill="x", pady=(0, 15)) + + # Divider 4 + self.divider4 = ctk.CTkFrame(self.body_frame, fg_color=COLOR_OUTLINE_LIGHT, height=1) + self.divider4.pack(fill="x", pady=15) + + # Section Header: Mobile Device Compliance Policies + self.compliance_header = ctk.CTkLabel( + self.body_frame, + text="Mobile Device Compliance Policies", + font=FONT_HEADER_SMALL, + text_color=COLOR_TEXT_MAIN + ) + self.compliance_header.pack(anchor="w", pady=(0, 15)) + + # 6. Android Devices Compliance SubFrame + self.android_compliance_view = DeviceComplianceSubFrame( + self.body_frame, + log_callback=self.log_msg, + credentials_callback=self.get_credentials, + status_change_callback=self._subframe_status_changed, + semaphore=self.semaphore, + device_type="Android" + ) + self.android_compliance_view.pack(fill="x", pady=(0, 15)) + + # Divider 5 + self.divider5 = ctk.CTkFrame(self.body_frame, fg_color=COLOR_OUTLINE_LIGHT, height=1) + self.divider5.pack(fill="x", pady=15) + + # 7. iOS Devices Compliance SubFrame + self.ios_compliance_view = DeviceComplianceSubFrame( + self.body_frame, + log_callback=self.log_msg, + credentials_callback=self.get_credentials, + status_change_callback=self._subframe_status_changed, + semaphore=self.semaphore, + device_type="iOS" + ) + self.ios_compliance_view.pack(fill="x", pady=(0, 15)) + + # Divider BYOD + self.divider_byod = ctk.CTkFrame(self.body_frame, fg_color=COLOR_OUTLINE_LIGHT, height=1) + self.divider_byod.pack(fill="x", pady=15) + + # 7.5 Mobile BYOD Configurations SubFrame + self.byod_configs_view = ByodConfigsSubFrame( + self.body_frame, + log_callback=self.log_msg, + credentials_callback=self.get_credentials, + status_change_callback=self._subframe_status_changed, + semaphore=self.semaphore + ) + self.byod_configs_view.pack(fill="x", pady=(0, 15)) + + # Divider 6 + self.divider6 = ctk.CTkFrame(self.body_frame, fg_color=COLOR_OUTLINE_LIGHT, height=1) + self.divider6.pack(fill="x", pady=15) + + # 8. Mobile Device Management Policies SubFrame + self.mdm_policies_view = MdmPoliciesSubFrame( + self.body_frame, + log_callback=self.log_msg, + credentials_callback=self.get_credentials, + status_change_callback=self._subframe_status_changed, + semaphore=self.semaphore, + delegated_auth_callback=self.get_delegated_auth + ) + self.mdm_policies_view.pack(fill="x", pady=(0, 15)) + + self.reset_view() + + def _subframe_status_changed(self): + statuses = [ + self.mobile_apps_view.status, + self.detected_apps_view.status, + self.managed_devices_view.status, + self.vc_devices_view.status, + self.device_configs_view.status, + self.android_compliance_view.status, + self.ios_compliance_view.status, + self.byod_configs_view.status, + self.mdm_policies_view.status + ] + if "loading" in statuses: + self.status = "loading" + elif "error" in statuses: + self.status = "error" + elif "success" in statuses: + self.status = "success" + else: + self.status = None + self.on_status_change() + + def reset_view(self): + self.pack_forget() + self.status = None + self.mobile_apps_view.reset_view() + self.detected_apps_view.reset_view() + self.managed_devices_view.reset_view() + self.vc_devices_view.reset_view() + self.device_configs_view.reset_view() + self.android_compliance_view.reset_view() + self.ios_compliance_view.reset_view() + self.byod_configs_view.reset_view() + self.mdm_policies_view.reset_view() + + def trigger_fetch(self, tenant, client_id, client_secret): + usage_logger.info("Intune trigger_fetch called.") + self.pack(fill="x", expand=True, pady=10) + use_delegated = self.get_delegated_auth() if self.get_delegated_auth else False + + self.mobile_apps_view.trigger_fetch(tenant, client_id, client_secret) + self.detected_apps_view.trigger_fetch(tenant, client_id, client_secret) + self.managed_devices_view.trigger_fetch(tenant, client_id, client_secret) + self.vc_devices_view.trigger_fetch(tenant, client_id, client_secret) + self.device_configs_view.trigger_fetch(tenant, client_id, client_secret) + self.android_compliance_view.trigger_fetch(tenant, client_id, client_secret) + self.ios_compliance_view.trigger_fetch(tenant, client_id, client_secret) + self.byod_configs_view.trigger_fetch(tenant, client_id, client_secret) + self.mdm_policies_view.trigger_fetch(tenant, client_id, client_secret, use_delegated_auth=use_delegated) + + def cancel(self): + usage_logger.info("Intune cancel called.") + self.mobile_apps_view.cancel() + self.detected_apps_view.cancel() + self.managed_devices_view.cancel() + self.vc_devices_view.cancel() + self.device_configs_view.cancel() + self.android_compliance_view.cancel() + self.ios_compliance_view.cancel() + self.byod_configs_view.cancel() + self.mdm_policies_view.cancel() + + @property + def last_data(self): + return { + "total_device_configs": getattr(self.device_configs_view, "total_device_configs", 0), + "total_config_policies": getattr(self.device_configs_view, "total_config_policies", 0), + "table_rows": self.device_configs_view.last_data, + "mobile_apps": self.mobile_apps_view.last_data, + "detected_apps": self.detected_apps_view.last_data, + "managed_devices": self.managed_devices_view.last_data, + "vc_devices": self.vc_devices_view.last_data, + "android_compliance": self.android_compliance_view.last_data, + "ios_compliance": self.ios_compliance_view.last_data, + "byod_configs": self.byod_configs_view.last_data, + "mdm_policies": self.mdm_policies_view.last_data + } diff --git a/telemetry/intune/byod_configs.py b/telemetry/intune/byod_configs.py new file mode 100644 index 00000000..687f8e73 --- /dev/null +++ b/telemetry/intune/byod_configs.py @@ -0,0 +1,303 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""UI component for Mobile BYOD Configurations telemetry.""" + +import os +import csv +import logging +import threading +import shutil +import pandas as pd +import customtkinter as ctk +import sqlite3 +import asyncio +from tkinter import filedialog, messagebox + +from core.graph.db import import_csv_to_sqlite +from core.graph.intune.byod_configs import run_byod_configs_pipeline +from telemetry.styles import * + +usage_logger = logging.getLogger("M365TelemetryAsyncLogger.ByodConfigsUI") + +class ByodConfigsSubFrame(ctk.CTkFrame): + """Sub-frame for Mobile BYOD Configurations.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, semaphore=None, **kwargs): + super().__init__(master, fg_color="transparent", **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.semaphore = semaphore + self.status = None + self.page = 0 + self.ITEMS_PER_PAGE = 5 + self.csv_path = None + self.is_cancelled = False + + self.build_ui() + + def build_ui(self): + self.header_frame = ctk.CTkFrame(self, fg_color="transparent") + self.header_frame.pack(fill="x", pady=(0, 5)) + + ctk.CTkLabel(self.header_frame, text="Mobile BYOD Configurations", font=FONT_BODY_BOLD, text_color=COLOR_TEXT_MAIN).pack(side="left") + + self.btn_reload = ctk.CTkButton( + self.header_frame, text="↻ Reload", width=80, height=24, + font=ctk.CTkFont(family="Segoe UI", size=12), + fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color="#2563EB", hover_color="#DBEAFE", + command=self.trigger_fetch_individual + ) + self.btn_reload.pack(side="right", padx=(10, 0)) + + self.btn_export = ctk.CTkButton( + self.header_frame, text="Export BYOD Configs", width=180, height=32, corner_radius=16, + font=FONT_BODY_BOLD, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, + command=self._export + ) + self.btn_export.pack(side="right") + + self.state_frame = ctk.CTkFrame(self, fg_color="transparent") + + self.summary_lbl = ctk.CTkLabel(self, text="Total Extracted: 0 BYOD Configurations", font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_SUB) + + self.grid_frame = ctk.CTkFrame(self, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + + headers = [ + "Display Name", "Description", "Priority", + "Last Modified Date Time", "iOS Restrictions", + "Windows Mobile Restrictions", "Android Restrictions" + ] + weights = [2, 3, 1, 2, 4, 4, 4] + for col_idx, (head_text, weight) in enumerate(zip(headers, weights)): + self.grid_frame.grid_columnconfigure(col_idx, weight=weight) + cell = ctk.CTkFrame(self.grid_frame, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=1, pady=1) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + # Pagination controls frame (stays fixed/centered) + self.pagination_frame = ctk.CTkFrame(self, fg_color="transparent") + + self.reset_view() + + def reset_view(self): + self.status = None + self.is_cancelled = False + self.csv_path = None + self.page = 0 + self.state_frame.pack_forget() + self.summary_lbl.pack_forget() + self.grid_frame.pack_forget() + self.pagination_frame.pack_forget() + for w in self.state_frame.winfo_children(): w.destroy() + for w in self.grid_frame.winfo_children(): + info = w.grid_info() + if "row" in info and int(info["row"]) > 0: w.destroy() + for w in self.pagination_frame.winfo_children(): w.destroy() + + def _set_loading_state(self, msg): + for w in self.state_frame.winfo_children(): w.destroy() + self.state_frame.pack(fill="x", expand=True) + ctk.CTkLabel(self.state_frame, text=f"⏳ {msg}", text_color=COLOR_TEXT_SUB, font=FONT_BODY_MEDIUM).pack(pady=(15, 5)) + pb = ctk.CTkProgressBar(self.state_frame, mode="indeterminate", width=200, fg_color=COLOR_SURFACE_VARIANT, progress_color=COLOR_PRIMARY) + pb.pack(pady=(0, 15)) + pb.start() + + def trigger_fetch_individual(self): + tenant, clients, secrets = self.get_credentials() + if tenant: + self.trigger_fetch(tenant, clients[0], secrets[0]) + + def trigger_fetch(self, tenant, client_id, client_secret): + self.status = "loading" + self.is_cancelled = False + self.page = 0 + self.btn_reload.configure(state="disabled") + self.btn_export.configure(state="disabled") + self.grid_frame.pack_forget() + self.pagination_frame.pack_forget() + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + if os.path.basename(script_dir) == "intune": + script_dir = os.path.dirname(script_dir) + reports_dir = os.path.join(script_dir, "reports", f"{tenant}_{client_id}") + self.csv_path = os.path.join(reports_dir, "intune_byod_configs.csv") + + self._set_loading_state("Scanning Mobile BYOD Configurations...") + self.on_status_change() + + threading.Thread(target=self._execute_worker, args=(tenant, client_id, client_secret), daemon=True).start() + + def _execute_worker(self, tenant, client_id, client_secret): + if self.semaphore: self.semaphore.acquire() + try: + run_byod_configs_pipeline( + client_id=client_id, + client_secret=client_secret, + tenant_id=tenant, + csv_path=self.csv_path, + is_cancelled_callback=lambda: self.is_cancelled + ) + + if not self.is_cancelled: + reports_dir = os.path.dirname(self.csv_path) + db_path = os.path.join(reports_dir, "telemetry_cache.db") + if os.path.exists(self.csv_path): + asyncio.run(import_csv_to_sqlite(self.csv_path, db_path, "byod_configs")) + + self.status = "success" + self.after(0, self._render_success) + except Exception as e: + usage_logger.error(f"BYOD configs fetch error: {e}", exc_info=True) + self.status = "error" + self.after(0, self._render_error, str(e)) + finally: + if self.semaphore: self.semaphore.release() + self.after(0, self.on_status_change) + + def _render_success(self): + self.btn_reload.configure(state="normal") + self.btn_export.configure(state="normal") + self.state_frame.pack_forget() + self.summary_lbl.pack(anchor="w", padx=10, pady=5) + self.grid_frame.pack(fill="x") + self.pagination_frame.pack(fill="x", pady=(5, 10)) + + self._update_grid() + + def _render_error(self, err): + self.btn_reload.configure(state="normal") + self.grid_frame.pack_forget() + self.pagination_frame.pack_forget() + self.summary_lbl.pack_forget() + for w in self.state_frame.winfo_children(): w.destroy() + self.state_frame.pack(fill="x", expand=True) + ctk.CTkLabel(self.state_frame, text=f"⏳ {err}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center", wraplength=700).pack(pady=(15, 5)) + ctk.CTkButton(self.state_frame, text="Try Again", command=self.trigger_fetch_individual, width=100, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 15)) + + def _load_page(self): + if not self.csv_path or not os.path.exists(self.csv_path): return [], 0 + + reports_dir = os.path.dirname(self.csv_path) + db_path = os.path.join(reports_dir, "telemetry_cache.db") + if not os.path.exists(db_path): return [], 0 + + try: + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + cursor = conn.cursor() + + cursor.execute("SELECT COUNT(*) FROM byod_configs") + total = cursor.fetchone()[0] + + offset = self.page * self.ITEMS_PER_PAGE + cursor.execute("SELECT * FROM byod_configs LIMIT ? OFFSET ?", (self.ITEMS_PER_PAGE, offset)) + rows = cursor.fetchall() + + conn.close() + return rows, total + except Exception as e: + usage_logger.error(f"Error reading SQLite byod_configs: {e}") + return [], 0 + + def _update_grid(self): + for w in self.grid_frame.winfo_children(): + info = w.grid_info() + if "row" in info and int(info["row"]) > 0: w.destroy() + + page_data, total_count = self._load_page() + total_pages = (total_count - 1) // self.ITEMS_PER_PAGE + 1 if total_count > 0 else 1 + + self.summary_lbl.configure(text=f"Total Extracted: {total_count} BYOD Configurations") + + for row_idx, item in enumerate(page_data, start=1): + bg_style = COLOR_SURFACE if row_idx % 2 == 0 else COLOR_SURFACE_VARIANT + vals = [ + item.get("displayName") or "N/A", + item.get("description") or "N/A", + str(item.get("priority") if item.get("priority") is not None else 0), + item.get("lastModifiedDateTime") or "N/A", + item.get("iosRestrictions") or "N/A", + item.get("windowsMobileRestrictions") or "N/A", + item.get("androidRestrictions") or "N/A" + ] + for col_idx, val in enumerate(vals): + cell = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + cell.grid(row=row_idx, column=col_idx, sticky="nsew", padx=1, pady=1) + lbl = ctk.CTkLabel(cell, text=str(val), font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN, wraplength=180, justify="left", anchor="w") + lbl.pack(padx=10, pady=8, fill="x") + + # Re-build pagination controls + for w in self.pagination_frame.winfo_children(): w.destroy() + + center_container = ctk.CTkFrame(self.pagination_frame, fg_color="transparent") + center_container.pack(pady=5) + + prev_state = "normal" if self.page > 0 else "disabled" + ctk.CTkButton( + center_container, text="◀ Prev", width=70, height=22, corner_radius=6, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, state=prev_state, + command=lambda: self._change_page(-1) + ).pack(side="left", padx=5) + + ctk.CTkLabel(center_container, text=f"Page {self.page + 1} of {total_pages}", font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_SUB).pack(side="left", padx=15) + + next_state = "normal" if self.page < total_pages - 1 else "disabled" + ctk.CTkButton( + center_container, text="Next ▶", width=70, height=22, corner_radius=6, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, state=next_state, + command=lambda: self._change_page(1) + ).pack(side="left", padx=5) + + def _change_page(self, delta): + self.page += delta + self._update_grid() + + def _export(self): + if not self.csv_path or not os.path.exists(self.csv_path): + messagebox.showerror("Export Error", "No data available to export yet.") + return + + dest_path = filedialog.asksaveasfilename( + defaultextension=".csv", + filetypes=[("CSV files", "*.csv"), ("All files", "*.*")], + title="Export BYOD Configs CSV", + initialfile="M365_Intune_BYOD_Configs.csv", + parent=self + ) + if not dest_path: return + + try: + shutil.copyfile(self.csv_path, dest_path) + messagebox.showinfo("Export Successful", f"BYOD Configurations successfully exported to:\n{dest_path}") + except Exception as e: + messagebox.showerror("Export Failed", f"Failed to save CSV file: {e}") + + @property + def last_data(self): + if not self.csv_path or not os.path.exists(self.csv_path): return [] + try: + df = pd.read_csv(self.csv_path) + return df.head(200).fillna("N/A").to_dict('records') + except Exception: + return [] + + def cancel(self): + self.is_cancelled = True + self.status = None diff --git a/telemetry/intune/detected_apps.py b/telemetry/intune/detected_apps.py new file mode 100644 index 00000000..0beaf38a --- /dev/null +++ b/telemetry/intune/detected_apps.py @@ -0,0 +1,261 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""UI component for Detected Apps telemetry.""" + +import os +import time +import logging +import threading +import pandas as pd +import customtkinter as ctk +from tkinter import filedialog, messagebox + +from core.graph.intune.detected_apps import run_detected_apps_pipeline +from telemetry.styles import * + +usage_logger = logging.getLogger("M365TelemetryAsyncLogger.DetectedAppsUI") + +class DetectedAppsSubFrame(ctk.CTkFrame): + """Sub-frame for Detected Apps.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, semaphore=None, **kwargs): + super().__init__(master, fg_color="transparent", **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.semaphore = semaphore + self.status = None + self.page = 0 + self.ITEMS_PER_PAGE = 5 + self.csv_path = None + self.is_cancelled = False + + self.build_ui() + + def build_ui(self): + self.header_frame = ctk.CTkFrame(self, fg_color="transparent") + self.header_frame.pack(fill="x", pady=(0, 5)) + + ctk.CTkLabel(self.header_frame, text="Detected Apps", font=FONT_BODY_BOLD, text_color=COLOR_TEXT_MAIN).pack(side="left") + + self.btn_reload = ctk.CTkButton( + self.header_frame, text="↻ Reload", width=80, height=24, + font=ctk.CTkFont(family="Segoe UI", size=12), + fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color="#2563EB", hover_color="#DBEAFE", + command=self.trigger_fetch_individual + ) + self.btn_reload.pack(side="right", padx=(10, 0)) + + self.btn_export = ctk.CTkButton( + self.header_frame, text="Export Detected Apps", width=180, height=32, corner_radius=16, + font=FONT_BODY_BOLD, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, + command=self._export + ) + self.btn_export.pack(side="right") + + self.state_frame = ctk.CTkFrame(self, fg_color="transparent") + + self.grid_frame = ctk.CTkFrame(self, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + + headers = ["App Name", "Version", "Publisher", "Platform"] + for i in range(4): + self.grid_frame.grid_columnconfigure(i, weight=2 if i == 0 else 1) + + for col_idx, head_text in enumerate(headers): + cell = ctk.CTkFrame(self.grid_frame, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=1, pady=1) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + self.reset_view() + + def reset_view(self): + self.status = None + self.is_cancelled = False + self.csv_path = None + self.page = 0 + self.state_frame.pack_forget() + self.grid_frame.pack_forget() + for w in self.state_frame.winfo_children(): w.destroy() + for w in self.grid_frame.winfo_children(): + info = w.grid_info() + if "row" in info and int(info["row"]) > 0: w.destroy() + + def _set_loading_state(self, msg): + for w in self.state_frame.winfo_children(): w.destroy() + self.state_frame.pack(fill="x", expand=True) + ctk.CTkLabel(self.state_frame, text=f"⏳ {msg}", text_color=COLOR_TEXT_SUB, font=FONT_BODY_MEDIUM).pack(pady=(15, 5)) + pb = ctk.CTkProgressBar(self.state_frame, mode="indeterminate", width=200, fg_color=COLOR_SURFACE_VARIANT, progress_color=COLOR_PRIMARY) + pb.pack(pady=(0, 15)) + pb.start() + + def trigger_fetch_individual(self): + tenant, clients, secrets = self.get_credentials() + if tenant: + self.trigger_fetch(tenant, clients[0], secrets[0]) + + def trigger_fetch(self, tenant, client_id, client_secret): + self.status = "loading" + self.is_cancelled = False + self.page = 0 + self.btn_reload.configure(state="disabled") + self.btn_export.configure(state="disabled") + self.grid_frame.pack_forget() + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + if os.path.basename(script_dir) == "intune": + script_dir = os.path.dirname(script_dir) + reports_dir = os.path.join(script_dir, "reports", f"{tenant}_{client_id}") + self.csv_path = os.path.join(reports_dir, "intune_detected_apps.csv") + + self._set_loading_state("Scanning Detected Apps (v1.0)...") + self.on_status_change() + + threading.Thread(target=self._execute_worker, args=(tenant, client_id, client_secret), daemon=True).start() + + def _execute_worker(self, tenant, client_id, client_secret): + if self.semaphore: self.semaphore.acquire() + try: + run_detected_apps_pipeline( + client_id=client_id, + client_secret=client_secret, + tenant_id=tenant, + csv_path=self.csv_path, + max_rows=10000, + is_cancelled_callback=lambda: self.is_cancelled + ) + self.status = "success" + self.after(0, self._render_success) + except Exception as e: + usage_logger.error(f"Detected apps fetch error: {e}", exc_info=True) + self.status = "error" + self.after(0, self._render_error, str(e)) + finally: + if self.semaphore: self.semaphore.release() + self.after(0, self.on_status_change) + + def _render_success(self): + self.btn_reload.configure(state="normal") + self.btn_export.configure(state="normal") + self.state_frame.pack_forget() + self.grid_frame.pack(fill="x") + + self._update_grid() + + def _render_error(self, err): + self.btn_reload.configure(state="normal") + self.grid_frame.pack_forget() + for w in self.state_frame.winfo_children(): w.destroy() + self.state_frame.pack(fill="x", expand=True) + ctk.CTkLabel(self.state_frame, text=f"✖ {err}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center", wraplength=700).pack(pady=(15, 5)) + ctk.CTkButton(self.state_frame, text="Try Again", command=self.trigger_fetch_individual, width=100, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 15)) + + def _load_page(self): + if not self.csv_path or not os.path.exists(self.csv_path): return [], 0 + try: + df = pd.read_csv(self.csv_path).fillna("N/A") + total = len(df) + start = self.page * self.ITEMS_PER_PAGE + end = start + self.ITEMS_PER_PAGE + return df.iloc[start:end].to_dict('records'), total + except Exception as e: + usage_logger.error(f"Error reading CSV {self.csv_path}: {e}") + return [], 0 + + def _update_grid(self): + for w in self.grid_frame.winfo_children(): + info = w.grid_info() + if "row" in info and int(info["row"]) > 0: w.destroy() + + page_data, total_count = self._load_page() + total_pages = (total_count - 1) // self.ITEMS_PER_PAGE + 1 if total_count > 0 else 1 + + for row_idx, item in enumerate(page_data, start=1): + bg_style = COLOR_SURFACE if row_idx % 2 == 0 else COLOR_SURFACE_VARIANT + vals = [ + item.get("displayName", "N/A"), + item.get("version", "N/A"), + item.get("publisher", "N/A"), + item.get("platform", "unknown") + ] + for col_idx, val in enumerate(vals): + cell = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + cell.grid(row=row_idx, column=col_idx, sticky="nsew", padx=1, pady=1) + lbl = ctk.CTkLabel(cell, text=str(val), font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN, wraplength=200, justify="left", anchor="w") + lbl.pack(padx=10, pady=8, fill="x") + + # Pagination controls row + control_frame = ctk.CTkFrame(self.grid_frame, fg_color="transparent") + control_frame.grid(row=self.ITEMS_PER_PAGE + 1, column=0, columnspan=4, pady=0, sticky="ew") + + center_container = ctk.CTkFrame(control_frame, fg_color="transparent") + center_container.pack(pady=(5, 10)) + + prev_state = "normal" if self.page > 0 else "disabled" + ctk.CTkButton( + center_container, text="◀ Prev", width=70, height=22, corner_radius=6, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, state=prev_state, + command=lambda: self._change_page(-1) + ).pack(side="left", padx=5) + + ctk.CTkLabel(center_container, text=f"Page {self.page + 1} of {total_pages}", font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_SUB).pack(side="left", padx=15) + + next_state = "normal" if self.page < total_pages - 1 else "disabled" + ctk.CTkButton( + center_container, text="Next ▶", width=70, height=22, corner_radius=6, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, state=next_state, + command=lambda: self._change_page(1) + ).pack(side="left", padx=5) + + def _change_page(self, delta): + self.page += delta + self._update_grid() + + def _export(self): + if not self.csv_path or not os.path.exists(self.csv_path): + messagebox.showerror("Export Error", "No data available to export yet.") + return + + dest_path = filedialog.asksaveasfilename( + defaultextension=".csv", + filetypes=[("CSV files", "*.csv"), ("All files", "*.*")], + title="Export Detected Apps CSV", + initialfile="M365_Intune_Detected_Apps.csv", + parent=self + ) + if not dest_path: return + + try: + df = pd.read_csv(self.csv_path) + df.fillna("N/A").to_csv(dest_path, index=False, encoding="utf-8-sig") + messagebox.showinfo("Export Successful", f"Detected Apps successfully exported to:\n{dest_path}") + except Exception as e: + messagebox.showerror("Export Failed", f"Failed to save CSV file: {e}") + + @property + def last_data(self): + if not self.csv_path or not os.path.exists(self.csv_path): return [] + try: + df = pd.read_csv(self.csv_path) + return df.head(200).fillna("N/A").to_dict('records') + except Exception: + return [] + + def cancel(self): + self.is_cancelled = True + self.status = None diff --git a/telemetry/intune/device_compliance.py b/telemetry/intune/device_compliance.py new file mode 100644 index 00000000..aea1bae1 --- /dev/null +++ b/telemetry/intune/device_compliance.py @@ -0,0 +1,299 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""UI component for Device Compliance Policies telemetry.""" + +import os +import csv +import logging +import threading +import shutil +import pandas as pd +import customtkinter as ctk +import sqlite3 +import asyncio +from tkinter import filedialog, messagebox + +from core.graph.db import import_csv_to_sqlite +from core.graph.intune.device_compliance import run_device_compliance_pipeline +from telemetry.styles import * + +usage_logger = logging.getLogger("M365TelemetryAsyncLogger.DeviceComplianceUI") + +class DeviceComplianceSubFrame(ctk.CTkFrame): + """Sub-frame for Device Compliance Policies (Android/iOS).""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, semaphore=None, device_type="Android", **kwargs): + self.device_type = device_type # "Android" or "iOS" + super().__init__(master, fg_color="transparent", **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.semaphore = semaphore + self.status = None + self.page = 0 + self.ITEMS_PER_PAGE = 5 + self.csv_path = None + self.is_cancelled = False + + self.build_ui() + + def build_ui(self): + self.header_frame = ctk.CTkFrame(self, fg_color="transparent") + self.header_frame.pack(fill="x", pady=(0, 5)) + + ctk.CTkLabel(self.header_frame, text=f"{self.device_type} Devices", font=FONT_BODY_BOLD, text_color=COLOR_TEXT_MAIN).pack(side="left") + + self.btn_reload = ctk.CTkButton( + self.header_frame, text="↻ Reload", width=80, height=24, + font=ctk.CTkFont(family="Segoe UI", size=12), + fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color="#2563EB", hover_color="#DBEAFE", + command=self.trigger_fetch_individual + ) + self.btn_reload.pack(side="right", padx=(10, 0)) + + self.btn_export = ctk.CTkButton( + self.header_frame, text=f"Export {self.device_type} Policies", width=220, height=32, corner_radius=16, + font=FONT_BODY_BOLD, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, + command=self._export + ) + self.btn_export.pack(side="right") + + self.state_frame = ctk.CTkFrame(self, fg_color="transparent") + + self.summary_lbl = ctk.CTkLabel(self, text=f"Total Extracted: 0 {self.device_type} Compliance Policies", font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_SUB) + + self.grid_frame = ctk.CTkFrame(self, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + + headers = ["Display Name", "Description", "Created Time", "Last Modified", "Version"] + weights = [2, 3, 2, 2, 1] + for col_idx, (head_text, weight) in enumerate(zip(headers, weights)): + self.grid_frame.grid_columnconfigure(col_idx, weight=weight) + cell = ctk.CTkFrame(self.grid_frame, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=1, pady=1) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + # Pagination controls frame (stays fixed/centered) + self.pagination_frame = ctk.CTkFrame(self, fg_color="transparent") + + self.reset_view() + + def reset_view(self): + self.status = None + self.is_cancelled = False + self.csv_path = None + self.page = 0 + self.state_frame.pack_forget() + self.summary_lbl.pack_forget() + self.grid_frame.pack_forget() + self.pagination_frame.pack_forget() + for w in self.state_frame.winfo_children(): w.destroy() + for w in self.grid_frame.winfo_children(): + info = w.grid_info() + if "row" in info and int(info["row"]) > 0: w.destroy() + for w in self.pagination_frame.winfo_children(): w.destroy() + + def _set_loading_state(self, msg): + for w in self.state_frame.winfo_children(): w.destroy() + self.state_frame.pack(fill="x", expand=True) + ctk.CTkLabel(self.state_frame, text=f"⏳ {msg}", text_color=COLOR_TEXT_SUB, font=FONT_BODY_MEDIUM).pack(pady=(15, 5)) + pb = ctk.CTkProgressBar(self.state_frame, mode="indeterminate", width=200, fg_color=COLOR_SURFACE_VARIANT, progress_color=COLOR_PRIMARY) + pb.pack(pady=(0, 15)) + pb.start() + + def trigger_fetch_individual(self): + tenant, clients, secrets = self.get_credentials() + if tenant: + self.trigger_fetch(tenant, clients[0], secrets[0]) + + def trigger_fetch(self, tenant, client_id, client_secret): + self.status = "loading" + self.is_cancelled = False + self.page = 0 + self.btn_reload.configure(state="disabled") + self.btn_export.configure(state="disabled") + self.grid_frame.pack_forget() + self.pagination_frame.pack_forget() + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + if os.path.basename(script_dir) == "intune": + script_dir = os.path.dirname(script_dir) + reports_dir = os.path.join(script_dir, "reports", f"{tenant}_{client_id}") + self.csv_path = os.path.join(reports_dir, f"intune_{self.device_type.lower()}_compliance.csv") + + self._set_loading_state(f"Scanning {self.device_type} Compliance Policies...") + self.on_status_change() + + threading.Thread(target=self._execute_worker, args=(tenant, client_id, client_secret), daemon=True).start() + + def _execute_worker(self, tenant, client_id, client_secret): + if self.semaphore: self.semaphore.acquire() + try: + run_device_compliance_pipeline( + client_id=client_id, + client_secret=client_secret, + tenant_id=tenant, + csv_path=self.csv_path, + filter_type=f"microsoft.graph.{self.device_type.lower()}CompliancePolicy", + is_cancelled_callback=lambda: self.is_cancelled + ) + + if not self.is_cancelled: + reports_dir = os.path.dirname(self.csv_path) + db_path = os.path.join(reports_dir, "telemetry_cache.db") + if os.path.exists(self.csv_path): + asyncio.run(import_csv_to_sqlite(self.csv_path, db_path, f"{self.device_type.lower()}_compliance")) + + self.status = "success" + self.after(0, self._render_success) + except Exception as e: + usage_logger.error(f"{self.device_type} compliance fetch error: {e}", exc_info=True) + self.status = "error" + self.after(0, self._render_error, str(e)) + finally: + if self.semaphore: self.semaphore.release() + self.after(0, self.on_status_change) + + def _render_success(self): + self.btn_reload.configure(state="normal") + self.btn_export.configure(state="normal") + self.state_frame.pack_forget() + self.summary_lbl.pack(anchor="w", padx=10, pady=5) + self.grid_frame.pack(fill="x") + self.pagination_frame.pack(fill="x", pady=(5, 10)) + + self._update_grid() + + def _render_error(self, err): + self.btn_reload.configure(state="normal") + self.grid_frame.pack_forget() + self.pagination_frame.pack_forget() + self.summary_lbl.pack_forget() + for w in self.state_frame.winfo_children(): w.destroy() + self.state_frame.pack(fill="x", expand=True) + ctk.CTkLabel(self.state_frame, text=f"✖ {err}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center", wraplength=700).pack(pady=(15, 5)) + ctk.CTkButton(self.state_frame, text="Try Again", command=self.trigger_fetch_individual, width=100, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 15)) + + def _load_page(self): + if not self.csv_path or not os.path.exists(self.csv_path): return [], 0 + + reports_dir = os.path.dirname(self.csv_path) + db_path = os.path.join(reports_dir, "telemetry_cache.db") + if not os.path.exists(db_path): return [], 0 + + try: + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + cursor = conn.cursor() + + cursor.execute(f"SELECT COUNT(*) FROM {self.device_type.lower()}_compliance") + total = cursor.fetchone()[0] + + offset = self.page * self.ITEMS_PER_PAGE + cursor.execute(f"SELECT * FROM {self.device_type.lower()}_compliance LIMIT ? OFFSET ?", (self.ITEMS_PER_PAGE, offset)) + rows = cursor.fetchall() + + conn.close() + return rows, total + except Exception as e: + usage_logger.error(f"Error reading SQLite {self.device_type.lower()}_compliance: {e}") + return [], 0 + + def _update_grid(self): + for w in self.grid_frame.winfo_children(): + info = w.grid_info() + if "row" in info and int(info["row"]) > 0: w.destroy() + + page_data, total_count = self._load_page() + total_pages = (total_count - 1) // self.ITEMS_PER_PAGE + 1 if total_count > 0 else 1 + + self.summary_lbl.configure(text=f"Total Extracted: {total_count} {self.device_type} Compliance Policies") + + for row_idx, item in enumerate(page_data, start=1): + bg_style = COLOR_SURFACE if row_idx % 2 == 0 else COLOR_SURFACE_VARIANT + vals = [ + item.get("displayName") or "N/A", + item.get("description") or "N/A", + item.get("createdDateTime") or "N/A", + item.get("lastModifiedDateTime") or "N/A", + item.get("version") or 0 + ] + for col_idx, val in enumerate(vals): + cell = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + cell.grid(row=row_idx, column=col_idx, sticky="nsew", padx=1, pady=1) + lbl = ctk.CTkLabel(cell, text=str(val), font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN, wraplength=280, justify="left", anchor="w") + lbl.pack(padx=10, pady=8, fill="x") + + # Re-build pagination controls + for w in self.pagination_frame.winfo_children(): w.destroy() + + center_container = ctk.CTkFrame(self.pagination_frame, fg_color="transparent") + center_container.pack(pady=5) + + prev_state = "normal" if self.page > 0 else "disabled" + ctk.CTkButton( + center_container, text="◀ Prev", width=70, height=22, corner_radius=6, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, state=prev_state, + command=lambda: self._change_page(-1) + ).pack(side="left", padx=5) + + ctk.CTkLabel(center_container, text=f"Page {self.page + 1} of {total_pages}", font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_SUB).pack(side="left", padx=15) + + next_state = "normal" if self.page < total_pages - 1 else "disabled" + ctk.CTkButton( + center_container, text="Next ▶", width=70, height=22, corner_radius=6, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, state=next_state, + command=lambda: self._change_page(1) + ).pack(side="left", padx=5) + + def _change_page(self, delta): + self.page += delta + self._update_grid() + + def _export(self): + if not self.csv_path or not os.path.exists(self.csv_path): + messagebox.showerror("Export Error", "No data available to export yet.") + return + + dest_path = filedialog.asksaveasfilename( + defaultextension=".csv", + filetypes=[("CSV files", "*.csv"), ("All files", "*.*")], + title=f"Export {self.device_type} Compliance Policies CSV", + initialfile=f"M365_Intune_{self.device_type}_Compliance_Policies.csv", + parent=self + ) + if not dest_path: return + + try: + shutil.copyfile(self.csv_path, dest_path) + messagebox.showinfo("Export Successful", f"{self.device_type} Compliance Policies successfully exported to:\n{dest_path}") + except Exception as e: + messagebox.showerror("Export Failed", f"Failed to save CSV file: {e}") + + @property + def last_data(self): + if not self.csv_path or not os.path.exists(self.csv_path): return [] + try: + df = pd.read_csv(self.csv_path) + return df.head(200).fillna("N/A").to_dict('records') + except Exception: + return [] + + def cancel(self): + self.is_cancelled = True + self.status = None diff --git a/telemetry/intune/device_configs.py b/telemetry/intune/device_configs.py new file mode 100644 index 00000000..882b4a07 --- /dev/null +++ b/telemetry/intune/device_configs.py @@ -0,0 +1,343 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""UI component for Device Configurations telemetry.""" + +import os +import csv +import time +import logging +import threading +from collections import defaultdict +import customtkinter as ctk +from tkinter import filedialog, messagebox +import asyncio +import sqlite3 +import pandas as pd + +from core.graph.db import import_csv_to_sqlite +from core.graph.intune.device_configs import run_device_configs_pipeline +from telemetry.styles import * + +usage_logger = logging.getLogger("M365TelemetryAsyncLogger.DeviceConfigsUI") + +class DeviceConfigsSubFrame(ctk.CTkFrame): + """Sub-frame for Device Configurations & Policies.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, semaphore=None, **kwargs): + super().__init__(master, fg_color="transparent", **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.semaphore = semaphore + self.status = None + self.page = 0 + self.ITEMS_PER_PAGE = 5 + self.csv_path_configs = None + self.csv_path_policies = None + self.is_cancelled = False + + self.build_ui() + + def build_ui(self): + self.header_frame = ctk.CTkFrame(self, fg_color="transparent") + self.header_frame.pack(fill="x", pady=(0, 5)) + + ctk.CTkLabel(self.header_frame, text="Device Configurations", font=FONT_BODY_BOLD, text_color=COLOR_TEXT_MAIN).pack(side="left") + + self.btn_reload = ctk.CTkButton( + self.header_frame, text="↻ Reload", width=80, height=24, + font=ctk.CTkFont(family="Segoe UI", size=12), + fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color="#2563EB", hover_color="#DBEAFE", + command=self.trigger_fetch_individual + ) + self.btn_reload.pack(side="right", padx=(10, 0)) + + self.btn_export = ctk.CTkButton( + self.header_frame, text="Export configurations", width=180, height=32, corner_radius=16, + font=FONT_BODY_BOLD, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, + command=self._export + ) + self.btn_export.pack(side="right") + + self.state_frame = ctk.CTkFrame(self, fg_color="transparent") + + self.summary_lbl = ctk.CTkLabel(self, text="Total Extracted: 0 Device Configurations | 0 Configuration Policies", font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_SUB) + + self.grid_frame = ctk.CTkFrame(self, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + + headers = ["Platform", "Policy Type", "Number of Policies"] + for i in range(3): + self.grid_frame.grid_columnconfigure(i, weight=1 if i == 2 else 2) + + for col_idx, head_text in enumerate(headers): + cell = ctk.CTkFrame(self.grid_frame, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=1, pady=1) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + self.reset_view() + + def reset_view(self): + self.status = None + self.is_cancelled = False + self.csv_path_configs = None + self.csv_path_policies = None + self.page = 0 + self.state_frame.pack_forget() + self.summary_lbl.pack_forget() + self.grid_frame.pack_forget() + for w in self.state_frame.winfo_children(): w.destroy() + for w in self.grid_frame.winfo_children(): + info = w.grid_info() + if "row" in info and int(info["row"]) > 0: w.destroy() + + def _set_loading_state(self, msg): + for w in self.state_frame.winfo_children(): w.destroy() + self.state_frame.pack(fill="x", expand=True) + ctk.CTkLabel(self.state_frame, text=f"⏳ {msg}", text_color=COLOR_TEXT_SUB, font=FONT_BODY_MEDIUM).pack(pady=(15, 5)) + pb = ctk.CTkProgressBar(self.state_frame, mode="indeterminate", width=200, fg_color=COLOR_SURFACE_VARIANT, progress_color=COLOR_PRIMARY) + pb.pack(pady=(0, 15)) + pb.start() + + def trigger_fetch_individual(self): + tenant, clients, secrets = self.get_credentials() + if tenant: + self.trigger_fetch(tenant, clients[0], secrets[0]) + + def trigger_fetch(self, tenant, client_id, client_secret): + self.status = "loading" + self.is_cancelled = False + self.page = 0 + self.btn_reload.configure(state="disabled") + self.btn_export.configure(state="disabled") + self.grid_frame.pack_forget() + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + if os.path.basename(script_dir) == "intune": + script_dir = os.path.dirname(script_dir) + reports_dir = os.path.join(script_dir, "reports", f"{tenant}_{client_id}") + self.csv_path_configs = os.path.join(reports_dir, "intune_device_configs.csv") + self.csv_path_policies = os.path.join(reports_dir, "intune_config_policies.csv") + + self._set_loading_state("Scanning Intune device configurations & policies...") + self.on_status_change() + + threading.Thread(target=self._execute_worker, args=(tenant, client_id, client_secret), daemon=True).start() + + def _execute_worker(self, tenant, client_id, client_secret): + if self.semaphore: self.semaphore.acquire() + try: + temp_path_configs = self.csv_path_configs + ".tmp" + temp_path_policies = self.csv_path_policies + ".tmp" + + with open(temp_path_configs, 'w', encoding='utf-8', newline='') as f: + writer = csv.writer(f) + writer.writerow(["displayName", "platform", "policyType"]) + with open(temp_path_policies, 'w', encoding='utf-8', newline='') as f: + writer = csv.writer(f) + writer.writerow(["displayName", "platform", "policyType"]) + + # Run in parallel internally or sequentially + run_device_configs_pipeline( + client_id=client_id, + client_secret=client_secret, + tenant_id=tenant, + endpoint_name="deviceConfigurations", + csv_path=temp_path_configs, + is_cancelled_callback=lambda: self.is_cancelled + ) + run_device_configs_pipeline( + client_id=client_id, + client_secret=client_secret, + tenant_id=tenant, + endpoint_name="configurationPolicies", + csv_path=temp_path_policies, + is_cancelled_callback=lambda: self.is_cancelled + ) + + if not self.is_cancelled: + if os.path.exists(temp_path_configs): + if os.path.exists(self.csv_path_configs): os.remove(self.csv_path_configs) + os.rename(temp_path_configs, self.csv_path_configs) + if os.path.exists(temp_path_policies): + if os.path.exists(self.csv_path_policies): os.remove(self.csv_path_policies) + os.rename(temp_path_policies, self.csv_path_policies) + + reports_dir = os.path.dirname(self.csv_path_configs) + db_path = os.path.join(reports_dir, "telemetry_cache.db") + if os.path.exists(self.csv_path_configs): + asyncio.run(import_csv_to_sqlite(self.csv_path_configs, db_path, "device_configs")) + if os.path.exists(self.csv_path_policies): + asyncio.run(import_csv_to_sqlite(self.csv_path_policies, db_path, "device_policies")) + + self.status = "success" + self.after(0, self._render_success) + except Exception as e: + usage_logger.error(f"Device configs fetch error: {e}", exc_info=True) + self.status = "error" + self.after(0, self._render_error, str(e)) + finally: + if 'temp_path_configs' in locals() and os.path.exists(temp_path_configs): + try: os.remove(temp_path_configs) + except Exception: pass + if 'temp_path_policies' in locals() and os.path.exists(temp_path_policies): + try: os.remove(temp_path_policies) + except Exception: pass + if self.semaphore: self.semaphore.release() + self.after(0, self.on_status_change) + + def _render_success(self): + self.btn_reload.configure(state="normal") + self.btn_export.configure(state="normal") + self.state_frame.pack_forget() + self.summary_lbl.pack(anchor="w", padx=10, pady=5) + self.grid_frame.pack(fill="x") + + self._update_grid() + + def _render_error(self, err): + self.btn_reload.configure(state="normal") + self.grid_frame.pack_forget() + self.summary_lbl.pack_forget() + for w in self.state_frame.winfo_children(): w.destroy() + self.state_frame.pack(fill="x", expand=True) + ctk.CTkLabel(self.state_frame, text=f"✖ {err}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center", wraplength=700).pack(pady=(15, 5)) + ctk.CTkButton(self.state_frame, text="Try Again", command=self.trigger_fetch_individual, width=100, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 15)) + + def _load_page(self): + if not self.csv_path_configs or not os.path.exists(self.csv_path_configs): return [], 0, 0, 0 + + counts = defaultdict(int) + total_dc = 0 + total_cp = 0 + + reports_dir = os.path.dirname(self.csv_path_configs) + db_path = os.path.join(reports_dir, "telemetry_cache.db") + if not os.path.exists(db_path): return [], 0, 0, 0 + + try: + conn = sqlite3.connect(db_path) + cursor = conn.cursor() + + cursor.execute("SELECT platform, policyType FROM device_configs") + for row in cursor.fetchall(): + if len(row) >= 2: + plat, p_type = row[0], row[1] + if plat and p_type: + counts[(plat, p_type)] += 1 + total_dc += 1 + + try: + cursor.execute("SELECT platform, policyType FROM device_policies") + for row in cursor.fetchall(): + if len(row) >= 2: + plat, p_type = row[0], row[1] + if plat and p_type: + counts[(plat, p_type)] += 1 + total_cp += 1 + except sqlite3.OperationalError: + pass + + conn.close() + except Exception as e: + usage_logger.error(f"Error loading device configurations DB: {e}") + + rows_data = [] + for (platform, p_type), count in sorted(counts.items()): + rows_data.append((platform, p_type, str(count))) + + total = len(rows_data) + start = self.page * self.ITEMS_PER_PAGE + end = start + self.ITEMS_PER_PAGE + + return rows_data[start:end], total, total_dc, total_cp + + def _update_grid(self): + for w in self.grid_frame.winfo_children(): + info = w.grid_info() + if "row" in info and int(info["row"]) > 0: w.destroy() + + page_data, total_count, total_dc, total_cp = self._load_page() + total_pages = (total_count - 1) // self.ITEMS_PER_PAGE + 1 if total_count > 0 else 1 + + self.summary_lbl.configure(text=f"Total Extracted: {total_dc} Device Configurations | {total_cp} Configuration Policies") + + for row_idx, (platform, p_type, count) in enumerate(page_data, start=1): + bg_style = COLOR_SURFACE if row_idx % 2 == 0 else COLOR_SURFACE_VARIANT + vals = [platform, p_type, count] + for col_idx, val in enumerate(vals): + cell = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + cell.grid(row=row_idx, column=col_idx, sticky="nsew", padx=1, pady=1) + lbl = ctk.CTkLabel(cell, text=str(val), font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN, wraplength=450, justify="left", anchor="w") + lbl.pack(padx=10, pady=8, fill="x") + + # Pagination controls row + control_frame = ctk.CTkFrame(self.grid_frame, fg_color="transparent") + control_frame.grid(row=self.ITEMS_PER_PAGE + 1, column=0, columnspan=3, pady=0, sticky="ew") + + center_container = ctk.CTkFrame(control_frame, fg_color="transparent") + center_container.pack(pady=(5, 10)) + + prev_state = "normal" if self.page > 0 else "disabled" + ctk.CTkButton( + center_container, text="◀ Prev", width=70, height=22, corner_radius=6, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, state=prev_state, + command=lambda: self._change_page(-1) + ).pack(side="left", padx=5) + + ctk.CTkLabel(center_container, text=f"Page {self.page + 1} of {total_pages}", font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_SUB).pack(side="left", padx=15) + + next_state = "normal" if self.page < total_pages - 1 else "disabled" + ctk.CTkButton( + center_container, text="Next ▶", width=70, height=22, corner_radius=6, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, state=next_state, + command=lambda: self._change_page(1) + ).pack(side="left", padx=5) + + def _change_page(self, delta): + self.page += delta + self._update_grid() + + def _export(self): + if not self.csv_path_configs or not os.path.exists(self.csv_path_configs): + messagebox.showerror("Export Error", "No data available to export yet.") + return + + dest_path = filedialog.asksaveasfilename( + defaultextension=".csv", + filetypes=[("CSV files", "*.csv"), ("All files", "*.*")], + title="Export Device Configurations CSV", + initialfile="M365_Intune_Device_Configurations.csv", + parent=self + ) + if not dest_path: return + + try: + df = pd.read_csv(self.csv_path_configs) + df.fillna("N/A").to_csv(dest_path, index=False, encoding="utf-8-sig") + messagebox.showinfo("Export Successful", f"Device Configurations successfully exported to:\n{dest_path}") + except Exception as e: + messagebox.showerror("Export Failed", f"Failed to save CSV file: {e}") + + @property + def last_data(self): + return self._load_page()[0] + + def cancel(self): + self.is_cancelled = True + self.status = None diff --git a/telemetry/intune/managed_devices.py b/telemetry/intune/managed_devices.py new file mode 100644 index 00000000..9394a852 --- /dev/null +++ b/telemetry/intune/managed_devices.py @@ -0,0 +1,299 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""UI component for Managed Devices telemetry.""" + +import os +import csv +import logging +import threading +import pandas as pd +import customtkinter as ctk +import sqlite3 +import asyncio +from tkinter import filedialog, messagebox + +from core.graph.db import import_csv_to_sqlite +from core.graph.intune.managed_devices import run_managed_devices_pipeline +from telemetry.styles import * + +usage_logger = logging.getLogger("M365TelemetryAsyncLogger.ManagedDevicesUI") + +class ManagedDevicesSubFrame(ctk.CTkFrame): + """Sub-frame for Managed Devices.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, semaphore=None, **kwargs): + super().__init__(master, fg_color="transparent", **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.semaphore = semaphore + self.status = None + self.page = 0 + self.ITEMS_PER_PAGE = 5 + self.csv_path = None + self.is_cancelled = False + + self.build_ui() + + def build_ui(self): + self.header_frame = ctk.CTkFrame(self, fg_color="transparent") + self.header_frame.pack(fill="x", pady=(0, 5)) + + ctk.CTkLabel(self.header_frame, text="Managed Devices", font=FONT_BODY_BOLD, text_color=COLOR_TEXT_MAIN).pack(side="left") + + self.btn_reload = ctk.CTkButton( + self.header_frame, text="↻ Reload", width=80, height=24, + font=ctk.CTkFont(family="Segoe UI", size=12), + fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color="#2563EB", hover_color="#DBEAFE", + command=self.trigger_fetch_individual + ) + self.btn_reload.pack(side="right", padx=(10, 0)) + + self.btn_export = ctk.CTkButton( + self.header_frame, text="Export Managed Devices", width=180, height=32, corner_radius=16, + font=FONT_BODY_BOLD, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, + command=self._export + ) + self.btn_export.pack(side="right") + + self.state_frame = ctk.CTkFrame(self, fg_color="transparent") + + self.summary_lbl = ctk.CTkLabel(self, text="Total Extracted: 0 Managed Devices", font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_SUB) + + self.grid_frame = ctk.CTkFrame(self, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + + headers = ["User ID", "Device Name", "Operating System", "Management Agent", "Registration State", "Model", "Manufacturer"] + for col_idx, head_text in enumerate(headers): + self.grid_frame.grid_columnconfigure(col_idx, weight=1 if col_idx != 0 else 2) # userId gets slightly more weight + cell = ctk.CTkFrame(self.grid_frame, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=1, pady=1) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + # Pagination controls frame (stays fixed/centered) + self.pagination_frame = ctk.CTkFrame(self, fg_color="transparent") + + self.reset_view() + + def reset_view(self): + self.status = None + self.is_cancelled = False + self.csv_path = None + self.page = 0 + self.state_frame.pack_forget() + self.summary_lbl.pack_forget() + self.grid_frame.pack_forget() + self.pagination_frame.pack_forget() + for w in self.state_frame.winfo_children(): w.destroy() + for w in self.grid_frame.winfo_children(): + info = w.grid_info() + if "row" in info and int(info["row"]) > 0: w.destroy() + for w in self.pagination_frame.winfo_children(): w.destroy() + + def _set_loading_state(self, msg): + for w in self.state_frame.winfo_children(): w.destroy() + self.state_frame.pack(fill="x", expand=True) + ctk.CTkLabel(self.state_frame, text=f"⏳ {msg}", text_color=COLOR_TEXT_SUB, font=FONT_BODY_MEDIUM).pack(pady=(15, 5)) + pb = ctk.CTkProgressBar(self.state_frame, mode="indeterminate", width=200, fg_color=COLOR_SURFACE_VARIANT, progress_color=COLOR_PRIMARY) + pb.pack(pady=(0, 15)) + pb.start() + + def trigger_fetch_individual(self): + tenant, clients, secrets = self.get_credentials() + if tenant: + self.trigger_fetch(tenant, clients[0], secrets[0]) + + def trigger_fetch(self, tenant, client_id, client_secret): + self.status = "loading" + self.is_cancelled = False + self.page = 0 + self.btn_reload.configure(state="disabled") + self.btn_export.configure(state="disabled") + self.grid_frame.pack_forget() + self.pagination_frame.pack_forget() + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + if os.path.basename(script_dir) == "intune": + script_dir = os.path.dirname(script_dir) + reports_dir = os.path.join(script_dir, "reports", f"{tenant}_{client_id}") + self.csv_path = os.path.join(reports_dir, "intune_managed_devices.csv") + + self._set_loading_state("Scanning Managed Devices...") + self.on_status_change() + + threading.Thread(target=self._execute_worker, args=(tenant, client_id, client_secret), daemon=True).start() + + def _execute_worker(self, tenant, client_id, client_secret): + if self.semaphore: self.semaphore.acquire() + try: + run_managed_devices_pipeline( + client_id=client_id, + client_secret=client_secret, + tenant_id=tenant, + csv_path=self.csv_path, + is_cancelled_callback=lambda: self.is_cancelled + ) + + if not self.is_cancelled: + reports_dir = os.path.dirname(self.csv_path) + db_path = os.path.join(reports_dir, "telemetry_cache.db") + if os.path.exists(self.csv_path): + asyncio.run(import_csv_to_sqlite(self.csv_path, db_path, "managed_devices")) + + self.status = "success" + self.after(0, self._render_success) + except Exception as e: + usage_logger.error(f"Managed devices fetch error: {e}", exc_info=True) + self.status = "error" + self.after(0, self._render_error, str(e)) + finally: + if self.semaphore: self.semaphore.release() + self.after(0, self.on_status_change) + + def _render_success(self): + self.btn_reload.configure(state="normal") + self.btn_export.configure(state="normal") + self.state_frame.pack_forget() + self.summary_lbl.pack(anchor="w", padx=10, pady=5) + self.grid_frame.pack(fill="x") + self.pagination_frame.pack(fill="x", pady=(5, 10)) + + self._update_grid() + + def _render_error(self, err): + self.btn_reload.configure(state="normal") + self.grid_frame.pack_forget() + self.pagination_frame.pack_forget() + self.summary_lbl.pack_forget() + for w in self.state_frame.winfo_children(): w.destroy() + self.state_frame.pack(fill="x", expand=True) + ctk.CTkLabel(self.state_frame, text=f"✖ {err}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center", wraplength=700).pack(pady=(15, 5)) + ctk.CTkButton(self.state_frame, text="Try Again", command=self.trigger_fetch_individual, width=100, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 15)) + + def _load_page(self): + if not self.csv_path or not os.path.exists(self.csv_path): return [], 0 + + reports_dir = os.path.dirname(self.csv_path) + db_path = os.path.join(reports_dir, "telemetry_cache.db") + if not os.path.exists(db_path): return [], 0 + + try: + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + cursor = conn.cursor() + + cursor.execute("SELECT COUNT(*) FROM managed_devices") + total = cursor.fetchone()[0] + + offset = self.page * self.ITEMS_PER_PAGE + cursor.execute("SELECT * FROM managed_devices LIMIT ? OFFSET ?", (self.ITEMS_PER_PAGE, offset)) + rows = cursor.fetchall() + + conn.close() + return rows, total + except Exception as e: + usage_logger.error(f"Error reading SQLite managed_devices: {e}") + return [], 0 + + def _update_grid(self): + for w in self.grid_frame.winfo_children(): + info = w.grid_info() + if "row" in info and int(info["row"]) > 0: w.destroy() + + page_data, total_count = self._load_page() + total_pages = (total_count - 1) // self.ITEMS_PER_PAGE + 1 if total_count > 0 else 1 + + self.summary_lbl.configure(text=f"Total Extracted: {total_count} Managed Devices") + + for row_idx, item in enumerate(page_data, start=1): + bg_style = COLOR_SURFACE if row_idx % 2 == 0 else COLOR_SURFACE_VARIANT + # Field keys from DB (case-insensitive because db.py intercepts dictionary queries) + vals = [ + item.get("userId") or "N/A", + item.get("deviceName") or "N/A", + item.get("operatingSystem") or "N/A", + item.get("managementAgent") or "unknown", + item.get("deviceRegistrationState") or "unknown", + item.get("model") or "N/A", + item.get("manufacturer") or "N/A" + ] + for col_idx, val in enumerate(vals): + cell = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + cell.grid(row=row_idx, column=col_idx, sticky="nsew", padx=1, pady=1) + lbl = ctk.CTkLabel(cell, text=str(val), font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN, wraplength=250, justify="left", anchor="w") + lbl.pack(padx=10, pady=8, fill="x") + + # Re-build pagination controls + for w in self.pagination_frame.winfo_children(): w.destroy() + + center_container = ctk.CTkFrame(self.pagination_frame, fg_color="transparent") + center_container.pack(pady=5) + + prev_state = "normal" if self.page > 0 else "disabled" + ctk.CTkButton( + center_container, text="◀ Prev", width=70, height=22, corner_radius=6, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, state=prev_state, + command=lambda: self._change_page(-1) + ).pack(side="left", padx=5) + + ctk.CTkLabel(center_container, text=f"Page {self.page + 1} of {total_pages}", font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_SUB).pack(side="left", padx=15) + + next_state = "normal" if self.page < total_pages - 1 else "disabled" + ctk.CTkButton( + center_container, text="Next ▶", width=70, height=22, corner_radius=6, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, state=next_state, + command=lambda: self._change_page(1) + ).pack(side="left", padx=5) + + def _change_page(self, delta): + self.page += delta + self._update_grid() + + def _export(self): + if not self.csv_path or not os.path.exists(self.csv_path): + messagebox.showerror("Export Error", "No data available to export yet.") + return + + dest_path = filedialog.asksaveasfilename( + defaultextension=".csv", + filetypes=[("CSV files", "*.csv"), ("All files", "*.*")], + title="Export Managed Devices CSV", + initialfile="M365_Intune_Managed_Devices.csv", + parent=self + ) + if not dest_path: return + + try: + df = pd.read_csv(self.csv_path) + df.fillna("N/A").to_csv(dest_path, index=False, encoding="utf-8-sig") + messagebox.showinfo("Export Successful", f"Managed Devices successfully exported to:\n{dest_path}") + except Exception as e: + messagebox.showerror("Export Failed", f"Failed to save CSV file: {e}") + + @property + def last_data(self): + if not self.csv_path or not os.path.exists(self.csv_path): return [] + try: + df = pd.read_csv(self.csv_path) + return df.head(200).fillna("N/A").to_dict('records') + except Exception: + return [] + + def cancel(self): + self.is_cancelled = True + self.status = None diff --git a/telemetry/intune/mdm_policies.py b/telemetry/intune/mdm_policies.py new file mode 100644 index 00000000..a6d3c746 --- /dev/null +++ b/telemetry/intune/mdm_policies.py @@ -0,0 +1,314 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""UI component for Mobile Device Management (MDM) Policies telemetry.""" + +import os +import csv +import logging +import threading +import shutil +import pandas as pd +import customtkinter as ctk +import sqlite3 +import asyncio +from tkinter import filedialog, messagebox + +from core.graph.db import import_csv_to_sqlite +from core.graph.intune.mdm_policies import run_mdm_policies_pipeline +from telemetry.styles import * + +usage_logger = logging.getLogger("M365TelemetryAsyncLogger.MdmPoliciesUI") + +class MdmPoliciesSubFrame(ctk.CTkFrame): + """Sub-frame for Mobile Device Management Policies.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, semaphore=None, **kwargs): + self.get_delegated_auth = kwargs.pop("delegated_auth_callback", None) + super().__init__(master, fg_color="transparent", **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.semaphore = semaphore + self.status = None + self.page = 0 + self.ITEMS_PER_PAGE = 5 + self.csv_path = None + self.is_cancelled = False + + self.build_ui() + + def build_ui(self): + self.header_frame = ctk.CTkFrame(self, fg_color="transparent") + self.header_frame.pack(fill="x", pady=(0, 5)) + + ctk.CTkLabel(self.header_frame, text="Mobile Device Management Policies", font=FONT_BODY_BOLD, text_color=COLOR_TEXT_MAIN).pack(side="left") + + self.btn_reload = ctk.CTkButton( + self.header_frame, text="↻ Reload", width=80, height=24, + font=ctk.CTkFont(family="Segoe UI", size=12), + fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color="#2563EB", hover_color="#DBEAFE", + command=self.trigger_fetch_individual + ) + self.btn_reload.pack(side="right", padx=(10, 0)) + + self.btn_export = ctk.CTkButton( + self.header_frame, text="Export MDM Policies", width=180, height=32, corner_radius=16, + font=FONT_BODY_BOLD, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, + command=self._export + ) + self.btn_export.pack(side="right") + + self.state_frame = ctk.CTkFrame(self, fg_color="transparent") + + self.summary_lbl = ctk.CTkLabel(self, text="Total Extracted: 0 MDM Policies", font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_SUB) + + self.grid_frame = ctk.CTkFrame(self, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + + headers = ["Display Name", "Description", "Applies To", "Discovery URL", "Terms of Use URL", "Compliance URL"] + weights = [2, 3, 2, 3, 3, 3] + for col_idx, (head_text, weight) in enumerate(zip(headers, weights)): + self.grid_frame.grid_columnconfigure(col_idx, weight=weight) + cell = ctk.CTkFrame(self.grid_frame, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=1, pady=1) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + # Pagination controls frame (stays fixed/centered) + self.pagination_frame = ctk.CTkFrame(self, fg_color="transparent") + + self.reset_view() + + def reset_view(self): + self.status = None + self.is_cancelled = False + self.csv_path = None + self.page = 0 + self.state_frame.pack_forget() + self.summary_lbl.pack_forget() + self.grid_frame.pack_forget() + self.pagination_frame.pack_forget() + for w in self.state_frame.winfo_children(): w.destroy() + for w in self.grid_frame.winfo_children(): + info = w.grid_info() + if "row" in info and int(info["row"]) > 0: w.destroy() + for w in self.pagination_frame.winfo_children(): w.destroy() + + def _set_loading_state(self, msg): + for w in self.state_frame.winfo_children(): w.destroy() + self.state_frame.pack(fill="x", expand=True) + ctk.CTkLabel(self.state_frame, text=f"⏳ {msg}", text_color=COLOR_TEXT_SUB, font=FONT_BODY_MEDIUM).pack(pady=(15, 5)) + pb = ctk.CTkProgressBar(self.state_frame, mode="indeterminate", width=200, fg_color=COLOR_SURFACE_VARIANT, progress_color=COLOR_PRIMARY) + pb.pack(pady=(0, 15)) + pb.start() + + def trigger_fetch_individual(self): + tenant, clients, secrets = self.get_credentials() + use_delegated = self.get_delegated_auth() if self.get_delegated_auth else False + if tenant: + self.trigger_fetch(tenant, clients[0], secrets[0], use_delegated_auth=use_delegated) + + def trigger_fetch(self, tenant, client_id, client_secret, use_delegated_auth=False): + self.status = "loading" + self.is_cancelled = False + self.page = 0 + self.btn_reload.configure(state="disabled") + self.btn_export.configure(state="disabled") + self.grid_frame.pack_forget() + self.pagination_frame.pack_forget() + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + if os.path.basename(script_dir) == "intune": + script_dir = os.path.dirname(script_dir) + reports_dir = os.path.join(script_dir, "reports", f"{tenant}_{client_id}") + self.csv_path = os.path.join(reports_dir, "intune_mdm_policies.csv") + + if use_delegated_auth: + self._set_loading_state("Authenticating via MSAL...") + else: + self._set_loading_state("Scanning MDM Policies...") + self.on_status_change() + + threading.Thread(target=self._execute_worker, args=(tenant, client_id, client_secret, use_delegated_auth), daemon=True).start() + + def _execute_worker(self, tenant, client_id, client_secret, use_delegated_auth): + if self.semaphore: self.semaphore.acquire() + try: + delegated_token = None + if use_delegated_auth: + from core.graph.delegated_auth import DelegatedAuthClient + auth_client = DelegatedAuthClient(tenant, client_id, client_secret) + delegated_token = auth_client.get_token(scopes=["https://graph.microsoft.com/.default"]) + if not delegated_token: + raise Exception("Failed to acquire delegated auth token. User may have cancelled or app is misconfigured.") + + self.after(0, lambda: self._set_loading_state("Scanning MDM Policies...")) + + run_mdm_policies_pipeline( + client_id=client_id, + client_secret=client_secret, + tenant_id=tenant, + csv_path=self.csv_path, + delegated_token=delegated_token, + is_cancelled_callback=lambda: self.is_cancelled + ) + + if not self.is_cancelled: + reports_dir = os.path.dirname(self.csv_path) + db_path = os.path.join(reports_dir, "telemetry_cache.db") + if os.path.exists(self.csv_path): + asyncio.run(import_csv_to_sqlite(self.csv_path, db_path, "mdm_policies")) + + self.status = "success" + self.after(0, self._render_success) + except Exception as e: + usage_logger.error(f"MDM policies fetch error: {e}", exc_info=True) + self.status = "error" + self.after(0, self._render_error, str(e)) + finally: + if self.semaphore: self.semaphore.release() + self.after(0, self.on_status_change) + + def _render_success(self): + self.btn_reload.configure(state="normal") + self.btn_export.configure(state="normal") + self.state_frame.pack_forget() + self.summary_lbl.pack(anchor="w", padx=10, pady=5) + self.grid_frame.pack(fill="x") + self.pagination_frame.pack(fill="x", pady=(5, 10)) + + self._update_grid() + + def _render_error(self, err): + self.btn_reload.configure(state="normal") + self.grid_frame.pack_forget() + self.pagination_frame.pack_forget() + self.summary_lbl.pack_forget() + for w in self.state_frame.winfo_children(): w.destroy() + self.state_frame.pack(fill="x", expand=True) + ctk.CTkLabel(self.state_frame, text=f"✖ {err}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center", wraplength=700).pack(pady=(15, 5)) + ctk.CTkButton(self.state_frame, text="Try Again", command=self.trigger_fetch_individual, width=100, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 15)) + + def _load_page(self): + if not self.csv_path or not os.path.exists(self.csv_path): return [], 0 + + reports_dir = os.path.dirname(self.csv_path) + db_path = os.path.join(reports_dir, "telemetry_cache.db") + if not os.path.exists(db_path): return [], 0 + + try: + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + cursor = conn.cursor() + + cursor.execute("SELECT COUNT(*) FROM mdm_policies") + total = cursor.fetchone()[0] + + offset = self.page * self.ITEMS_PER_PAGE + cursor.execute("SELECT * FROM mdm_policies LIMIT ? OFFSET ?", (self.ITEMS_PER_PAGE, offset)) + rows = cursor.fetchall() + + conn.close() + return rows, total + except Exception as e: + usage_logger.error(f"Error reading SQLite mdm_policies: {e}") + return [], 0 + + def _update_grid(self): + for w in self.grid_frame.winfo_children(): + info = w.grid_info() + if "row" in info and int(info["row"]) > 0: w.destroy() + + page_data, total_count = self._load_page() + total_pages = (total_count - 1) // self.ITEMS_PER_PAGE + 1 if total_count > 0 else 1 + + self.summary_lbl.configure(text=f"Total Extracted: {total_count} MDM Policies") + + for row_idx, item in enumerate(page_data, start=1): + bg_style = COLOR_SURFACE if row_idx % 2 == 0 else COLOR_SURFACE_VARIANT + vals = [ + item.get("displayName") or "N/A", + item.get("description") or "N/A", + item.get("appliesTo") or "None", + item.get("discoveryUrl") or "N/A", + item.get("termsOfUseUrl") or "N/A", + item.get("complianceUrl") or "N/A" + ] + for col_idx, val in enumerate(vals): + cell = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + cell.grid(row=row_idx, column=col_idx, sticky="nsew", padx=1, pady=1) + lbl = ctk.CTkLabel(cell, text=str(val), font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN, wraplength=180, justify="left", anchor="w") + lbl.pack(padx=10, pady=8, fill="x") + + # Re-build pagination controls + for w in self.pagination_frame.winfo_children(): w.destroy() + + center_container = ctk.CTkFrame(self.pagination_frame, fg_color="transparent") + center_container.pack(pady=5) + + prev_state = "normal" if self.page > 0 else "disabled" + ctk.CTkButton( + center_container, text="◀ Prev", width=70, height=22, corner_radius=6, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, state=prev_state, + command=lambda: self._change_page(-1) + ).pack(side="left", padx=5) + + ctk.CTkLabel(center_container, text=f"Page {self.page + 1} of {total_pages}", font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_SUB).pack(side="left", padx=15) + + next_state = "normal" if self.page < total_pages - 1 else "disabled" + ctk.CTkButton( + center_container, text="Next ▶", width=70, height=22, corner_radius=6, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, state=next_state, + command=lambda: self._change_page(1) + ).pack(side="left", padx=5) + + def _change_page(self, delta): + self.page += delta + self._update_grid() + + def _export(self): + if not self.csv_path or not os.path.exists(self.csv_path): + messagebox.showerror("Export Error", "No data available to export yet.") + return + + dest_path = filedialog.asksaveasfilename( + defaultextension=".csv", + filetypes=[("CSV files", "*.csv"), ("All files", "*.*")], + title="Export MDM Policies CSV", + initialfile="M365_Intune_MDM_Policies.csv", + parent=self + ) + if not dest_path: return + + try: + shutil.copyfile(self.csv_path, dest_path) + messagebox.showinfo("Export Successful", f"MDM Policies successfully exported to:\n{dest_path}") + except Exception as e: + messagebox.showerror("Export Failed", f"Failed to save CSV file: {e}") + + @property + def last_data(self): + if not self.csv_path or not os.path.exists(self.csv_path): return [] + try: + df = pd.read_csv(self.csv_path) + return df.head(200).fillna("N/A").to_dict('records') + except Exception: + return [] + + def cancel(self): + self.is_cancelled = True + self.status = None diff --git a/telemetry/intune/mobile_apps.py b/telemetry/intune/mobile_apps.py new file mode 100644 index 00000000..976076c9 --- /dev/null +++ b/telemetry/intune/mobile_apps.py @@ -0,0 +1,182 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""UI component for Managed Mobile Apps.""" + +import os +import csv +import logging +import threading +import customtkinter as ctk +import asyncio +import sqlite3 + +from core.graph.db import import_csv_to_sqlite +from core.graph.intune.mobile_apps import run_mobile_apps_pipeline +from telemetry.styles import * + +usage_logger = logging.getLogger("M365TelemetryAsyncLogger.MobileAppsUI") + +class MobileAppsSubFrame(ctk.CTkFrame): + """Sub-frame for Managed Mobile Apps telemetry.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, semaphore=None, **kwargs): + super().__init__(master, fg_color="transparent", **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.semaphore = semaphore + self.status = None + self.csv_path = None + self.is_cancelled = False + self._cached_apps = [] + + self.build_ui() + + def build_ui(self): + self.inner_frame = ctk.CTkFrame(self, fg_color="transparent") + self.inner_frame.pack(fill="x", padx=10, pady=(10, 5)) + + self.lbl_title = ctk.CTkLabel( + self.inner_frame, + text="⚙️ Managed Mobile Apps: ", + font=FONT_BODY_BOLD, + text_color=COLOR_TEXT_MAIN, + anchor="w" + ) + self.lbl_title.pack(side="left", anchor="nw") + + self.lbl_content = ctk.CTkLabel( + self.inner_frame, + text="No apps found or scanning...", + font=FONT_BODY_MEDIUM, + text_color=COLOR_TEXT_SUB, + justify="left", + anchor="w" + ) + self.lbl_content.pack(side="left", fill="x", expand=True, anchor="nw") + + def make_configure_handler(lbl=self.lbl_content): + def on_configure(event): + lbl.configure(wraplength=max(200, event.width - 200)) + return on_configure + + self.inner_frame.bind("", make_configure_handler()) + + def reset_view(self): + self.status = None + self.is_cancelled = False + self.csv_path = None + self._cached_apps = [] + self.lbl_content.configure(text="No apps found or scanning...", text_color=COLOR_TEXT_SUB) + + def trigger_fetch_individual(self): + tenant, clients, secrets = self.get_credentials() + if tenant: + self.trigger_fetch(tenant, clients[0], secrets[0]) + + def trigger_fetch(self, tenant, client_id, client_secret): + self.status = "loading" + self.is_cancelled = False + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + if os.path.basename(script_dir) == "intune": + script_dir = os.path.dirname(script_dir) + reports_dir = os.path.join(script_dir, "reports", f"{tenant}_{client_id}") + self.csv_path = os.path.join(reports_dir, "intune_apps.csv") + + self.lbl_content.configure(text="⏳ Scanning Managed Mobile Apps in background...", text_color=COLOR_TEXT_SUB) + self.on_status_change() + + threading.Thread(target=self._execute_worker, args=(tenant, client_id, client_secret), daemon=True).start() + + def _execute_worker(self, tenant, client_id, client_secret): + if self.semaphore: self.semaphore.acquire() + try: + temp_csv_path = self.csv_path + ".tmp" + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + if os.path.basename(script_dir) == "intune": + script_dir = os.path.dirname(script_dir) + reports_dir = os.path.join(script_dir, "reports", f"{tenant}_{client_id}") + os.makedirs(reports_dir, exist_ok=True) + + with open(temp_csv_path, 'w', encoding='utf-8', newline='') as f: + writer = csv.writer(f) + writer.writerow(["displayName"]) + + run_mobile_apps_pipeline( + client_id=client_id, + client_secret=client_secret, + tenant_id=tenant, + csv_path=temp_csv_path, + is_cancelled_callback=lambda: self.is_cancelled + ) + + if not self.is_cancelled: + if os.path.exists(temp_csv_path): + if os.path.exists(self.csv_path): os.remove(self.csv_path) + os.rename(temp_csv_path, self.csv_path) + + db_path = os.path.join(reports_dir, "telemetry_cache.db") + if os.path.exists(self.csv_path): + asyncio.run(import_csv_to_sqlite(self.csv_path, db_path, "mobile_apps")) + + self.status = "success" + self.after(0, self._render_success) + except Exception as e: + usage_logger.error(f"Mobile apps fetch error: {e}", exc_info=True) + self.status = "error" + self.after(0, self._render_error, str(e)) + finally: + if 'temp_csv_path' in locals() and os.path.exists(temp_csv_path): + try: os.remove(temp_csv_path) + except Exception: pass + if self.semaphore: self.semaphore.release() + self.after(0, self.on_status_change) + + def _render_success(self): + apps = self._load_data_from_csv() + self._cached_apps = apps + display_text = ", ".join(apps) if apps else "No managed apps detected." + self.lbl_content.configure(text=display_text, text_color=COLOR_TEXT_MAIN) + + def _render_error(self, err): + self.lbl_content.configure(text=f"✖ {err}", text_color=COLOR_ERROR) + + def _load_data_from_csv(self): + if not self.csv_path or not os.path.exists(self.csv_path): return [] + items = [] + reports_dir = os.path.dirname(self.csv_path) + db_path = os.path.join(reports_dir, "telemetry_cache.db") + if not os.path.exists(db_path): return [] + try: + conn = sqlite3.connect(db_path) + cursor = conn.cursor() + cursor.execute("SELECT displayName FROM mobile_apps") + rows = cursor.fetchall() + for row in rows: + if row and row[0]: items.append(row[0]) + conn.close() + except Exception as e: + usage_logger.error(f"Error reading mobile apps DB: {e}") + return sorted(items) + + @property + def last_data(self): + if self._cached_apps: return self._cached_apps + return self._load_data_from_csv() + + def cancel(self): + self.is_cancelled = True + self.status = None diff --git a/telemetry/intune/vc_devices.py b/telemetry/intune/vc_devices.py new file mode 100644 index 00000000..a485bdd6 --- /dev/null +++ b/telemetry/intune/vc_devices.py @@ -0,0 +1,365 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""UI component for VC Devices telemetry.""" + +import os +import csv +import logging +import threading +import pandas as pd +import customtkinter as ctk +import sqlite3 +import asyncio +import shutil +from tkinter import filedialog, messagebox + +from core.graph.db import import_csv_to_sqlite +from core.graph.client import GraphClient +from core.graph.directory import DirectoryService +from core.powershell.client import PowerShellClient +from core.powershell.calendar import CalendarStatsService +from telemetry.styles import * + +usage_logger = logging.getLogger("M365TelemetryAsyncLogger.VCDevicesUI") + +class VCDevicesSubFrame(ctk.CTkFrame): + """Sub-frame for Room-registered Video Conferencing (VC) Devices.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, semaphore=None, **kwargs): + super().__init__(master, fg_color="transparent", **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.semaphore = semaphore + self.status = None + self.page = 0 + self.ITEMS_PER_PAGE = 5 + self.csv_path = None + self.is_cancelled = False + self.rooms_count = 0 + + self.build_ui() + + def build_ui(self): + self.header_frame = ctk.CTkFrame(self, fg_color="transparent") + self.header_frame.pack(fill="x", pady=(0, 5)) + + ctk.CTkLabel(self.header_frame, text="VC Devices", font=FONT_BODY_BOLD, text_color=COLOR_TEXT_MAIN).pack(side="left") + + self.btn_reload = ctk.CTkButton( + self.header_frame, text="↻ Reload", width=80, height=24, + font=ctk.CTkFont(family="Segoe UI", size=12), + fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color="#2563EB", hover_color="#DBEAFE", + command=self.trigger_fetch_individual + ) + self.btn_reload.pack(side="right", padx=(10, 0)) + + self.btn_export = ctk.CTkButton( + self.header_frame, text="Export VC Devices", width=180, height=32, corner_radius=16, + font=FONT_BODY_BOLD, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, + command=self._export + ) + self.btn_export.pack(side="right") + + self.state_frame = ctk.CTkFrame(self, fg_color="transparent") + + # Room metric and total counts container + self.metrics_container = ctk.CTkFrame(self, fg_color="transparent") + self.rooms_metric_lbl = ctk.CTkLabel(self.metrics_container, text="Room Mailboxes Discovered: 0", font=FONT_BODY_BOLD, text_color=COLOR_SUCCESS) + self.rooms_metric_lbl.pack(side="left", padx=(10, 20)) + + self.summary_lbl = ctk.CTkLabel(self.metrics_container, text="Total Extracted VC Devices: 0", font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_SUB) + self.summary_lbl.pack(side="left") + + self.grid_frame = ctk.CTkFrame(self, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + + headers = ["User ID", "Device Name", "Operating System", "Management Agent", "Registration State", "Model", "Manufacturer"] + for col_idx, head_text in enumerate(headers): + self.grid_frame.grid_columnconfigure(col_idx, weight=1 if col_idx != 0 else 2) # userId gets slightly more weight + cell = ctk.CTkFrame(self.grid_frame, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=1, pady=1) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + # Pagination controls frame (stays fixed/centered) + self.pagination_frame = ctk.CTkFrame(self, fg_color="transparent") + + self.reset_view() + + def reset_view(self): + self.status = None + self.is_cancelled = False + self.csv_path = None + self.page = 0 + self.rooms_count = 0 + self.state_frame.pack_forget() + self.metrics_container.pack_forget() + self.grid_frame.pack_forget() + self.pagination_frame.pack_forget() + for w in self.state_frame.winfo_children(): w.destroy() + for w in self.grid_frame.winfo_children(): + info = w.grid_info() + if "row" in info and int(info["row"]) > 0: w.destroy() + for w in self.pagination_frame.winfo_children(): w.destroy() + + def _set_loading_state(self, msg): + for w in self.state_frame.winfo_children(): w.destroy() + self.state_frame.pack(fill="x", expand=True) + ctk.CTkLabel(self.state_frame, text=f"⏳ {msg}", text_color=COLOR_TEXT_SUB, font=FONT_BODY_MEDIUM).pack(pady=(15, 5)) + pb = ctk.CTkProgressBar(self.state_frame, mode="indeterminate", width=200, fg_color=COLOR_SURFACE_VARIANT, progress_color=COLOR_PRIMARY) + pb.pack(pady=(0, 15)) + pb.start() + + def trigger_fetch_individual(self): + tenant, clients, secrets = self.get_credentials() + if tenant: + self.trigger_fetch(tenant, clients[0], secrets[0]) + + def trigger_fetch(self, tenant, client_id, client_secret): + self.status = "loading" + self.is_cancelled = False + self.page = 0 + self.btn_reload.configure(state="disabled") + self.btn_export.configure(state="disabled") + self.grid_frame.pack_forget() + self.pagination_frame.pack_forget() + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + if os.path.basename(script_dir) == "intune": + script_dir = os.path.dirname(script_dir) + reports_dir = os.path.join(script_dir, "reports", f"{tenant}_{client_id}") + self.csv_path = os.path.join(reports_dir, "intune_vc_devices.csv") + + self._set_loading_state("Filtering Video Conferencing (VC) Devices...") + self.on_status_change() + + threading.Thread(target=self._execute_worker, args=(tenant, client_id, client_secret), daemon=True).start() + + def _execute_worker(self, tenant, client_id, client_secret): + if self.semaphore: self.semaphore.acquire() + try: + reports_dir = os.path.dirname(self.csv_path) + + # Fetch primary domain name for Exchange connection + tenant_domain = tenant + client = None + try: + client = GraphClient( + tenant_id=tenant, + client_ids=client_id, + client_secrets=client_secret, + concurrency=1, + retries=5, + backoff=2 + ) + client.authenticate() + dir_svc = DirectoryService(client) + tenant_domain = dir_svc.get_tenant_primary_domain() + except Exception as e: + usage_logger.warning(f"Could not retrieve tenant domain via Graph. Falling back to Tenant ID Guid: {e}") + finally: + if client: + client.close() + + # Query Room Mailboxes from Exchange Online PowerShell directly + usage_logger.info("Connecting to Exchange Online PowerShell for room mailboxes...") + ps_client = PowerShellClient( + tenant_id=tenant_domain, + client_id=client_id, + client_secret=client_secret, + cert_tenant_id=tenant + ) + cal_service = CalendarStatsService(ps_client) + rooms_list = cal_service.fetch_room_mailboxes() + room_emails = set(r.lower() for r in rooms_list if r) + + if not room_emails: + raise ValueError("No room mailboxes found.") + self.rooms_count = len(room_emails) + + # Save the discovered room list to CSV + rooms_csv = os.path.join(reports_dir, "room_mailboxes.csv") + os.makedirs(reports_dir, exist_ok=True) + with open(rooms_csv, 'w', encoding='utf-8', newline='') as f: + writer = csv.writer(f) + writer.writerow(["PrimarySmtpAddress"]) + for room in rooms_list: + writer.writerow([room]) + + # Load managed devices + devices_csv = os.path.join(reports_dir, "intune_managed_devices.csv") + if os.path.exists(devices_csv): + df_devices = pd.read_csv(devices_csv) + else: + df_devices = pd.DataFrame(columns=["userId", "deviceName", "operatingSystem", "managementAgent", "deviceRegistrationState", "model", "manufacturer", "userPrincipalName", "emailAddress"]) + + # Filter where userPrincipalName or emailAddress in room_emails + upn_match = df_devices["userPrincipalName"].fillna("").str.lower().isin(room_emails) if "userPrincipalName" in df_devices.columns else pd.Series([False]*len(df_devices)) + email_match = df_devices["emailAddress"].fillna("").str.lower().isin(room_emails) if "emailAddress" in df_devices.columns else pd.Series([False]*len(df_devices)) + + df_vc = df_devices[upn_match | email_match] + + # Save to intune_vc_devices.csv + df_vc.to_csv(self.csv_path, index=False, encoding='utf-8') + + # Import to sqlite db cache table vc_devices + db_path = os.path.join(reports_dir, "telemetry_cache.db") + if os.path.exists(self.csv_path): + asyncio.run(import_csv_to_sqlite(self.csv_path, db_path, "vc_devices")) + + self.status = "success" + self.after(0, self._render_success) + except Exception as e: + usage_logger.error(f"VC devices filter error: {e}", exc_info=True) + self.status = "error" + self.after(0, self._render_error, str(e)) + finally: + if self.semaphore: self.semaphore.release() + self.after(0, self.on_status_change) + + def _render_success(self): + self.btn_reload.configure(state="normal") + self.btn_export.configure(state="normal") + self.state_frame.pack_forget() + self.metrics_container.pack(anchor="w", padx=10, pady=5) + self.grid_frame.pack(fill="x") + self.pagination_frame.pack(fill="x", pady=(5, 10)) + + self.rooms_metric_lbl.configure(text=f"Room Mailboxes Discovered: {self.rooms_count}") + self._update_grid() + + def _render_error(self, err): + self.btn_reload.configure(state="normal") + self.grid_frame.pack_forget() + self.pagination_frame.pack_forget() + self.metrics_container.pack_forget() + for w in self.state_frame.winfo_children(): w.destroy() + self.state_frame.pack(fill="x", expand=True) + ctk.CTkLabel(self.state_frame, text=f"✖ {err}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center", wraplength=700).pack(pady=(15, 5)) + ctk.CTkButton(self.state_frame, text="Try Again", command=self.trigger_fetch_individual, width=100, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 15)) + + def _load_page(self): + if not self.csv_path or not os.path.exists(self.csv_path): return [], 0 + + reports_dir = os.path.dirname(self.csv_path) + db_path = os.path.join(reports_dir, "telemetry_cache.db") + if not os.path.exists(db_path): return [], 0 + + try: + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + cursor = conn.cursor() + + cursor.execute("SELECT COUNT(*) FROM vc_devices") + total = cursor.fetchone()[0] + + offset = self.page * self.ITEMS_PER_PAGE + cursor.execute("SELECT * FROM vc_devices LIMIT ? OFFSET ?", (self.ITEMS_PER_PAGE, offset)) + rows = cursor.fetchall() + + conn.close() + return rows, total + except Exception as e: + usage_logger.error(f"Error reading SQLite vc_devices: {e}") + return [], 0 + + def _update_grid(self): + for w in self.grid_frame.winfo_children(): + info = w.grid_info() + if "row" in info and int(info["row"]) > 0: w.destroy() + + page_data, total_count = self._load_page() + total_pages = (total_count - 1) // self.ITEMS_PER_PAGE + 1 if total_count > 0 else 1 + + self.summary_lbl.configure(text=f"Total Extracted VC Devices: {total_count}") + + for row_idx, item in enumerate(page_data, start=1): + bg_style = COLOR_SURFACE if row_idx % 2 == 0 else COLOR_SURFACE_VARIANT + vals = [ + item.get("userId") or "N/A", + item.get("deviceName") or "N/A", + item.get("operatingSystem") or "N/A", + item.get("managementAgent") or "unknown", + item.get("deviceRegistrationState") or "unknown", + item.get("model") or "N/A", + item.get("manufacturer") or "N/A" + ] + for col_idx, val in enumerate(vals): + cell = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + cell.grid(row=row_idx, column=col_idx, sticky="nsew", padx=1, pady=1) + lbl = ctk.CTkLabel(cell, text=str(val), font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN, wraplength=250, justify="left", anchor="w") + lbl.pack(padx=10, pady=8, fill="x") + + # Re-build pagination controls + for w in self.pagination_frame.winfo_children(): w.destroy() + + center_container = ctk.CTkFrame(self.pagination_frame, fg_color="transparent") + center_container.pack(pady=5) + + prev_state = "normal" if self.page > 0 else "disabled" + ctk.CTkButton( + center_container, text="◀ Prev", width=70, height=22, corner_radius=6, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, state=prev_state, + command=lambda: self._change_page(-1) + ).pack(side="left", padx=5) + + ctk.CTkLabel(center_container, text=f"Page {self.page + 1} of {total_pages}", font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_SUB).pack(side="left", padx=15) + + next_state = "normal" if self.page < total_pages - 1 else "disabled" + ctk.CTkButton( + center_container, text="Next ▶", width=70, height=22, corner_radius=6, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, state=next_state, + command=lambda: self._change_page(1) + ).pack(side="left", padx=5) + + def _change_page(self, delta): + self.page += delta + self._update_grid() + + def _export(self): + if not self.csv_path or not os.path.exists(self.csv_path): + messagebox.showerror("Export Error", "No data available to export yet.") + return + + dest_path = filedialog.asksaveasfilename( + defaultextension=".csv", + filetypes=[("CSV files", "*.csv"), ("All files", "*.*")], + title="Export VC Devices CSV", + initialfile="M365_Intune_VC_Devices.csv", + parent=self + ) + if not dest_path: return + + try: + shutil.copyfile(self.csv_path, dest_path) + messagebox.showinfo("Export Successful", f"VC Devices successfully exported to:\n{dest_path}") + except Exception as e: + messagebox.showerror("Export Failed", f"Failed to save CSV file: {e}") + + @property + def last_data(self): + if not self.csv_path or not os.path.exists(self.csv_path): return [] + try: + df = pd.read_csv(self.csv_path) + return df.head(200).fillna("N/A").to_dict('records') + except Exception: + return [] + + def cancel(self): + self.is_cancelled = True + self.status = None diff --git a/telemetry/intune_policies.py b/telemetry/intune_policies.py new file mode 100644 index 00000000..0d244655 --- /dev/null +++ b/telemetry/intune_policies.py @@ -0,0 +1,23 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Backward compatibility facade for Microsoft Intune telemetry.""" + +# Re-export pipeline from core backend +from core.graph.intune import run_intune_policies_pipeline +from core.graph.client import GraphClient +from core.graph.intune import IntuneService + +# Re-export UI container from telemetry package +from telemetry.intune import IntunePoliciesFrame diff --git a/telemetry/m365_apps/__init__.py b/telemetry/m365_apps/__init__.py new file mode 100644 index 00000000..d1f8b7f6 --- /dev/null +++ b/telemetry/m365_apps/__init__.py @@ -0,0 +1,134 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Consolidated M365 Apps Telemetry Container.""" + +import logging +import customtkinter as ctk + +from telemetry.m365_apps.active_users import ActiveUsersUsageFrame +from telemetry.m365_apps.active_users_trend import ActiveUsersTrendFrame +from telemetry.m365_apps.app_usage import M365AppUsageFrame +from telemetry.styles import * + +usage_logger = logging.getLogger("M365TelemetryAsyncLogger.M365AppsUI") + +class M365AppsTelemetryFrame(ctk.CTkFrame): + """Uber section container hosting Active Users Usage, Trend, and M365 App Usage frames vertically stacked.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, **kwargs): + self.semaphore = kwargs.pop("concurrency_semaphore", None) + super().__init__(master, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=12, **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.status = None + + self.build_ui() + + def build_ui(self): + """Instantiates and stacks modular usage sub-frames.""" + self.pack(fill="x", expand=True, pady=10) + + self.inner_pad = ctk.CTkFrame(self, fg_color="transparent") + self.inner_pad.pack(fill="both", expand=True, padx=20, pady=20) + + # Uber Title Heading + ctk.CTkLabel( + self.inner_pad, + text="M365 Apps", + font=FONT_HEADER_SMALL, + text_color=COLOR_TEXT_MAIN + ).pack(anchor="w", pady=(0, 5)) + + # Active Users Usage Sub-frame + self.active_users_view = ActiveUsersUsageFrame( + master=self.inner_pad, + log_callback=self.log_msg, + credentials_callback=self.get_credentials, + status_change_callback=self._check_overall_status, + concurrency_semaphore=self.semaphore + ) + self.active_users_view.configure(fg_color="transparent", border_width=0) + self.active_users_view.pack(fill="x", expand=True, pady=(0, 5)) + + # Divider 1 + self.divider1 = ctk.CTkFrame(self.inner_pad, height=1, fg_color=COLOR_OUTLINE_LIGHT) + self.divider1.pack(fill="x", pady=10) + + # Active Users Trend Sub-frame + self.active_users_trend_view = ActiveUsersTrendFrame( + master=self.inner_pad, + log_callback=self.log_msg, + credentials_callback=self.get_credentials, + status_change_callback=self._check_overall_status, + concurrency_semaphore=self.semaphore + ) + self.active_users_trend_view.configure(fg_color="transparent", border_width=0) + self.active_users_trend_view.pack(fill="x", expand=True, pady=(0, 5)) + + # Divider 2 + self.divider2 = ctk.CTkFrame(self.inner_pad, height=1, fg_color=COLOR_OUTLINE_LIGHT) + self.divider2.pack(fill="x", pady=10) + + # M365 App Usage Sub-frame + self.m365_apps_view = M365AppUsageFrame( + master=self.inner_pad, + log_callback=self.log_msg, + credentials_callback=self.get_credentials, + status_change_callback=self._check_overall_status, + concurrency_semaphore=self.semaphore + ) + self.m365_apps_view.configure(fg_color="transparent", border_width=0) + self.m365_apps_view.pack(fill="x", expand=True, pady=(0, 5)) + + self.reset_view() + + def reset_view(self): + """Resets all sub-views and hides container.""" + self.pack_forget() + self.status = None + self.active_users_view.reset_view() + self.active_users_trend_view.reset_view() + self.m365_apps_view.reset_view() + + def trigger_fetch(self, tenant, client_id, client_secret): + """Displays container and delegates fetches to all sub-views.""" + self.status = "loading" + self.on_status_change() + + self.pack(fill="x", expand=True, pady=10) + + self.active_users_view.trigger_fetch(tenant, client_id, client_secret) + self.active_users_trend_view.trigger_fetch(tenant, client_id, client_secret) + self.m365_apps_view.trigger_fetch(tenant, client_id, client_secret) + + def _check_overall_status(self): + """Updates main container status based on sub-frame statuses.""" + sub_statuses = [self.active_users_view.status, self.active_users_trend_view.status, self.m365_apps_view.status] + if "loading" in sub_statuses: + self.status = "loading" + else: + if "success" in sub_statuses: + self.status = "success" + else: + self.status = "error" + self.on_status_change() + + def cancel(self): + """Cancels all child views in this container.""" + self.active_users_view.cancel() + self.active_users_trend_view.cancel() + self.m365_apps_view.cancel() + self.status = None diff --git a/telemetry/m365_apps/active_users.py b/telemetry/m365_apps/active_users.py new file mode 100644 index 00000000..4f1350a8 --- /dev/null +++ b/telemetry/m365_apps/active_users.py @@ -0,0 +1,191 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""UI frame for O365 Active Users Usage telemetry.""" + +import os +import logging +import threading +import customtkinter as ctk + +from core.graph.m365_apps.active_users import run_o365_pipeline +from telemetry.styles import * + +usage_logger = logging.getLogger("M365TelemetryAsyncLogger.ActiveUsersUsageUI") + +class ActiveUsersUsageFrame(ctk.CTkFrame): + """Self-contained component wrapping O365 Active Users Usage UI.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, **kwargs): + self.semaphore = kwargs.pop("concurrency_semaphore", None) + super().__init__(master, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=12, **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.status = None # 'loading', 'success', 'error', None + self.ITEMS_PER_PAGE = 5 + self.current_page = 0 + self.last_data = None + + self.build_ui() + + def build_ui(self): + self.pack(fill="x", expand=True, pady=10) + + self.inner_pad = ctk.CTkFrame(self, fg_color="transparent") + self.inner_pad.pack(fill="both", expand=True, padx=20, pady=20) + + self.header = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.header.pack(fill="x", pady=(0, 10)) + ctk.CTkLabel(self.header, text="O365 Active Users Usage", font=FONT_HEADER_SMALL, text_color=COLOR_TEXT_MAIN).pack(side="left") + self.reload_btn = ctk.CTkButton( + self.header, + state="disabled", text="↻ Reload", + width=80, + height=24, + font=ctk.CTkFont(family="Segoe UI", size=12), + fg_color="transparent", + border_width=1, + text_color="#2563EB", + hover_color="#DBEAFE", + command=self._retry_fetch + ) + self.reload_btn.pack(side="right") + + self.state_frame = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.grid_frame = ctk.CTkFrame(self.inner_pad, fg_color=COLOR_OUTLINE_LIGHT, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + + self.reset_view() + + def reset_view(self): + self.pack_forget() + self.state_frame.pack_forget() + self.grid_frame.pack_forget() + + for w in self.state_frame.winfo_children(): + w.destroy() + for w in self.grid_frame.winfo_children(): + w.destroy() + self.current_page = 0 + self.last_data = None + + def _set_state_loading(self, msg="Loading..."): + for w in self.state_frame.winfo_children(): + w.destroy() + self.loading_label = ctk.CTkLabel(self.state_frame, text=f"⏳ {msg}", text_color="#6b7280", font=ctk.CTkFont(family="Segoe UI", size=13)) + self.loading_label.pack(pady=(20, 5)) + pb = ctk.CTkProgressBar(self.state_frame, mode="indeterminate", width=250, fg_color="#F3F4F6", progress_color="#2563EB") + pb.pack(pady=(0, 20)) + pb.start() + self.state_frame.pack(fill="x", expand=True) + + def _set_state_error(self, error_msg): + for w in self.state_frame.winfo_children(): + w.destroy() + + display_msg = error_msg + if "401" in error_msg or "403" in error_msg or "unauthorized" in error_msg.lower() or "forbidden" in error_msg.lower(): + display_msg = "Reports telemetry permission required.\nPlease grant the 'Reports.Read.All' application permission to your App Registration in Entra ID." + + ctk.CTkLabel(self.state_frame, text=f"✖ {display_msg}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center").pack(pady=(20, 5)) + ctk.CTkButton(self.state_frame, text="Try Again", command=self._retry_fetch, width=120, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 20)) + self.state_frame.pack(fill="x", expand=True) + + def _retry_fetch(self): + if hasattr(self, 'reload_btn') and self.reload_btn.winfo_exists(): + self.reload_btn.configure(state="disabled") + tenant, clients, secrets = self.get_credentials() + if tenant: + self.trigger_fetch(tenant, clients[0], secrets[0]) + + def trigger_fetch(self, tenant, client_id, client_secret): + usage_logger.info("Active Users Usage trigger_fetch called. Spawning background worker thread...") + self.status = "loading" + self.on_status_change() + + self.pack(fill="x", expand=True, pady=10) + self.grid_frame.pack_forget() + + self._set_state_loading("Downloading and parsing O365 Active Users reports...") + + threading.Thread( + target=self._execute_worker, + args=(tenant, client_id, client_secret), + daemon=True + ).start() + + def _execute_worker(self, tenant: str, client_id: str, client_secret: str): + if self.semaphore: + self.semaphore.acquire() + try: + o365_data = run_o365_pipeline(client_id, client_secret, tenant) + usage_logger.info("Successfully completed O365 usage data fetch.") + self.after(0, self._render_success, o365_data) + except Exception as e: + usage_logger.error("Exception caught in ActiveUsersUsage worker.", exc_info=True) + self.after(0, self._render_error, str(e)) + finally: + if self.semaphore: + self.semaphore.release() + + def _render_success(self, o365_data: list): + self.last_data = o365_data + if hasattr(self, 'reload_btn') and self.reload_btn.winfo_exists(): + self.reload_btn.configure(state="normal") + self.state_frame.pack_forget() + for w in self.grid_frame.winfo_children(): + w.destroy() + self.current_page = 0 + + self.grid_frame.pack(fill="x", expand=True) + + self.grid_frame.grid_columnconfigure(0, weight=2) + self.grid_frame.grid_columnconfigure(1, weight=1) + self.grid_frame.grid_columnconfigure(2, weight=1) + self.grid_frame.grid_columnconfigure(3, weight=1) + + headers_o365 = ["Service", "30 Days", "90 Days", "180 Days"] + for col_idx, head_text in enumerate(headers_o365): + cell = ctk.CTkFrame(self.grid_frame, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + if not o365_data: + empty_cell = ctk.CTkFrame(self.grid_frame, fg_color="transparent") + empty_cell.grid(row=1, column=0, columnspan=4, sticky="nsew", pady=15) + ctk.CTkLabel(empty_cell, text="No O365 usage data found.", text_color=COLOR_TEXT_SUB).pack() + else: + for r_idx, row_data in enumerate(o365_data, start=1): + bg_style = COLOR_SURFACE if r_idx % 2 == 0 else COLOR_SURFACE_VARIANT + for c_idx, val in enumerate(row_data): + cell = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + cell.grid(row=r_idx, column=c_idx, sticky="nsew", padx=0, pady=(0, 1)) + fnt = FONT_BODY_BOLD if c_idx == 0 else FONT_BODY_MEDIUM + + display_val = f"{val:,}" if isinstance(val, int) else str(val) + ctk.CTkLabel(cell, text=display_val, font=fnt, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=8, anchor="nw") + + self.status = "success" + self.on_status_change() + + def _render_error(self, err_msg): + usage_logger.warning(f"Active Users Usage fetch failed: {err_msg}") + if hasattr(self, 'reload_btn') and self.reload_btn.winfo_exists(): + self.reload_btn.configure(state="normal") + self._set_state_error(err_msg) + self.status = "error" + self.on_status_change() + + def cancel(self): + pass # O365 API runs via reports CSV sync which doesn't support pagination-level cancel diff --git a/telemetry/m365_apps/active_users_trend.py b/telemetry/m365_apps/active_users_trend.py new file mode 100644 index 00000000..39cd18ad --- /dev/null +++ b/telemetry/m365_apps/active_users_trend.py @@ -0,0 +1,235 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""UI frame for O365 Active Users Trend telemetry.""" + +import os +import logging +import threading +import customtkinter as ctk + +# Safely import matplotlib +try: + from matplotlib.figure import Figure + from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg + MATPLOTLIB_AVAILABLE = True +except ImportError: + MATPLOTLIB_AVAILABLE = False + +from core.graph.m365_apps.active_users_trend import run_o365_trend_pipeline +from telemetry.styles import * + +usage_logger = logging.getLogger("M365TelemetryAsyncLogger.ActiveUsersTrendUI") + +class ActiveUsersTrendFrame(ctk.CTkFrame): + """Self-contained component wrapping O365 Active User Trend Chart and height controls.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, **kwargs): + self.semaphore = kwargs.pop("concurrency_semaphore", None) + super().__init__(master, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=12, **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.status = None # 'loading', 'success', 'error', None + self.ITEMS_PER_PAGE = 5 + self.current_page = 0 + self.last_data = None + self.trend_data = {} + + self.build_ui() + + def build_ui(self): + self.pack(fill="x", expand=True, pady=10) + + self.inner_pad = ctk.CTkFrame(self, fg_color="transparent") + self.inner_pad.pack(fill="both", expand=True, padx=20, pady=20) + + trend_header = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + trend_header.pack(fill="x", pady=(0, 10)) + + ctk.CTkLabel(trend_header, text="O365 30-Day Active User Trend", font=FONT_HEADER_SMALL, text_color=COLOR_TEXT_MAIN).pack(side="left") + + self.trend_height_var = ctk.DoubleVar(value=400) + + slider_frame = ctk.CTkFrame(trend_header, fg_color="transparent") + slider_frame.pack(side="right") + + self.lbl_trend_height = ctk.CTkLabel(slider_frame, text="Height: 400px", font=FONT_BODY_SMALL, text_color=COLOR_TEXT_SUB) + self.lbl_trend_height.pack(side="left", padx=(0, 10)) + + self.slider_trend_height = ctk.CTkSlider( + slider_frame, from_=200, to=800, number_of_steps=60, + variable=self.trend_height_var, width=120, height=16, + command=self._on_trend_height_slider_change + ) + self.slider_trend_height.pack(side="left") + + self.state_frame = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + + self.grid_frame = ctk.CTkFrame( + self.inner_pad, fg_color=COLOR_SURFACE, + border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8, + height=400 + ) + self.grid_frame.pack_propagate(False) + + self.reset_view() + + def reset_view(self): + self.pack_forget() + self.state_frame.pack_forget() + self.grid_frame.pack_forget() + self.trend_data = {} + + for w in self.state_frame.winfo_children(): + w.destroy() + for w in self.grid_frame.winfo_children(): + w.destroy() + self.current_page = 0 + self.last_data = None + + def _set_state_loading(self, msg="Loading..."): + for w in self.state_frame.winfo_children(): + w.destroy() + self.loading_label = ctk.CTkLabel(self.state_frame, text=f"⏳ {msg}", text_color="#6b7280", font=ctk.CTkFont(family="Segoe UI", size=13)) + self.loading_label.pack(pady=(20, 5)) + pb = ctk.CTkProgressBar(self.state_frame, mode="indeterminate", width=250, fg_color="#F3F4F6", progress_color="#2563EB") + pb.pack(pady=(0, 20)) + pb.start() + self.state_frame.pack(fill="x", expand=True) + + def _set_state_error(self, error_msg): + for w in self.state_frame.winfo_children(): + w.destroy() + + display_msg = error_msg + if "401" in error_msg or "403" in error_msg or "unauthorized" in error_msg.lower() or "forbidden" in error_msg.lower(): + display_msg = "Reports telemetry permission required.\nPlease grant the 'Reports.Read.All' application permission to your App Registration in Entra ID." + + ctk.CTkLabel(self.state_frame, text=f"✖ {display_msg}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center").pack(pady=(20, 5)) + ctk.CTkButton(self.state_frame, text="Try Again", command=self._retry_fetch, width=120, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 20)) + self.state_frame.pack(fill="x", expand=True) + + def _retry_fetch(self): + tenant, clients, secrets = self.get_credentials() + if tenant: + self.trigger_fetch(tenant, clients[0], secrets[0]) + + def trigger_fetch(self, tenant, client_id, client_secret): + usage_logger.info("Active Users Trend trigger_fetch called. Spawning background worker thread...") + self.status = "loading" + self.on_status_change() + + self.pack(fill="x", expand=True, pady=10) + self.grid_frame.pack_forget() + + self._set_state_loading("Downloading and parsing O365 Trend reports...") + + threading.Thread( + target=self._execute_worker, + args=(tenant, client_id, client_secret), + daemon=True + ).start() + + def _execute_worker(self, tenant: str, client_id: str, client_secret: str): + if self.semaphore: + self.semaphore.acquire() + try: + trend_data = run_o365_trend_pipeline(client_id, client_secret, tenant) + usage_logger.info("Successfully completed O365 trend data fetch.") + self.after(0, self._render_success, trend_data) + except Exception as e: + usage_logger.error("Exception caught in ActiveUsersTrend worker.", exc_info=True) + self.after(0, self._render_error, str(e)) + finally: + if self.semaphore: + self.semaphore.release() + + def _render_success(self, trend_data: dict): + self.trend_data = trend_data + self.last_data = trend_data + self.state_frame.pack_forget() + for w in self.grid_frame.winfo_children(): + w.destroy() + self.current_page = 0 + + self.grid_frame.pack(fill="both", expand=True) + + if not MATPLOTLIB_AVAILABLE: + empty_cell = ctk.CTkFrame(self.grid_frame, fg_color="transparent") + empty_cell.pack(fill="x", expand=True, pady=15) + ctk.CTkLabel(empty_cell, text="Matplotlib is required to render charts.\nPlease install it using 'pip install matplotlib'.", text_color=COLOR_ERROR).pack() + self.status = "error" + self.on_status_change() + return + + if not trend_data or not trend_data.get("dates"): + empty_cell = ctk.CTkFrame(self.grid_frame, fg_color="transparent") + empty_cell.pack(fill="x", expand=True, pady=15) + ctk.CTkLabel(empty_cell, text="No O365 trend data found.", text_color=COLOR_TEXT_SUB).pack() + else: + try: + fig = Figure(figsize=(8, 4), dpi=100) + ax = fig.add_subplot(111) + fig.patch.set_facecolor(COLOR_SURFACE) + ax.set_facecolor(COLOR_SURFACE) + + dates = trend_data["dates"] + + ax.plot(dates, trend_data["office365"], marker='o', label='Office 365') + ax.plot(dates, trend_data["exchange"], marker='o', label='Exchange') + ax.plot(dates, trend_data["onedrive"], marker='o', label='OneDrive') + ax.plot(dates, trend_data["sharepoint"], marker='o', label='SharePoint') + ax.plot(dates, trend_data["teams"], marker='o', label='Teams') + + ax.set_xlabel("Date", fontsize=10, color=COLOR_TEXT_SUB) + ax.set_ylabel("Active Users", fontsize=10, color=COLOR_TEXT_SUB) + + ax.tick_params(axis='x', colors=COLOR_TEXT_SUB, rotation=45, labelsize=8) + ax.tick_params(axis='y', colors=COLOR_TEXT_SUB) + + if len(dates) > 10: + ax.set_xticks(dates[::max(1, len(dates)//10)]) + + for spine in ax.spines.values(): + spine.set_color(COLOR_OUTLINE_LIGHT) + + ax.legend(facecolor=COLOR_SURFACE, edgecolor=COLOR_OUTLINE_LIGHT, labelcolor=COLOR_TEXT_MAIN, fontsize=9) + fig.tight_layout() + + canvas = FigureCanvasTkAgg(fig, master=self.grid_frame) + canvas.draw() + canvas.get_tk_widget().pack(fill="both", expand=True, padx=10, pady=10) + except Exception as e: + usage_logger.error(f"Error drawing matplotlib plot: {e}", exc_info=True) + empty_cell = ctk.CTkFrame(self.grid_frame, fg_color="transparent") + empty_cell.pack(fill="x", expand=True, pady=15) + ctk.CTkLabel(empty_cell, text="Failed to render trend graph (Matplotlib constraint).", text_color=COLOR_ERROR).pack() + + self.status = "success" + self.on_status_change() + + def _render_error(self, err_msg): + usage_logger.warning(f"Active Users Trend fetch failed: {err_msg}") + self._set_state_error(err_msg) + self.status = "error" + self.on_status_change() + + def _on_trend_height_slider_change(self, val): + height_val = int(val) + self.lbl_trend_height.configure(text=f"Height: {height_val}px") + self.grid_frame.configure(height=height_val) + + def cancel(self): + pass diff --git a/telemetry/m365_apps/app_usage.py b/telemetry/m365_apps/app_usage.py new file mode 100644 index 00000000..2b390cfa --- /dev/null +++ b/telemetry/m365_apps/app_usage.py @@ -0,0 +1,260 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""UI frame for M365 Apps Usage (180 Days) telemetry.""" + +import os +import logging +import threading +import customtkinter as ctk + +from core.graph.m365_apps.app_usage import run_m365_pipeline +from telemetry.styles import * + +usage_logger = logging.getLogger("M365TelemetryAsyncLogger.M365AppUsageUI") + +class M365AppUsageFrame(ctk.CTkFrame): + """Self-contained component wrapping M365 App Usage UI.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, **kwargs): + self.semaphore = kwargs.pop("concurrency_semaphore", None) + super().__init__(master, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=12, **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.status = None # 'loading', 'success', 'error', None + self.ITEMS_PER_PAGE = 5 + self.current_page = 0 + self.last_data = None + + self.build_ui() + + def build_ui(self): + self.pack(fill="x", expand=True, pady=10) + + self.inner_pad = ctk.CTkFrame(self, fg_color="transparent") + self.inner_pad.pack(fill="both", expand=True, padx=20, pady=20) + + self.header = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.header.pack(fill="x", pady=(0, 10)) + ctk.CTkLabel(self.header, text="M365 App Usage (180 Days)", font=FONT_HEADER_SMALL, text_color=COLOR_TEXT_MAIN).pack(side="left") + self.reload_btn = ctk.CTkButton( + self.header, + state="disabled", text="↻ Reload", + width=80, + height=24, + font=ctk.CTkFont(family="Segoe UI", size=12), + fg_color="transparent", + border_width=1, + text_color="#2563EB", + hover_color="#DBEAFE", + command=self._retry_fetch + ) + self.reload_btn.pack(side="right") + + self.state_frame = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.grid_frame = ctk.CTkFrame(self.inner_pad, fg_color=COLOR_OUTLINE_LIGHT, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + + self.reset_view() + + def reset_view(self): + self.pack_forget() + self.state_frame.pack_forget() + self.grid_frame.pack_forget() + + for w in self.state_frame.winfo_children(): + w.destroy() + for w in self.grid_frame.winfo_children(): + w.destroy() + self.current_page = 0 + self.last_data = None + + def _set_state_loading(self, msg="Loading..."): + for w in self.state_frame.winfo_children(): + w.destroy() + self.loading_label = ctk.CTkLabel(self.state_frame, text=f"⏳ {msg}", text_color="#6b7280", font=ctk.CTkFont(family="Segoe UI", size=13)) + self.loading_label.pack(pady=(20, 5)) + pb = ctk.CTkProgressBar(self.state_frame, mode="indeterminate", width=250, fg_color="#F3F4F6", progress_color="#2563EB") + pb.pack(pady=(0, 20)) + pb.start() + self.state_frame.pack(fill="x", expand=True) + + def _set_state_error(self, error_msg): + for w in self.state_frame.winfo_children(): + w.destroy() + + display_msg = error_msg + if "401" in error_msg or "403" in error_msg or "unauthorized" in error_msg.lower() or "forbidden" in error_msg.lower(): + display_msg = "Reports telemetry permission required.\nPlease grant the 'Reports.Read.All' application permission to your App Registration in Entra ID." + + ctk.CTkLabel(self.state_frame, text=f"✖ {display_msg}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center").pack(pady=(20, 5)) + ctk.CTkButton(self.state_frame, text="Try Again", command=self._retry_fetch, width=120, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 20)) + self.state_frame.pack(fill="x", expand=True) + + def _retry_fetch(self): + if hasattr(self, 'reload_btn') and self.reload_btn.winfo_exists(): + self.reload_btn.configure(state="disabled") + tenant, clients, secrets = self.get_credentials() + if tenant: + self.trigger_fetch(tenant, clients[0], secrets[0]) + + def trigger_fetch(self, tenant, client_id, client_secret): + usage_logger.info("M365 App Usage trigger_fetch called. Spawning background worker thread...") + self.status = "loading" + self.on_status_change() + + self.pack(fill="x", expand=True, pady=10) + self.grid_frame.pack_forget() + + self._set_state_loading("Downloading and parsing M365 App Usage reports...") + + threading.Thread( + target=self._execute_worker, + args=(tenant, client_id, client_secret), + daemon=True + ).start() + + def _execute_worker(self, tenant: str, client_id: str, client_secret: str): + if self.semaphore: + self.semaphore.acquire() + try: + m365_data = run_m365_pipeline(client_id, client_secret, tenant) + usage_logger.info("Successfully completed M365 Apps usage data fetch.") + self.after(0, self._render_success, m365_data) + except Exception as e: + usage_logger.error("Exception caught in M365AppUsage worker.", exc_info=True) + self.after(0, self._render_error, str(e)) + finally: + if self.semaphore: + self.semaphore.release() + + def _render_success(self, m365_data: list): + if hasattr(self, 'reload_btn') and self.reload_btn.winfo_exists(): + self.reload_btn.configure(state="normal") + self.state_frame.pack_forget() + self.grid_frame.pack(fill="x") + + self.last_data = m365_data + self.current_page = 0 + self._update_ui_paginated() + + self.status = "success" + self.on_status_change() + + def _update_ui_paginated(self, data=None): + if data is None: + data = self.last_data + + for w in self.grid_frame.winfo_children(): + info = w.grid_info() + if "row" in info and int(info["row"]) > 0: + w.destroy() + + for i in range(2): + self.grid_frame.grid_columnconfigure(i, weight=1) + self.grid_frame.grid_columnconfigure(2, weight=0) + self.grid_frame.grid_columnconfigure(3, weight=0) + + headers = ["App / Platform", "Users Count"] + + if not self.grid_frame.winfo_children(): + for col_idx, head_text in enumerate(headers): + cell = ctk.CTkFrame(self.grid_frame, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + total_count = len(data) if data else 0 + start_idx = self.current_page * self.ITEMS_PER_PAGE + end_idx = start_idx + self.ITEMS_PER_PAGE + page_data = data[start_idx:end_idx] if data else [] + + if not data: + empty_cell = ctk.CTkFrame(self.grid_frame, fg_color="transparent") + empty_cell.grid(row=1, column=0, columnspan=2, sticky="nsew", pady=15) + ctk.CTkLabel(empty_cell, text="No M365 App usage data found.", text_color=COLOR_TEXT_SUB).pack() + else: + for r_idx, (platform, count) in enumerate(page_data, start=1): + bg_style = COLOR_SURFACE if r_idx % 2 != 0 else COLOR_SURFACE_VARIANT + vals = [platform, count] + for c_idx, val in enumerate(vals): + cell = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + cell.grid(row=r_idx, column=c_idx, sticky="nsew", padx=0, pady=(0, 1)) + fnt = FONT_BODY_BOLD if c_idx == 0 else FONT_BODY_MEDIUM + + display_val = f"{val:,}" if isinstance(val, int) else str(val) + ctk.CTkLabel(cell, text=display_val, font=fnt, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=6, anchor="nw") + + self._draw_pagination_controls(total_count, data) + + def _draw_pagination_controls(self, total_count, data): + total_pages = max(1, (total_count + self.ITEMS_PER_PAGE - 1) // self.ITEMS_PER_PAGE) + + # Check if control frame already exists to avoid duplication + control_frame = None + for w in self.grid_frame.winfo_children(): + info = w.grid_info() + if "row" in info and int(info["row"]) == self.ITEMS_PER_PAGE + 1: + control_frame = w + break + + if not control_frame: + control_frame = ctk.CTkFrame(self.grid_frame, fg_color=COLOR_SURFACE) + control_frame.grid(row=self.ITEMS_PER_PAGE + 1, column=0, columnspan=2, pady=0, sticky="ew") + else: + for w in control_frame.winfo_children(): + w.destroy() + + left_spacer = ctk.CTkFrame(control_frame, fg_color="transparent") + left_spacer.pack(side="left", fill="x", expand=True) + + center_container = ctk.CTkFrame(control_frame, fg_color="transparent") + center_container.pack(side="left") + + prev_state = "normal" if self.current_page > 0 else "disabled" + btn_prev = ctk.CTkButton( + center_container, text="◀ Prev", width=70, height=22, corner_radius=6, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, + state=prev_state, + command=lambda: self._change_page(-1, data) + ) + btn_prev.pack(side="left", padx=5) + + page_lbl = ctk.CTkLabel( + center_container, + text=f"Page {self.current_page + 1} of {total_pages}", + font=FONT_BODY_MEDIUM, + text_color=COLOR_TEXT_SUB + ) + page_lbl.pack(side="left", padx=15) + + next_state = "normal" if self.current_page < total_pages - 1 else "disabled" + btn_next = ctk.CTkButton( + center_container, text="Next ▶", width=70, height=22, corner_radius=6, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, + state=next_state, + command=lambda: self._change_page(1, data) + ) + btn_next.pack(side="left", padx=5) + + right_spacer = ctk.CTkFrame(control_frame, fg_color="transparent") + right_spacer.pack(side="right", fill="x", expand=True) + + def _change_page(self, delta, data): + self.current_page += delta + self._update_ui_paginated(data) + + def cancel(self): + pass diff --git a/telemetry/m365_telemetry.py b/telemetry/m365_telemetry.py new file mode 100644 index 00000000..84cce0e5 --- /dev/null +++ b/telemetry/m365_telemetry.py @@ -0,0 +1,1126 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Master coordinator tab and logging setup for Microsoft 365 tenant telemetry dashboard.""" + +import os +import sys +import time +import queue +import logging +import csv +import psutil +import threading +from typing import Any, Dict, List, Optional +import customtkinter as ctk + +# Performance optimizations for CustomTkinter across OS +ctk.set_window_scaling(1.0) +ctk.set_widget_scaling(1.0) +from tkinter import messagebox +from logging.handlers import QueueHandler, QueueListener + +# Import modular view frames from their consolidated code/view modules +from telemetry.subscribed_skus import SubscribedSKUsFrame +from telemetry.directory import DirectoryFrame +from telemetry.m365_apps import M365AppsTelemetryFrame +from telemetry.power_automate import PowerAutomateUsageFrame + + +# Import existing modular views +from telemetry.files.msteams_overview import MsTeamsOverviewFrame +from telemetry.files import FilesTelemetryFrame +from telemetry.devices_apps_telemetry import DevicesAppsTelemetryFrame +from telemetry.email_client_support import EmailClientSupportFrame +from telemetry.exchange import ExchangeOnlineFrame +from telemetry.data_security_governance import DataSecurityGovernanceFrame +from telemetry.intune import IntunePoliciesFrame + + +from telemetry.styles import * + + +# ================================================================================= +# ASYNC FILE LOGGING SETUP +# ================================================================================= + +_current_dir = os.path.dirname(os.path.abspath(__file__)) +_log_queue = queue.Queue(-1) +_formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +_file_handler = None +_queue_listener = None + +# Configure the root logger to send all records to the log queue and clear other handlers (stdout/stderr) +root_logger = logging.getLogger() +root_logger.setLevel(logging.WARNING) +for h in list(root_logger.handlers): + root_logger.removeHandler(h) +root_logger.addHandler(QueueHandler(_log_queue)) + +# Set logger level to INFO for our own application packages/loggers +async_logger = logging.getLogger("M365TelemetryAsyncLogger") +async_logger.setLevel(logging.INFO) +async_logger.propagate = True + +logging.getLogger("core").setLevel(logging.INFO) +logging.getLogger("util").setLevel(logging.INFO) +logging.getLogger("chat").setLevel(logging.INFO) +logging.getLogger("PowerShellClient").setLevel(logging.INFO) + +# Global unhandled exception & sys.stderr logging redirection setup +_stderr_logger = logging.getLogger("M365TelemetryAsyncLogger.sys.stderr") + +class LoggedStderr: + def __init__(self, logger): + self.logger = logger + self.line_buffer = "" + self.original_stderr = sys.__stderr__ + + def write(self, buf): + self.original_stderr.write(buf) + for line in buf.splitlines(keepends=True): + self.line_buffer += line + if self.line_buffer.endswith('\n'): + stripped = self.line_buffer.rstrip('\r\n') + if stripped: + self.logger.error(stripped) + self.line_buffer = "" + + def flush(self): + self.original_stderr.flush() + if self.line_buffer: + stripped = self.line_buffer.rstrip('\r\n') + if stripped: + self.logger.error(stripped) + self.line_buffer = "" + +sys.stderr = LoggedStderr(_stderr_logger) + +def handle_unhandled_exception(exc_type, exc_value, exc_traceback): + if issubclass(exc_type, KeyboardInterrupt): + sys.__excepthook__(exc_type, exc_value, exc_traceback) + return + _stderr_logger.critical("Unhandled system exception", exc_info=(exc_type, exc_value, exc_traceback)) + +sys.excepthook = handle_unhandled_exception + +def handle_thread_exception(args): + _stderr_logger.critical( + f"Unhandled thread exception in {args.thread.name if args.thread else 'unknown thread'}", + exc_info=(args.exc_type, args.exc_value, args.exc_traceback) + ) + +threading.excepthook = handle_thread_exception + + +def update_log_directory(tenant_id: Optional[str] = None, client_id: Optional[str] = None) -> None: + """Updates the log directory dynamically once tenant and client ID are known.""" + global _file_handler, _queue_listener + + try: + if _queue_listener: + _queue_listener.stop() + except Exception: + pass + + try: + if _file_handler: + _file_handler.close() + except Exception: + pass + + if not tenant_id or not client_id: + return + + new_log_dir = os.path.join(_current_dir, 'logs', f"{tenant_id}_{client_id}") + os.makedirs(new_log_dir, exist_ok=True) + new_log_file_path = os.path.join(new_log_dir, 'telemetry_log.txt') + + _file_handler = logging.FileHandler(new_log_file_path, mode='a', encoding='utf-8') + _file_handler.setFormatter(_formatter) + + _queue_listener = QueueListener(_log_queue, _file_handler) + _queue_listener.start() + + +# ================================================================================= +# MAIN TAB COORDINATOR +# ================================================================================= + +class M365TelemetryTab(ctk.CTkScrollableFrame): + """Encapsulates the UI coordinator for the Microsoft 365 Telemetry & Audit dashboard tab.""" + + def __init__(self, master, log_callback, retries_var, backoff_var, **kwargs): + super().__init__( + master, + fg_color="transparent", + scrollbar_button_color="white", + scrollbar_button_hover_color=COLOR_SECONDARY_HOVER, + **kwargs + ) + async_logger.info("Initializing M365TelemetryTab instance.") + + self.log_msg = log_callback + self.retries = retries_var + self.backoff = backoff_var + + self.lic_tenant_id = ctk.StringVar() + self.lic_client_ids = ctk.StringVar() + self.lic_client_secrets = ctk.StringVar() + self.use_delegated_auth = ctk.BooleanVar(value=False) + + self.on_all_done_callback = None + self.telemetry_semaphore = threading.Semaphore(3) + self.is_fetching = False + + self.build_ui() + + self.batches = [ + [self.subscribed_skus_view], + [self.directory_view], + [self.m365_apps_view], + [self.exchange_online_view], + [self.files_view, self.msteams_overview_view], + [self.devices_apps_view, self.intune_policies_view], + [self.network_security_view], + [self.security_gov_view], + [self.power_automate_view] + ] + self.current_batch_index = 0 + + # Bind mouse wheel globally to scroll this tab when hovered + self.bind_all("", self._handle_global_mousewheel, add="+") + self.bind_all("", self._handle_global_mousewheel, add="+") + self.bind_all("", self._handle_global_mousewheel, add="+") + + # Start a background daemon thread to monitor memory consumption every 30s + self.mem_monitor_active = True + self.mem_monitor_thread = threading.Thread(target=self._monitor_memory_loop, daemon=True) + self.mem_monitor_thread.start() + + def _create_entry(self, parent, label, var, show=None): + f = ctk.CTkFrame(parent, fg_color="transparent") + f.pack(fill="x", pady=5) + ctk.CTkLabel(f, text=label, width=100, anchor="w", text_color=COLOR_TEXT_SUB).pack(side="left") + ctk.CTkEntry( + f, textvariable=var, show=show, height=40, corner_radius=4, + border_width=1, border_color=COLOR_OUTLINE, fg_color="transparent", + text_color=COLOR_TEXT_MAIN, + ).pack(side="left", fill="x", expand=True) + + def build_ui(self): + async_logger.info("Building graphical UI elements for M365 Telemetry Tab.") + + ctk.CTkLabel(self, text="Connect your Microsoft Azure account to authenticate and audit tenant licensing bundle inventories and usage.", font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_SUB).pack(anchor="w", pady=(0, 15)) + + self.inputs_frame = ctk.CTkFrame(self, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + self.inputs_frame.pack(fill="x", pady=5) + + inner_pad = ctk.CTkFrame(self.inputs_frame, fg_color="transparent") + inner_pad.pack(fill="x", padx=15, pady=15) + + self._create_entry(inner_pad, "Tenant ID", self.lic_tenant_id) + self._create_entry(inner_pad, "Client ID", self.lic_client_ids) + self._create_entry(inner_pad, "Client Secret", self.lic_client_secrets, show="*") + + actions_frame = ctk.CTkFrame(self, fg_color="transparent") + actions_frame.pack(fill="x", pady=(20, 5)) + + self.btn_lic_submit = ctk.CTkButton( + actions_frame, text="Submit", width=160, height=40, corner_radius=20, + font=FONT_BODY_BOLD, fg_color=COLOR_PRIMARY, hover_color=COLOR_PRIMARY_HOVER, + command=self.authenticate_licenses_tab, + ) + self.btn_lic_submit.pack(side="left") + + self.lbl_lic_status = ctk.CTkLabel(actions_frame, text="", font=FONT_BODY_MEDIUM) + self.lbl_lic_status.pack(side="left", padx=20) + + # ---------------------------------------------------- + # MODULAR UI SECTIONS + # ---------------------------------------------------- + + # 1. Subscribed SKUs Section + self.subscribed_skus_view = SubscribedSKUsFrame( + master=self, + log_callback=self.log_msg, + credentials_callback=self._get_credentials, + status_change_callback=self._check_all_done, + retries_var=self.retries, + backoff_var=self.backoff, + concurrency_semaphore=self.telemetry_semaphore + ) + + # 5d. Devices & Apps Section (Microsoft Entra Data) + self.devices_apps_view = DevicesAppsTelemetryFrame( + master=self, + log_callback=self.log_msg, + credentials_callback=self._get_credentials, + status_change_callback=self._check_all_done, + concurrency_semaphore=self.telemetry_semaphore + ) + + # 1b. Directory Groups Section + self.directory_view = DirectoryFrame( + master=self, + log_callback=self.log_msg, + credentials_callback=self._get_credentials, + status_change_callback=self._check_all_done, + retries_var=self.retries, + backoff_var=self.backoff, + concurrency_semaphore=self.telemetry_semaphore + ) + + # 2. M365 Apps Section (Uber Container) + self.m365_apps_view = M365AppsTelemetryFrame( + master=self, + log_callback=self.log_msg, + credentials_callback=self._get_credentials, + status_change_callback=self._check_all_done, + concurrency_semaphore=self.telemetry_semaphore + ) + + # 5. Exchange Online Usage Section + self.exchange_online_view = ExchangeOnlineFrame( + master=self, + log_callback=self.log_msg, + credentials_callback=self._get_credentials, + status_change_callback=self._check_all_done, + concurrency_semaphore=self.telemetry_semaphore + ) + + + # 5c. Files (SharePoint & OneDrive) Section + self.files_view = FilesTelemetryFrame( + master=self, + log_callback=self.log_msg, + credentials_callback=self._get_credentials, + status_change_callback=self._check_all_done, + concurrency_semaphore=self.telemetry_semaphore + ) + + # 5d. MsTeams Overview Section + self.msteams_overview_view = MsTeamsOverviewFrame( + master=self, + log_callback=self.log_msg, + credentials_callback=self._get_credentials, + status_change_callback=self._check_all_done, + semaphore=self.telemetry_semaphore + ) + + # 5e. Network Security Section + from telemetry.network_security import NetworkSecurityFrame + self.network_security_view = NetworkSecurityFrame( + master=self, + log_callback=self.log_msg, + credentials_callback=self._get_credentials, + status_change_callback=self._check_all_done, + concurrency_semaphore=self.telemetry_semaphore + ) + + # 6. Data Security & Governance Section + self.security_gov_view = DataSecurityGovernanceFrame( + master=self, + log_callback=self.log_msg, + credentials_callback=self._get_credentials, + status_change_callback=self._check_all_done, + concurrency_semaphore=self.telemetry_semaphore, + delegated_auth_callback=self._get_delegated_auth_state + ) + + # 6.5. Intune Policies Section + self.intune_policies_view = IntunePoliciesFrame( + master=self, + log_callback=self.log_msg, + credentials_callback=self._get_credentials, + status_change_callback=self._check_all_done, + concurrency_semaphore=self.telemetry_semaphore, + delegated_auth_callback=self._get_delegated_auth_state + ) + + + # 7. Power Automate Section + self.power_automate_view = PowerAutomateUsageFrame( + master=self, + log_callback=self.log_msg, + credentials_callback=self._get_credentials, + status_change_callback=self._check_all_done, + concurrency_semaphore=self.telemetry_semaphore + ) + + + # Bind mouse wheel globally to scroll this tab when hovered + self.bind_all("", self._handle_global_mousewheel, add="+") + self.bind_all("", self._handle_global_mousewheel, add="+") + self.bind_all("", self._handle_global_mousewheel, add="+") + + self._hide_all_grids() + + # Wrap all leaf views to support cancellation and prevent race conditions + for leaf in self._get_all_leaf_views(): + self._wrap_view_for_cancellation(leaf) + + def _hide_all_grids(self): + views = [ + self.subscribed_skus_view, + self.directory_view, + self.m365_apps_view, + self.exchange_online_view, + self.files_view, + self.msteams_overview_view, + self.devices_apps_view, + self.network_security_view, + self.security_gov_view, + self.intune_policies_view, + self.power_automate_view + ] + for view in views: + view.reset_view() + view.status = None + + def reset_tab(self): + """Resets the coordinator status, credentials variables, submission button, and hides all grids.""" + if getattr(self, "is_fetching", False): + self.cancel_fetching() + + async_logger.info("Resetting M365TelemetryTab coordinator and hiding all sub-grids.") + self.lic_tenant_id.set("") + self.lic_client_ids.set("") + self.lic_client_secrets.set("") + self.btn_lic_submit.configure(state="normal", text="Submit") + self.lbl_lic_status.configure(text="") + self.current_batch_index = 0 + self._hide_all_grids() + + def _get_credentials(self): + tenant = self.lic_tenant_id.get().strip() + client_str = self.lic_client_ids.get().strip() + secret_str = self.lic_client_secrets.get().strip() + + if not tenant or not client_str or not secret_str: + return None, None, None + + clients = [x.strip() for x in client_str.split(",") if x.strip()] + secrets = [x.strip() for x in secret_str.split(",") if x.strip()] + return tenant, clients, secrets + + def _get_delegated_auth_state(self): + return self.use_delegated_auth.get() + + def _check_all_done(self): + """Checks if all sections of the current batch have resolved, then triggers next batch or finishes.""" + if not getattr(self, "is_fetching", False): + # If fetching was cancelled, ignore any further background thread completions + return + + if not hasattr(self, "batches"): + return + + current_views = self.batches[self.current_batch_index] + batch_states = [view.status for view in current_views if view not in [self.devices_apps_view, self.intune_policies_view]] + + if "loading" in batch_states: + return + + # Current batch completed. Check if there are more batches to run + if self.current_batch_index < len(self.batches) - 1: + self.current_batch_index += 1 + self.trigger_current_batch() + return + + # All batches have finished. Make sure no individual retries are still loading + all_views = [view for batch in self.batches for view in batch] + global_states = [v.status for v in all_views if v not in [self.devices_apps_view, self.intune_policies_view]] + if "loading" in global_states: + return + + # Re-enable the submit button + self.is_fetching = False + self.btn_lic_submit.configure(state="normal", text="Submit") + + success = all(s == "success" for s in global_states) + if success: + self.lbl_lic_status.configure(text="✔ All Inventory and Usage Reports Pulled Successfully!", text_color=COLOR_SUCCESS) + else: + self.lbl_lic_status.configure(text="⚠ Some reports failed. Please retry individually.", text_color=COLOR_ERROR) + + if hasattr(self, "on_all_done_callback") and self.on_all_done_callback: + self.on_all_done_callback(success) + + def trigger_current_batch(self): + """Triggers the fetches for the current batch of sections.""" + tenant, clients, secrets = self._get_credentials() + if not tenant: + return + + current_views = self.batches[self.current_batch_index] + async_logger.info(f"Triggering batch {self.current_batch_index + 1} with {len(current_views)} views.") + + for view in current_views: + if isinstance(view, SubscribedSKUsFrame): + view.trigger_fetch(tenant, clients, secrets) + else: + view.trigger_fetch(tenant, clients[0], secrets[0]) + + def authenticate_licenses_tab(self): + """Master full sequential fetch of sections, or cancel if already fetching.""" + if getattr(self, "is_fetching", False): + self.cancel_fetching() + return + + async_logger.info("Master Submit triggered. Restarting all fetches sequentially.") + + tenant, clients, secrets = self._get_credentials() + if not tenant: + async_logger.warning("Authentication aborted: Missing credential parameters.") + messagebox.showerror("Credential Error", "Please provide complete Tenant ID, Client ID, and Client Secret strings.", parent=self) + return + + self.is_fetching = True + self.btn_lic_submit.configure(state="normal", text="Cancel") + self.lbl_lic_status.configure(text="Querying Microsoft Graph APIs and Reports sequentially...", text_color=COLOR_TEXT_SUB) + + # Reset all views first + self._hide_all_grids() + + self.current_batch_index = 0 + self.trigger_current_batch() + + def cancel_fetching(self): + """Cancels the current fetching process and stops all subsequent batches.""" + async_logger.info("Cancellation triggered by user.") + self.is_fetching = False + + self.btn_lic_submit.configure(state="normal", text="Submit") + self.lbl_lic_status.configure(text="✖ Query cancelled by user.", text_color=COLOR_ERROR) + + # Propagate cancel to all top-level views + for batch in self.batches: + for view in batch: + if hasattr(view, "cancel"): + view.cancel() + + if hasattr(self, "on_all_done_callback") and self.on_all_done_callback: + self.on_all_done_callback(False) + + def _get_all_leaf_views(self): + """Returns a list of all active leaf/base telemetry views across all cards.""" + return [ + self.subscribed_skus_view, + self.devices_apps_view.auth_methods_subframe, + self.devices_apps_view.app_registrations_subframe, + self.devices_apps_view.app_signins_subframe, + self.devices_apps_view.user_signins_subframe, + self.directory_view.organization_frame, + self.directory_view.domains_frame, + self.directory_view.user_logs_frame, + self.directory_view.provisioning_logs_frame, + self.directory_view.users_groups_frame, + self.m365_apps_view.active_users_view, + self.m365_apps_view.active_users_trend_view, + self.m365_apps_view.m365_apps_view, + self.exchange_online_view.mailbox_view, + self.exchange_online_view.calendar_view, + self.exchange_online_view.apps_view, + self.exchange_online_view.mail_security_view, + self.exchange_online_view.transport_rules_view, + self.exchange_online_view.connectors_view, + self.exchange_online_view.email_clients_view, + self.exchange_online_view.pst_files_view, + self.files_view.sharepoint_view, + self.files_view.onedrive_view, + self.msteams_overview_view, + self.network_security_view.filtering_view, + self.network_security_view.ca_view, + self.network_security_view.fw_view, + self.security_gov_view.sensitivity_frame, + self.security_gov_view.retention_frame, + self.security_gov_view.dlp_frame, + self.security_gov_view.sit_frame, + self.security_gov_view.auth_frame, + self.security_gov_view.sso_frame, + self.security_gov_view.ediscovery_ui_view, + self.intune_policies_view.mobile_apps_view, + self.intune_policies_view.detected_apps_view, + self.intune_policies_view.device_configs_view, + self.power_automate_view + ] + + def _find_main_title_label(self, view): + """Recursively searches the widget tree to identify the exact header/title label of a leaf view.""" + known_titles = { + "SubscribedSKUsFrame": "Subscribed SKUs", + "DirectoryFrame": "Directory Summary", + "DirectoryOrganizationFrame": "Organization", + "DirectoryDomainsFrame": "Domains", + "DirectoryUserLogsFrame": "User Creation/Deletion logs", + "DirectoryProvisioningLogsFrame": "Provisioning Logs", + "DirectoryUsersGroupsFrame": "Groups & Users", + "ActiveUsersUsageFrame": "Active Users Usage", + "ActiveUsersTrendFrame": "Active Users Trend", + "M365AppUsageFrame": "M365 App Usage", + "MailboxUsageFrame": "Exchange Online Mailbox Usage", + "CalendarTelemetryFrame": "Exchange Online Calendar Environment", + "ExchangeAppsFrame": "Integrated Apps", + "ExchangeConnectorsFrame": "Exchange Connectors (Inbound & Outbound Routing)", + "MailSecurityFrame": "Mail Security", + "TransportRulesFrame": "Exchange Transport Rules", + "EmailClientSupportFrame": "Email Client Classification", + "PstFilesFrame": "PST Files", + "SharePointUsageFrame": "SharePoint Online Sites & Files Summary", + "SharePointDataTypesFrame": "SharePoint Data Types (Tenant Wide)", + "OneDriveUsageFrame": "OneDrive for Business Personal Accounts Summary", + "MsTeamsOverviewFrame": "Microsoft Teams Overview (180 Days)", + "FilteringPoliciesSubFrame": "Filtering Policies", + "ConditionalAccessSubFrame": "Conditional Access Policies", + "FirewallSubFrame": "Firewall and Proxy Configurations", + "DevicesAppsTelemetryFrame": "Devices & Apps Summary (Sign-in Telemetry)", + "AppRegistrationsSubFrame": "App Registrations", + "SensitivityLabelsSubFrame": "Sensitivity Labels", + "RetentionPoliciesSubFrame": "Retention Compliance Policies", + "DLPPoliciesSubFrame": "Data Loss Prevention (DLP) Policies", + "SensitiveInfoTypesSubFrame": "Sensitive Information Types (SIT)", + "AuthenticationSubFrame": "Authentication Mechanics (Conditional Access)", + "ServicePrincipalsSsoSubFrame": "Service Principals Single Sign-On (SSO) Modes", + "EDiscoveryFrame": "Microsoft Purview eDiscovery Cases", + "MobileAppsSubFrame": "Managed Mobile Apps", + "DetectedAppsSubFrame": "Detected Apps", + "ManagedDevicesSubFrame": "Managed Devices", + "DeviceConfigsSubFrame": "Device Configurations", + "MdmPoliciesSubFrame": "Mobile Device Management Policies", + "ByodConfigsSubFrame": "Mobile BYOD Configurations", + "PowerAutomateUsageFrame": "Power Automate (Workflows & Flows)" + } + + target_text = known_titles.get(view.__class__.__name__, "") + + def search(widget): + if isinstance(widget, ctk.CTkLabel): + try: + text = widget.cget("text") + if text == target_text or (target_text and target_text in text): + return widget + except Exception: + pass + if hasattr(widget, "winfo_children"): + for child in widget.winfo_children(): + res = search(child) + if res: + return res + return None + + lbl = search(view) + if lbl: + return lbl + + # Fallback: search for the first CTkLabel + def first_label(widget): + if isinstance(widget, ctk.CTkLabel): + return widget + if hasattr(widget, "winfo_children"): + for child in widget.winfo_children(): + res = first_label(child) + if res: + return res + return None + return first_label(view) + + def _wrap_view_for_cancellation(self, view): + """Wraps a leaf view's trigger, render/handle, reset and after methods dynamically to enforce thread safety, cancellation, stale thread filtering, and precise execution time tracking.""" + view.current_request_id = 0 + view.is_cancelled = False + view.fetch_time_lbl = None + view.fetch_start_time = 0.0 + view.sub_section_start_times = {} + view.sub_section_timer_labels = {} + + orig_trigger = view.trigger_fetch + orig_reset = view.reset_view + orig_after = view.after + + # Wrap the semaphore if present to capture the exact start time after acquisition + if getattr(view, "semaphore", None): + orig_sem = view.semaphore + + class WrappedSemaphore: + def __init__(self, sem): + self._sem = sem + self._acquired_threads = set() + def acquire(self, *args, **kwargs): + res = self._sem.acquire(*args, **kwargs) + cur_thread = threading.current_thread() + thread_req_id = getattr(cur_thread, "request_id", None) + if thread_req_id is not None and thread_req_id < view.current_request_id: + self._sem.release() + raise InterruptedError(f"Thread execution cancelled (stale request: {thread_req_id} < {view.current_request_id}).") + self._acquired_threads.add(cur_thread.ident) + sub_sec = getattr(cur_thread, "sub_section", None) + if sub_sec: + view.sub_section_start_times[sub_sec] = time.time() + else: + view.fetch_start_time = time.time() + return res + def release(self, *args, **kwargs): + cur_thread = threading.current_thread() + if cur_thread.ident in self._acquired_threads: + self._acquired_threads.remove(cur_thread.ident) + return self._sem.release(*args, **kwargs) + return None + def __getattr__(self, name): + return getattr(self._sem, name) + + view.semaphore = WrappedSemaphore(orig_sem) + + def display_fetch_time(elapsed): + # Destroy old label if exists + if hasattr(view, "fetch_time_lbl") and view.fetch_time_lbl: + try: + view.fetch_time_lbl.destroy() + except Exception: + pass + view.fetch_time_lbl = None + + # Create a new floating label next to the title + view.fetch_time_lbl = ctk.CTkLabel( + view, + text=f"⏱ {elapsed:.2f}s", + font=ctk.CTkFont(family="Segoe UI", size=11, weight="bold"), + text_color=COLOR_PRIMARY + ) + + # Automatically pack the timer into the header so Tkinter resolves layout clashes with buttons + header_target = getattr(view, "lic_header", getattr(view, "pa_header", getattr(view, "header", getattr(view, "header_frame", None)))) + if header_target: + view.fetch_time_lbl.pack( + in_=header_target, + side="right", + padx=(0, 15) + ) + else: + # Place at top right of the card container inline with text (for views with no buttons) + view.fetch_time_lbl.place(relx=0.98, rely=0.0, anchor="ne", y=20) + + def display_sub_section_time(sub_sec, header_frame, elapsed): + # Destroy old label if exists + if hasattr(view, "sub_section_timer_labels") and sub_sec in view.sub_section_timer_labels: + lbl = view.sub_section_timer_labels[sub_sec] + if lbl: + try: + lbl.destroy() + except Exception: + pass + view.sub_section_timer_labels[sub_sec] = None + + # Create a new floating label, parented to the view to avoid clipping + lbl = ctk.CTkLabel( + view, + text=f"⏱ {elapsed:.2f}s", + font=ctk.CTkFont(family="Segoe UI", size=11, weight="bold"), + text_color=COLOR_PRIMARY + ) + + # Pack safely to the right side of the header frame (auto-avoids export buttons) + lbl.pack( + in_=header_frame, + side="right", + padx=(0, 15) + ) + + view.sub_section_timer_labels[sub_sec] = lbl + + # Determine which rendering method is present + has_render = hasattr(view, "_render_success") and hasattr(view, "_render_error") + has_handle = hasattr(view, "_handle_result") + is_security_gov = hasattr(view, "_handle_labels_result") + + if is_security_gov: + view.sub_section_start_times = {} + view.sub_section_timer_labels = {} + + orig_labels_handle = view._handle_labels_result + orig_retention_handle = view._handle_retention_result + orig_dlp_handle = view._handle_dlp_result + orig_sit_handle = view._handle_sit_result + orig_auth_handle = view._handle_auth_result + orig_sso_handle = view._handle_sso_result + + def new_labels_handle(*args, **kwargs): + if view.is_cancelled: + return + elapsed = time.time() - view.sub_section_start_times.get("labels", time.time()) + display_sub_section_time("labels", view.labels_header_frame, elapsed) + orig_labels_handle(*args, **kwargs) + + def new_retention_handle(*args, **kwargs): + if view.is_cancelled: + return + elapsed = time.time() - view.sub_section_start_times.get("retention", time.time()) + display_sub_section_time("retention", view.retention_header_frame, elapsed) + orig_retention_handle(*args, **kwargs) + + def new_dlp_handle(*args, **kwargs): + if view.is_cancelled: + return + elapsed = time.time() - view.sub_section_start_times.get("dlp", time.time()) + display_sub_section_time("dlp", view.dlp_header_frame, elapsed) + orig_dlp_handle(*args, **kwargs) + + def new_sit_handle(*args, **kwargs): + if view.is_cancelled: + return + elapsed = time.time() - view.sub_section_start_times.get("sit", time.time()) + display_sub_section_time("sit", view.sit_header_frame, elapsed) + orig_sit_handle(*args, **kwargs) + + def new_auth_handle(*args, **kwargs): + if view.is_cancelled: + return + elapsed = time.time() - view.sub_section_start_times.get("auth", time.time()) + display_sub_section_time("auth", view.auth_header_frame, elapsed) + orig_auth_handle(*args, **kwargs) + + def new_sso_handle(*args, **kwargs): + if view.is_cancelled: + return + elapsed = time.time() - view.sub_section_start_times.get("sso", time.time()) + display_sub_section_time("sso", view.sso_header_frame, elapsed) + orig_sso_handle(*args, **kwargs) + + view._handle_labels_result = new_labels_handle + view._handle_retention_result = new_retention_handle + view._handle_dlp_result = new_dlp_handle + view._handle_sit_result = new_sit_handle + view._handle_auth_result = new_auth_handle + view._handle_sso_result = new_sso_handle + + if has_render: + orig_success = view._render_success + orig_error = view._render_error + + def new_success(*args, **kwargs): + if view.is_cancelled: + async_logger.info(f"Ignored _render_success for {view.__class__.__name__} because it was cancelled.") + return + elapsed = time.time() - getattr(view, "fetch_start_time", time.time()) + display_fetch_time(elapsed) + orig_success(*args, **kwargs) + + def new_error(*args, **kwargs): + if view.is_cancelled: + async_logger.info(f"Ignored _render_error for {view.__class__.__name__} because it was cancelled.") + return + elapsed = time.time() - getattr(view, "fetch_start_time", time.time()) + display_fetch_time(elapsed) + orig_error(*args, **kwargs) + + view._render_success = new_success + view._render_error = new_error + + elif has_handle: + orig_handle = view._handle_result + + def new_handle(*args, **kwargs): + if view.is_cancelled: + async_logger.info(f"Ignored _handle_result for {view.__class__.__name__} because it was cancelled.") + return + elapsed = time.time() - getattr(view, "fetch_start_time", time.time()) + display_fetch_time(elapsed) + orig_handle(*args, **kwargs) + + view._handle_result = new_handle + + def new_trigger(*args, **kwargs): + if hasattr(view, "fetch_time_lbl") and view.fetch_time_lbl: + try: + view.fetch_time_lbl.destroy() + except Exception: + pass + view.fetch_time_lbl = None + + if hasattr(view, "sub_section_timer_labels"): + for lbl in view.sub_section_timer_labels.values(): + if lbl: + try: + lbl.destroy() + except Exception: + pass + view.sub_section_timer_labels.clear() + + if is_security_gov: + view.sub_section_start_times.clear() + + # Only set fetch_start_time here if there is no semaphore (otherwise WrappedSemaphore handles it) + if not getattr(view, "semaphore", None): + view.fetch_start_time = time.time() + + view.current_request_id += 1 + view.is_cancelled = False + + # Temporarily tag spawned threads with the current request ID and sub-section + orig_thread_init = threading.Thread.__init__ + req_id = view.current_request_id + + def new_thread_init(thread_self, *t_args, **t_kwargs): + orig_thread_init(thread_self, *t_args, **t_kwargs) + thread_self.request_id = req_id + + # Tag with sub-section name based on target method name + target = t_kwargs.get("target") or (t_args[0] if t_args else None) + if target: + target_name = getattr(target, "__name__", "") + if "labels" in target_name: + thread_self.sub_section = "labels" + elif "retention" in target_name: + thread_self.sub_section = "retention" + elif "dlp" in target_name: + thread_self.sub_section = "dlp" + elif "sit" in target_name: + thread_self.sub_section = "sit" + elif "auth" in target_name: + thread_self.sub_section = "auth" + + threading.Thread.__init__ = new_thread_init + try: + orig_trigger(*args, **kwargs) + finally: + threading.Thread.__init__ = orig_thread_init + + def new_reset(*args, **kwargs): + view.is_cancelled = True + if hasattr(view, "fetch_time_lbl") and view.fetch_time_lbl: + try: + view.fetch_time_lbl.destroy() + except Exception: + pass + view.fetch_time_lbl = None + + if hasattr(view, "sub_section_timer_labels"): + for lbl in view.sub_section_timer_labels.values(): + if lbl: + try: + lbl.destroy() + except Exception: + pass + view.sub_section_timer_labels.clear() + orig_reset(*args, **kwargs) + + def new_after(ms, callback, *args, **kwargs): + # Identify the calling thread + cur_thread = threading.current_thread() + thread_req_id = getattr(cur_thread, "request_id", None) + + # If the calling thread has a request_id and it doesn't match the current one, discard it + if thread_req_id is not None and thread_req_id != view.current_request_id: + async_logger.warning( + f"Discarded after() callback for {view.__class__.__name__} from stale thread " + f"(thread req_id: {thread_req_id}, current req_id: {view.current_request_id})" + ) + return None + return orig_after(ms, callback, *args, **kwargs) + + def cancel_method(): + view.is_cancelled = True + view.current_request_id += 1 + if view.status == "loading": + view.status = "cancelled" + if hasattr(view, "_update_ui_lists"): + data = getattr(view, "last_data", {}) or {} + view._update_ui_lists(data) + elif hasattr(view, "_set_state_error"): + view._set_state_error("⚠️ Telemetry fetch cancelled by user.") + + view.trigger_fetch = new_trigger + view.reset_view = new_reset + view.after = new_after + view.cancel = cancel_method + + def get_all_telemetry_data(self) -> dict: + """Retrieves cached telemetry data and charts from all sub-views.""" + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + tenant = self.lic_tenant_id.get().strip() + client_str = self.lic_client_ids.get().strip() + client_ids = [x.strip() for x in client_str.split(",") if x.strip()] + client_id = client_ids[0] if client_ids else "" + reports_dir = os.path.join(script_dir, "reports", f"{tenant}_{client_id}") + + def load_csv(filename): + path = os.path.join(reports_dir, filename) + if not os.path.exists(path): + return [] + try: + with open(path, 'r', encoding='utf-8') as f: + return list(csv.DictReader(f)) + except Exception: + return [] + + from collections import defaultdict + intune_data = getattr(self.intune_policies_view, "last_data", {}) + if not isinstance(intune_data, dict): + intune_data = {} + else: + # Create a copy so we don't accidentally mutate state properties in-place + intune_data = dict(intune_data) + + if not intune_data.get("mobile_apps"): + intune_data["mobile_apps"] = [r.get("displayName") for r in load_csv("intune_apps.csv") if r.get("displayName")] + if not intune_data.get("detected_apps"): + intune_data["detected_apps"] = load_csv("intune_detected_apps.csv") + if not intune_data.get("managed_devices"): + intune_data["managed_devices"] = load_csv("intune_managed_devices.csv") + if not intune_data.get("vc_devices"): + intune_data["vc_devices"] = load_csv("intune_vc_devices.csv") + if not intune_data.get("android_compliance"): + intune_data["android_compliance"] = load_csv("intune_android_compliance.csv") + if not intune_data.get("ios_compliance"): + intune_data["ios_compliance"] = load_csv("intune_ios_compliance.csv") + if not intune_data.get("mdm_policies"): + intune_data["mdm_policies"] = load_csv("intune_mdm_policies.csv") + if not intune_data.get("byod_configs"): + intune_data["byod_configs"] = load_csv("intune_byod_configs.csv") + if not intune_data.get("table_rows"): + configs = load_csv("intune_device_configs.csv") + policies = load_csv("intune_config_policies.csv") + counts = defaultdict(int) + for r in configs: + plat, p_type = r.get("platform"), r.get("policyType") + if plat and p_type: counts[(plat, p_type)] += 1 + for r in policies: + plat, p_type = r.get("platform"), r.get("policyType") + if plat and p_type: counts[(plat, p_type)] += 1 + rows = [] + for (platform, p_type), count in sorted(counts.items()): + rows.append((platform, p_type, str(count))) + intune_data["table_rows"] = rows + + return { + "tenant_id": tenant, + "skus": getattr(self.subscribed_skus_view, "last_licenses_items", []), + "directory": { + "organization": getattr(self.directory_view, "last_organization", []), + "domains": getattr(self.directory_view, "last_domains", []), + "user_creation_logs": getattr(self.directory_view, "last_user_creation_logs", []), + "provisioning_logs": getattr(self.directory_view, "last_provisioning_logs", []), + "group_counts": getattr(self.directory_view, "last_group_counts", {}), + "user_counts": getattr(self.directory_view, "last_user_counts", {}) + }, + "o365_usage": getattr(self.m365_apps_view.active_users_view, "last_data", []), + "o365_trend": getattr(self.m365_apps_view.active_users_trend_view, "trend_data", {}), + "m365_apps": getattr(self.m365_apps_view.m365_apps_view, "last_data", []), + "mailbox": getattr(self.exchange_online_view.mailbox_view, "last_data", {}), + "calendar": getattr(self.exchange_online_view.calendar_view, "last_data", {}), + "mail_security": getattr(self.exchange_online_view.mail_security_view, "last_data", {}), + "connectors": getattr(self.exchange_online_view.connectors_view, "last_data", []), + "email_clients": self._get_email_clients_pdf_mapped(), + "pst_files": getattr(self.exchange_online_view.email_clients_view, "last_pst_data", {}), + "exchange_connectors": getattr(self.exchange_online_view.connectors_view, "last_data", []), + "mail_security": getattr(self.exchange_online_view.mail_security_view, "last_data", {}), + "transport_rules": getattr(getattr(self.exchange_online_view, 'transport_rules_view', None), "last_data", []), + "sharepoint": getattr(self.files_view.sharepoint_view, "last_data", {}), + "onedrive": getattr(self.files_view.onedrive_view, "last_data", {}), + "devices_apps": getattr(self.devices_apps_view, "last_data", {}), + "intune": intune_data, + "network_security": { + "filtering_policies": load_csv("network_filtering_policies.csv"), + "conditional_access": load_csv("network_conditional_access.csv"), + "firewall_policies": load_csv("network_firewall_policies.csv") + }, + "security_labels": getattr(self.security_gov_view.sensitivity_frame, "last_data", []), + "retention_policies": getattr(self.security_gov_view.retention_frame, "last_data", []), + "dlp_policies": getattr(self.security_gov_view.dlp_frame, "last_data", []), + "sensitive_info_types": getattr(self.security_gov_view.sit_frame, "last_data", []), + "service_principals_sso": getattr(self.security_gov_view.sso_frame, "last_data", []), + "conditional_access": getattr(self.security_gov_view.auth_frame, "last_data", []), + "ediscovery_cases": load_csv("ediscovery_cases.csv"), + "power_automate": getattr(self.power_automate_view, "last_results", {}), + "msteams_activity": load_csv("msteams_activity.csv") + } + + def _get_email_clients_pdf_mapped(self) -> dict: + raw = getattr(self.exchange_online_view.email_clients_view, "last_data", {}) + if not raw: + return {} + if raw.get("client_error"): + return {"client_error": raw["client_error"]} + adop = raw.get("client_adoption", {}) + if not adop: + return {} + return { + "client_browser": adop.get("browser_users", 0), + "client_win_outlook": adop.get("desktop_win", 0), + "client_mac_outlook": adop.get("desktop_mac", 0), + "client_mac_mail": adop.get("desktop_mail_mac", 0), + "client_desktop_other": 0, + "client_mobile_outlook": adop.get("mobile_outlook", 0), + "client_mobile_other": adop.get("mobile_other", 0), + "client_imap": adop.get("protocol_imap4", 0), + "client_pop": adop.get("protocol_pop3", 0), + "client_smtp": adop.get("protocol_smtp", 0) + } + + def is_descendant(self, parent, widget) -> bool: + """Recursively checks if a widget (or its Tkinter path name) is a descendant of parent.""" + if not widget: + return False + if isinstance(widget, str): + try: + widget = self.nametowidget(widget) + except Exception: + return False + if widget == parent: + return True + if hasattr(widget, "master") and widget.master is not None: + return self.is_descendant(parent, widget.master) + return False + + def _handle_global_mousewheel(self, event): + """Redirects mousewheel scrolling to the tab's parent canvas if hovered.""" + try: + widget = self.winfo_containing(event.x_root, event.y_root) + except Exception: + return + + if self.is_descendant(self, widget): + if event.num == 4: # Linux scroll up + self._parent_canvas.yview("scroll", -1, "units") + elif event.num == 5: # Linux scroll down + self._parent_canvas.yview("scroll", 1, "units") + else: # Windows / macOS + if sys.platform == "darwin": + # macOS trackpad/mouse delta + self._parent_canvas.yview("scroll", -event.delta, "units") + else: + # Windows delta (usually multiple of 120) + self._parent_canvas.yview("scroll", -int(event.delta / 120), "units") + + def _monitor_memory_loop(self): + """Periodically measures current resident set size (RSS) RAM usage of the python process and logs it.""" + + try: + process = psutil.Process(os.getpid()) + except Exception as err: + async_logger.warning(f"Could not initialize psutil Process: {err}") + process = None + + while getattr(self, "mem_monitor_active", False): + mem_mb = 0.0 + if process: + try: + mem_bytes = process.memory_info().rss + mem_mb = float(mem_bytes) / (1024.0 * 1024.0) + except Exception as get_err: + async_logger.warning(f"Error reading memory info via psutil: {get_err}") + + async_logger.info(f"💾 Application Current Memory Usage (RSS): {mem_mb:.2f} MB") + time.sleep(30) diff --git a/telemetry/mail_security.py b/telemetry/mail_security.py new file mode 100644 index 00000000..a7691aa1 --- /dev/null +++ b/telemetry/mail_security.py @@ -0,0 +1,21 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Backward compatibility facade for Exchange Online Mail Security telemetry.""" + +# Re-export pipeline from core backend +from core.graph.exchange.mail_security import run_mail_security_pipeline + +# Re-export UI subframe from telemetry package +from telemetry.exchange.mail_security import MailSecurityFrame diff --git a/telemetry/mailbox_usage.py b/telemetry/mailbox_usage.py new file mode 100644 index 00000000..dcf92cb8 --- /dev/null +++ b/telemetry/mailbox_usage.py @@ -0,0 +1,21 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Backward compatibility facade for Exchange Online Mailbox usage telemetry.""" + +# Re-export pipeline and helper from core backend +from core.graph.exchange.mailbox import run_mailbox_usage_pipeline, format_bytes, parse_mailbox_usage_csv + +# Re-export UI subframe from telemetry package +from telemetry.exchange.mailbox import MailboxUsageFrame diff --git a/telemetry/network_security.py b/telemetry/network_security.py new file mode 100644 index 00000000..186fec20 --- /dev/null +++ b/telemetry/network_security.py @@ -0,0 +1,18 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Backward compatibility facade for Network Security telemetry.""" + +# Re-export UI container from telemetry package +from telemetry.network_security import NetworkSecurityFrame diff --git a/telemetry/network_security/__init__.py b/telemetry/network_security/__init__.py new file mode 100644 index 00000000..ab5cb2e1 --- /dev/null +++ b/telemetry/network_security/__init__.py @@ -0,0 +1,135 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Consolidated Network Security Policies & Access Controls Telemetry Orchestrator Container.""" + +import logging +import customtkinter as ctk + +from telemetry.styles import * +from telemetry.network_security.filtering import FilteringPoliciesSubFrame +from telemetry.network_security.conditional_access import ConditionalAccessSubFrame +from telemetry.network_security.firewall import FirewallSubFrame + +usage_logger = logging.getLogger("M365TelemetryAsyncLogger.NetworkSecurityUI") + +class NetworkSecurityFrame(ctk.CTkFrame): + """CustomTkinter component wrapping Network Security UI, with independent sub-sections.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, **kwargs): + self.semaphore = kwargs.pop("concurrency_semaphore", None) + super().__init__(master, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=12, **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.status = None + + self.build_ui() + + def build_ui(self): + self.pack(fill="x", expand=True, pady=10) + + self.inner_pad = ctk.CTkFrame(self, fg_color="transparent") + self.inner_pad.pack(fill="both", expand=True, padx=20, pady=20) + + # Global Section Title + self.header_frame = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.header_frame.pack(fill="x", pady=(0, 15)) + + self.title_lbl = ctk.CTkLabel( + self.header_frame, + text="Network Security Policies & Access Controls", + font=FONT_HEADER_SMALL, + text_color=COLOR_TEXT_MAIN + ) + self.title_lbl.pack(side="left", anchor="w") + + self.body_frame = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.body_frame.pack(fill="x", expand=True) + + # 1. Filtering Policies Frame + self.filtering_view = FilteringPoliciesSubFrame( + self.body_frame, + log_callback=self.log_msg, + credentials_callback=self.get_credentials, + status_change_callback=self._subframe_status_changed, + semaphore=self.semaphore + ) + self.filtering_view.pack(fill="x", pady=(10, 15)) + + # Divider 1 + self.divider1 = ctk.CTkFrame(self.body_frame, fg_color=COLOR_OUTLINE_LIGHT, height=1) + self.divider1.pack(fill="x", pady=15) + + # 2. CA Policies Frame + self.ca_view = ConditionalAccessSubFrame( + self.body_frame, + log_callback=self.log_msg, + credentials_callback=self.get_credentials, + status_change_callback=self._subframe_status_changed, + semaphore=self.semaphore + ) + self.ca_view.pack(fill="x", pady=(0, 15)) + + # Divider 2 + self.divider2 = ctk.CTkFrame(self.body_frame, fg_color=COLOR_OUTLINE_LIGHT, height=1) + self.divider2.pack(fill="x", pady=15) + + # 3. Firewall configurations Frame + self.fw_view = FirewallSubFrame( + self.body_frame, + log_callback=self.log_msg, + credentials_callback=self.get_credentials, + status_change_callback=self._subframe_status_changed, + semaphore=self.semaphore + ) + self.fw_view.pack(fill="x", pady=(0, 15)) + + self.reset_view() + + def _subframe_status_changed(self): + statuses = [ + self.filtering_view.status, + self.ca_view.status, + self.fw_view.status + ] + if "loading" in statuses: + self.status = "loading" + elif "error" in statuses: + self.status = "error" + elif "success" in statuses: + self.status = "success" + else: + self.status = None + self.on_status_change() + + def reset_view(self): + self.pack_forget() + self.status = None + self.filtering_view.reset_view() + self.ca_view.reset_view() + self.fw_view.reset_view() + + def trigger_fetch(self, tenant, client_id, client_secret): + usage_logger.info("Network Security trigger_fetch called.") + self.pack(fill="x", expand=True, pady=10) + self.filtering_view.trigger_fetch(tenant, client_id, client_secret) + self.ca_view.trigger_fetch(tenant, client_id, client_secret) + self.fw_view.trigger_fetch(tenant, client_id, client_secret) + + def cancel(self): + usage_logger.info("Network Security cancel called.") + self.filtering_view.cancel() + self.ca_view.cancel() + self.fw_view.cancel() diff --git a/telemetry/network_security/conditional_access.py b/telemetry/network_security/conditional_access.py new file mode 100644 index 00000000..352a72f8 --- /dev/null +++ b/telemetry/network_security/conditional_access.py @@ -0,0 +1,246 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""UI frame for Conditional Access policies under Network Security.""" + +import os +import time +import logging +import threading +import pandas as pd +import customtkinter as ctk +from tkinter import filedialog, messagebox + +from core.graph.network_security.conditional_access import run_conditional_access_pipeline +from telemetry.styles import * + +usage_logger = logging.getLogger("M365TelemetryAsyncLogger.ConditionalAccessUI") + +class ConditionalAccessSubFrame(ctk.CTkFrame): + """Sub-frame for Conditional Access network scope policies.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, semaphore=None, **kwargs): + super().__init__(master, fg_color="transparent", **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.semaphore = semaphore + self.status = None + self.page = 0 + self.ITEMS_PER_PAGE = 5 + self.csv_path = None + self.is_cancelled = False + + self.build_ui() + + def build_ui(self): + self.header_frame = ctk.CTkFrame(self, fg_color="transparent") + self.header_frame.pack(fill="x", pady=(0, 10)) + + ctk.CTkLabel(self.header_frame, text="Conditional Access Policies", font=FONT_HEADER_SMALL, text_color=COLOR_TEXT_MAIN).pack(side="left") + + self.btn_reload = ctk.CTkButton( + self.header_frame, text="↻ Reload", width=80, height=24, + font=ctk.CTkFont(family="Segoe UI", size=12), + fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color="#2563EB", hover_color="#DBEAFE", + command=self.trigger_fetch_individual + ) + self.btn_reload.pack(side="right", padx=(10, 0)) + + self.btn_export = ctk.CTkButton( + self.header_frame, text="Export Conditional Access", width=200, height=32, corner_radius=16, + font=FONT_BODY_BOLD, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, + command=self._export + ) + self.btn_export.pack(side="right") + + self.state_frame = ctk.CTkFrame(self, fg_color="transparent") + + self.grid_frame = ctk.CTkFrame(self, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + + headers = ["Policy Name", "State", "Target Users", "Target Apps", "Grant Controls"] + weights = [3, 1, 2, 2, 2] + for col_idx, weight in enumerate(weights): + self.grid_frame.grid_columnconfigure(col_idx, weight=weight) + + for col_idx, head_text in enumerate(headers): + cell = ctk.CTkFrame(self.grid_frame, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=1, pady=1) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + self.reset_view() + + def reset_view(self): + self.status = None + self.is_cancelled = False + self.csv_path = None + self.state_frame.pack_forget() + self.grid_frame.pack_forget() + for w in self.state_frame.winfo_children(): w.destroy() + for w in self.grid_frame.winfo_children(): + info = w.grid_info() + if "row" in info and int(info["row"]) > 0: w.destroy() + + def _set_loading_state(self, msg): + for w in self.state_frame.winfo_children(): w.destroy() + self.state_frame.pack(fill="x", expand=True) + ctk.CTkLabel(self.state_frame, text=f"⏳ {msg}", text_color=COLOR_TEXT_SUB, font=FONT_BODY_MEDIUM).pack(pady=(15, 5)) + pb = ctk.CTkProgressBar(self.state_frame, mode="indeterminate", width=200, fg_color=COLOR_SURFACE_VARIANT, progress_color=COLOR_PRIMARY) + pb.pack(pady=(0, 15)) + pb.start() + + def trigger_fetch_individual(self): + tenant, clients, secrets = self.get_credentials() + if tenant: + self.trigger_fetch(tenant, clients[0], secrets[0]) + + def trigger_fetch(self, tenant, client_id, client_secret): + self.status = "loading" + self.is_cancelled = False + self.page = 0 + self.btn_reload.configure(state="disabled") + self.btn_export.configure(state="disabled") + self.grid_frame.pack_forget() + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + if os.path.basename(script_dir) == "network_security": + script_dir = os.path.dirname(script_dir) + reports_dir = os.path.join(script_dir, "reports", f"{tenant}_{client_id}") + self.csv_path = os.path.join(reports_dir, "network_conditional_access.csv") + + self._set_loading_state("Scanning Conditional Access network scope...") + self.on_status_change() + + threading.Thread(target=self._execute_worker, args=(tenant, client_id, client_secret), daemon=True).start() + + def _execute_worker(self, tenant, client_id, client_secret): + if self.semaphore: self.semaphore.acquire() + try: + run_conditional_access_pipeline( + client_id=client_id, + client_secret=client_secret, + tenant_id=tenant, + csv_path=self.csv_path, + is_cancelled_callback=lambda: self.is_cancelled + ) + self.status = "success" + self.after(0, self._render_success) + except Exception as e: + usage_logger.error(f"CA fetch error: {e}", exc_info=True) + self.status = "error" + self.after(0, self._render_error, str(e)) + finally: + if self.semaphore: self.semaphore.release() + self.after(0, self.on_status_change) + + def _render_success(self): + self.btn_reload.configure(state="normal") + self.btn_export.configure(state="normal") + self.state_frame.pack_forget() + self.grid_frame.pack(fill="x") + + self._update_grid() + + def _render_error(self, err_msg): + self.btn_reload.configure(state="normal") + self.grid_frame.pack_forget() + for w in self.state_frame.winfo_children(): w.destroy() + self.state_frame.pack(fill="x", expand=True) + ctk.CTkLabel(self.state_frame, text=f"✖ {err_msg}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center", wraplength=700).pack(pady=(15, 5)) + ctk.CTkButton(self.state_frame, text="Try Again", command=self.trigger_fetch_individual, width=100, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 15)) + + def _load_page(self): + if not self.csv_path or not os.path.exists(self.csv_path): return [], 0 + try: + df = pd.read_csv(self.csv_path).fillna("N/A") + total = len(df) + start = self.page * self.ITEMS_PER_PAGE + end = start + self.ITEMS_PER_PAGE + return df.iloc[start:end].to_dict('records'), total + except Exception as e: + usage_logger.error(f"Error reading CSV {self.csv_path}: {e}") + return [], 0 + + def _update_grid(self): + for w in self.grid_frame.winfo_children(): + info = w.grid_info() + if "row" in info and int(info["row"]) > 0: w.destroy() + + page_data, total_count = self._load_page() + total_pages = (total_count - 1) // self.ITEMS_PER_PAGE + 1 if total_count > 0 else 1 + + for row_idx, item in enumerate(page_data, start=1): + bg_style = COLOR_SURFACE if row_idx % 2 == 0 else COLOR_SURFACE_VARIANT + col_keys = ["name", "state", "target_users", "target_apps", "controls"] + for col_idx, key in enumerate(col_keys): + cell = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + cell.grid(row=row_idx, column=col_idx, sticky="nsew", padx=1, pady=1) + lbl = ctk.CTkLabel(cell, text=str(item.get(key, "N/A")), font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN, wraplength=220, justify="left", anchor="w") + lbl.pack(padx=10, pady=6, fill="x") + + # Pagination controls row + control_frame = ctk.CTkFrame(self.grid_frame, fg_color="transparent") + control_frame.grid(row=self.ITEMS_PER_PAGE + 1, column=0, columnspan=5, pady=0, sticky="ew") + + center_container = ctk.CTkFrame(control_frame, fg_color="transparent") + center_container.pack(pady=(5, 10)) + + prev_state = "normal" if self.page > 0 else "disabled" + ctk.CTkButton( + center_container, text="◀ Prev", width=70, height=22, corner_radius=6, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, state=prev_state, + command=lambda: self._change_page(-1) + ).pack(side="left", padx=5) + + ctk.CTkLabel(center_container, text=f"Page {self.page + 1} of {total_pages}", font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_SUB).pack(side="left", padx=15) + + next_state = "normal" if self.page < total_pages - 1 else "disabled" + ctk.CTkButton( + center_container, text="Next ▶", width=70, height=22, corner_radius=6, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, state=next_state, + command=lambda: self._change_page(1) + ).pack(side="left", padx=5) + + def _change_page(self, delta): + self.page += delta + self._update_grid() + + def _export(self): + if not self.csv_path or not os.path.exists(self.csv_path): + messagebox.showerror("Export Error", "No data available to export yet.") + return + + dest_path = filedialog.asksaveasfilename( + defaultextension=".csv", + filetypes=[("CSV files", "*.csv"), ("All files", "*.*")], + title="Export Conditional Access policies CSV", + initialfile="M365_CA_Network_Access.csv", + parent=self + ) + if not dest_path: return + + try: + df = pd.read_csv(self.csv_path) + df.fillna("N/A").to_csv(dest_path, index=False, encoding="utf-8-sig") + messagebox.showinfo("Export Successful", f"Conditional Access Policies successfully exported to:\n{dest_path}") + except Exception as e: + messagebox.showerror("Export Failed", f"Failed to save CSV file: {e}") + + def cancel(self): + self.is_cancelled = True + self.status = None diff --git a/telemetry/network_security/filtering.py b/telemetry/network_security/filtering.py new file mode 100644 index 00000000..e4477aa9 --- /dev/null +++ b/telemetry/network_security/filtering.py @@ -0,0 +1,246 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""UI frame for Global Secure Access Filtering policies.""" + +import os +import time +import logging +import threading +import pandas as pd +import customtkinter as ctk +from tkinter import filedialog, messagebox + +from core.graph.network_security.filtering import run_filtering_pipeline +from telemetry.styles import * + +usage_logger = logging.getLogger("M365TelemetryAsyncLogger.FilteringPoliciesUI") + +class FilteringPoliciesSubFrame(ctk.CTkFrame): + """Sub-frame for Global Secure Access Filtering Policies.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, semaphore=None, **kwargs): + super().__init__(master, fg_color="transparent", **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.semaphore = semaphore + self.status = None + self.page = 0 + self.ITEMS_PER_PAGE = 5 + self.csv_path = None + self.is_cancelled = False + + self.build_ui() + + def build_ui(self): + self.header_frame = ctk.CTkFrame(self, fg_color="transparent") + self.header_frame.pack(fill="x", pady=(0, 10)) + + ctk.CTkLabel(self.header_frame, text="Filtering Policies", font=FONT_HEADER_SMALL, text_color=COLOR_TEXT_MAIN).pack(side="left") + + self.btn_reload = ctk.CTkButton( + self.header_frame, text="↻ Reload", width=80, height=24, + font=ctk.CTkFont(family="Segoe UI", size=12), + fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color="#2563EB", hover_color="#DBEAFE", + command=self.trigger_fetch_individual + ) + self.btn_reload.pack(side="right", padx=(10, 0)) + + self.btn_export = ctk.CTkButton( + self.header_frame, text="Export Filtering Policies", width=200, height=32, corner_radius=16, + font=FONT_BODY_BOLD, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, + command=self._export + ) + self.btn_export.pack(side="right") + + self.state_frame = ctk.CTkFrame(self, fg_color="transparent") + + self.grid_frame = ctk.CTkFrame(self, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + + headers = ["Policy Name", "Description", "Version", "Action", "Rules Count"] + weights = [2, 3, 1, 1, 1] + for col_idx, weight in enumerate(weights): + self.grid_frame.grid_columnconfigure(col_idx, weight=weight) + + for col_idx, head_text in enumerate(headers): + cell = ctk.CTkFrame(self.grid_frame, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=1, pady=1) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + self.reset_view() + + def reset_view(self): + self.status = None + self.is_cancelled = False + self.csv_path = None + self.state_frame.pack_forget() + self.grid_frame.pack_forget() + for w in self.state_frame.winfo_children(): w.destroy() + for w in self.grid_frame.winfo_children(): + info = w.grid_info() + if "row" in info and int(info["row"]) > 0: w.destroy() + + def _set_loading_state(self, msg): + for w in self.state_frame.winfo_children(): w.destroy() + self.state_frame.pack(fill="x", expand=True) + ctk.CTkLabel(self.state_frame, text=f"⏳ {msg}", text_color=COLOR_TEXT_SUB, font=FONT_BODY_MEDIUM).pack(pady=(15, 5)) + pb = ctk.CTkProgressBar(self.state_frame, mode="indeterminate", width=200, fg_color=COLOR_SURFACE_VARIANT, progress_color=COLOR_PRIMARY) + pb.pack(pady=(0, 15)) + pb.start() + + def trigger_fetch_individual(self): + tenant, clients, secrets = self.get_credentials() + if tenant: + self.trigger_fetch(tenant, clients[0], secrets[0]) + + def trigger_fetch(self, tenant, client_id, client_secret): + self.status = "loading" + self.is_cancelled = False + self.page = 0 + self.btn_reload.configure(state="disabled") + self.btn_export.configure(state="disabled") + self.grid_frame.pack_forget() + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + if os.path.basename(script_dir) == "network_security": + script_dir = os.path.dirname(script_dir) + reports_dir = os.path.join(script_dir, "reports", f"{tenant}_{client_id}") + self.csv_path = os.path.join(reports_dir, "network_filtering_policies.csv") + + self._set_loading_state("Scanning Entra filtering policies...") + self.on_status_change() + + threading.Thread(target=self._execute_worker, args=(tenant, client_id, client_secret), daemon=True).start() + + def _execute_worker(self, tenant, client_id, client_secret): + if self.semaphore: self.semaphore.acquire() + try: + run_filtering_pipeline( + client_id=client_id, + client_secret=client_secret, + tenant_id=tenant, + csv_path=self.csv_path, + is_cancelled_callback=lambda: self.is_cancelled + ) + self.status = "success" + self.after(0, self._render_success) + except Exception as e: + usage_logger.error(f"Filtering fetch error: {e}", exc_info=True) + self.status = "error" + self.after(0, self._render_error, str(e)) + finally: + if self.semaphore: self.semaphore.release() + self.after(0, self.on_status_change) + + def _render_success(self): + self.btn_reload.configure(state="normal") + self.btn_export.configure(state="normal") + self.state_frame.pack_forget() + self.grid_frame.pack(fill="x") + + self._update_grid() + + def _render_error(self, err_msg): + self.btn_reload.configure(state="normal") + self.grid_frame.pack_forget() + for w in self.state_frame.winfo_children(): w.destroy() + self.state_frame.pack(fill="x", expand=True) + ctk.CTkLabel(self.state_frame, text=f"✖ {err_msg}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center", wraplength=700).pack(pady=(15, 5)) + ctk.CTkButton(self.state_frame, text="Try Again", command=self.trigger_fetch_individual, width=100, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 15)) + + def _load_page(self): + if not self.csv_path or not os.path.exists(self.csv_path): return [], 0 + try: + df = pd.read_csv(self.csv_path).fillna("N/A") + total = len(df) + start = self.page * self.ITEMS_PER_PAGE + end = start + self.ITEMS_PER_PAGE + return df.iloc[start:end].to_dict('records'), total + except Exception as e: + usage_logger.error(f"Error reading CSV {self.csv_path}: {e}") + return [], 0 + + def _update_grid(self): + for w in self.grid_frame.winfo_children(): + info = w.grid_info() + if "row" in info and int(info["row"]) > 0: w.destroy() + + page_data, total_count = self._load_page() + total_pages = (total_count - 1) // self.ITEMS_PER_PAGE + 1 if total_count > 0 else 1 + + for row_idx, item in enumerate(page_data, start=1): + bg_style = COLOR_SURFACE if row_idx % 2 == 0 else COLOR_SURFACE_VARIANT + col_keys = ["name", "description", "version", "action", "rules_count"] + for col_idx, key in enumerate(col_keys): + cell = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + cell.grid(row=row_idx, column=col_idx, sticky="nsew", padx=1, pady=1) + lbl = ctk.CTkLabel(cell, text=str(item.get(key, "N/A")), font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN, wraplength=220, justify="left", anchor="w") + lbl.pack(padx=10, pady=6, fill="x") + + # Pagination controls row + control_frame = ctk.CTkFrame(self.grid_frame, fg_color="transparent") + control_frame.grid(row=self.ITEMS_PER_PAGE + 1, column=0, columnspan=5, pady=0, sticky="ew") + + center_container = ctk.CTkFrame(control_frame, fg_color="transparent") + center_container.pack(pady=(5, 10)) + + prev_state = "normal" if self.page > 0 else "disabled" + ctk.CTkButton( + center_container, text="◀ Prev", width=70, height=22, corner_radius=6, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, state=prev_state, + command=lambda: self._change_page(-1) + ).pack(side="left", padx=5) + + ctk.CTkLabel(center_container, text=f"Page {self.page + 1} of {total_pages}", font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_SUB).pack(side="left", padx=15) + + next_state = "normal" if self.page < total_pages - 1 else "disabled" + ctk.CTkButton( + center_container, text="Next ▶", width=70, height=22, corner_radius=6, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, state=next_state, + command=lambda: self._change_page(1) + ).pack(side="left", padx=5) + + def _change_page(self, delta): + self.page += delta + self._update_grid() + + def _export(self): + if not self.csv_path or not os.path.exists(self.csv_path): + messagebox.showerror("Export Error", "No data available to export yet.") + return + + dest_path = filedialog.asksaveasfilename( + defaultextension=".csv", + filetypes=[("CSV files", "*.csv"), ("All files", "*.*")], + title="Export Filtering Policies CSV", + initialfile="M365_Secure_Access_Filtering.csv", + parent=self + ) + if not dest_path: return + + try: + df = pd.read_csv(self.csv_path) + df.fillna("N/A").to_csv(dest_path, index=False, encoding="utf-8-sig") + messagebox.showinfo("Export Successful", f"Filtering Policies successfully exported to:\n{dest_path}") + except Exception as e: + messagebox.showerror("Export Failed", f"Failed to save CSV file: {e}") + + def cancel(self): + self.is_cancelled = True + self.status = None diff --git a/telemetry/network_security/firewall.py b/telemetry/network_security/firewall.py new file mode 100644 index 00000000..6a49ddd8 --- /dev/null +++ b/telemetry/network_security/firewall.py @@ -0,0 +1,246 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""UI frame for Intune Firewall and Proxy configurations.""" + +import os +import time +import logging +import threading +import pandas as pd +import customtkinter as ctk +from tkinter import filedialog, messagebox + +from core.graph.network_security.firewall import run_firewall_pipeline +from telemetry.styles import * + +usage_logger = logging.getLogger("M365TelemetryAsyncLogger.FirewallUI") + +class FirewallSubFrame(ctk.CTkFrame): + """Sub-frame for Intune Firewall & Proxy configuration settings.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, semaphore=None, **kwargs): + super().__init__(master, fg_color="transparent", **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.semaphore = semaphore + self.status = None + self.page = 0 + self.ITEMS_PER_PAGE = 5 + self.csv_path = None + self.is_cancelled = False + + self.build_ui() + + def build_ui(self): + self.header_frame = ctk.CTkFrame(self, fg_color="transparent") + self.header_frame.pack(fill="x", pady=(0, 10)) + + ctk.CTkLabel(self.header_frame, text="Firewall and Proxy Configurations", font=FONT_HEADER_SMALL, text_color=COLOR_TEXT_MAIN).pack(side="left") + + self.btn_reload = ctk.CTkButton( + self.header_frame, text="↻ Reload", width=80, height=24, + font=ctk.CTkFont(family="Segoe UI", size=12), + fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color="#2563EB", hover_color="#DBEAFE", + command=self.trigger_fetch_individual + ) + self.btn_reload.pack(side="right", padx=(10, 0)) + + self.btn_export = ctk.CTkButton( + self.header_frame, text="Export Firewall & Proxy Configs", width=200, height=32, corner_radius=16, + font=FONT_BODY_BOLD, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, + command=self._export + ) + self.btn_export.pack(side="right") + + self.state_frame = ctk.CTkFrame(self, fg_color="transparent") + + self.grid_frame = ctk.CTkFrame(self, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + + headers = ["Configuration Name", "Policy Type", "Firewall Status", "Proxy Status"] + weights = [3, 2, 1, 1] + for col_idx, weight in enumerate(weights): + self.grid_frame.grid_columnconfigure(col_idx, weight=weight) + + for col_idx, head_text in enumerate(headers): + cell = ctk.CTkFrame(self.grid_frame, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=1, pady=1) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + self.reset_view() + + def reset_view(self): + self.status = None + self.is_cancelled = False + self.csv_path = None + self.state_frame.pack_forget() + self.grid_frame.pack_forget() + for w in self.state_frame.winfo_children(): w.destroy() + for w in self.grid_frame.winfo_children(): + info = w.grid_info() + if "row" in info and int(info["row"]) > 0: w.destroy() + + def _set_loading_state(self, msg): + for w in self.state_frame.winfo_children(): w.destroy() + self.state_frame.pack(fill="x", expand=True) + ctk.CTkLabel(self.state_frame, text=f"⏳ {msg}", text_color=COLOR_TEXT_SUB, font=FONT_BODY_MEDIUM).pack(pady=(15, 5)) + pb = ctk.CTkProgressBar(self.state_frame, mode="indeterminate", width=200, fg_color=COLOR_SURFACE_VARIANT, progress_color=COLOR_PRIMARY) + pb.pack(pady=(0, 15)) + pb.start() + + def trigger_fetch_individual(self): + tenant, clients, secrets = self.get_credentials() + if tenant: + self.trigger_fetch(tenant, clients[0], secrets[0]) + + def trigger_fetch(self, tenant, client_id, client_secret): + self.status = "loading" + self.is_cancelled = False + self.page = 0 + self.btn_reload.configure(state="disabled") + self.btn_export.configure(state="disabled") + self.grid_frame.pack_forget() + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + if os.path.basename(script_dir) == "network_security": + script_dir = os.path.dirname(script_dir) + reports_dir = os.path.join(script_dir, "reports", f"{tenant}_{client_id}") + self.csv_path = os.path.join(reports_dir, "network_firewall_policies.csv") + + self._set_loading_state("Scanning Intune Firewall and Proxy configurations...") + self.on_status_change() + + threading.Thread(target=self._execute_worker, args=(tenant, client_id, client_secret), daemon=True).start() + + def _execute_worker(self, tenant, client_id, client_secret): + if self.semaphore: self.semaphore.acquire() + try: + run_firewall_pipeline( + client_id=client_id, + client_secret=client_secret, + tenant_id=tenant, + csv_path=self.csv_path, + is_cancelled_callback=lambda: self.is_cancelled + ) + self.status = "success" + self.after(0, self._render_success) + except Exception as e: + usage_logger.error(f"Firewall fetch error: {e}", exc_info=True) + self.status = "error" + self.after(0, self._render_error, str(e)) + finally: + if self.semaphore: self.semaphore.release() + self.after(0, self.on_status_change) + + def _render_success(self): + self.btn_reload.configure(state="normal") + self.btn_export.configure(state="normal") + self.state_frame.pack_forget() + self.grid_frame.pack(fill="x") + + self._update_grid() + + def _render_error(self, err_msg): + self.btn_reload.configure(state="normal") + self.grid_frame.pack_forget() + for w in self.state_frame.winfo_children(): w.destroy() + self.state_frame.pack(fill="x", expand=True) + ctk.CTkLabel(self.state_frame, text=f"✖ {err_msg}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center", wraplength=700).pack(pady=(15, 5)) + ctk.CTkButton(self.state_frame, text="Try Again", command=self.trigger_fetch_individual, width=100, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 15)) + + def _load_page(self): + if not self.csv_path or not os.path.exists(self.csv_path): return [], 0 + try: + df = pd.read_csv(self.csv_path).fillna("N/A") + total = len(df) + start = self.page * self.ITEMS_PER_PAGE + end = start + self.ITEMS_PER_PAGE + return df.iloc[start:end].to_dict('records'), total + except Exception as e: + usage_logger.error(f"Error reading CSV {self.csv_path}: {e}") + return [], 0 + + def _update_grid(self): + for w in self.grid_frame.winfo_children(): + info = w.grid_info() + if "row" in info and int(info["row"]) > 0: w.destroy() + + page_data, total_count = self._load_page() + total_pages = (total_count - 1) // self.ITEMS_PER_PAGE + 1 if total_count > 0 else 1 + + for row_idx, item in enumerate(page_data, start=1): + bg_style = COLOR_SURFACE if row_idx % 2 == 0 else COLOR_SURFACE_VARIANT + col_keys = ["name", "policy_type", "firewall_status", "proxy_status"] + for col_idx, key in enumerate(col_keys): + cell = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + cell.grid(row=row_idx, column=col_idx, sticky="nsew", padx=1, pady=1) + lbl = ctk.CTkLabel(cell, text=str(item.get(key, "N/A")), font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN, wraplength=220, justify="left", anchor="w") + lbl.pack(padx=10, pady=6, fill="x") + + # Pagination controls row + control_frame = ctk.CTkFrame(self.grid_frame, fg_color="transparent") + control_frame.grid(row=self.ITEMS_PER_PAGE + 1, column=0, columnspan=4, pady=0, sticky="ew") + + center_container = ctk.CTkFrame(control_frame, fg_color="transparent") + center_container.pack(pady=(5, 10)) + + prev_state = "normal" if self.page > 0 else "disabled" + ctk.CTkButton( + center_container, text="◀ Prev", width=70, height=22, corner_radius=6, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, state=prev_state, + command=lambda: self._change_page(-1) + ).pack(side="left", padx=5) + + ctk.CTkLabel(center_container, text=f"Page {self.page + 1} of {total_pages}", font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_SUB).pack(side="left", padx=15) + + next_state = "normal" if self.page < total_pages - 1 else "disabled" + ctk.CTkButton( + center_container, text="Next ▶", width=70, height=22, corner_radius=6, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, state=next_state, + command=lambda: self._change_page(1) + ).pack(side="left", padx=5) + + def _change_page(self, delta): + self.page += delta + self._update_grid() + + def _export(self): + if not self.csv_path or not os.path.exists(self.csv_path): + messagebox.showerror("Export Error", "No data available to export yet.") + return + + dest_path = filedialog.asksaveasfilename( + defaultextension=".csv", + filetypes=[("CSV files", "*.csv"), ("All files", "*.*")], + title="Export Firewall & Proxy Configs CSV", + initialfile="M365_Firewall_Proxy_Configs.csv", + parent=self + ) + if not dest_path: return + + try: + df = pd.read_csv(self.csv_path) + df.fillna("N/A").to_csv(dest_path, index=False, encoding="utf-8-sig") + messagebox.showinfo("Export Successful", f"Firewall & Proxy Configurations successfully exported to:\n{dest_path}") + except Exception as e: + messagebox.showerror("Export Failed", f"Failed to save CSV file: {e}") + + def cancel(self): + self.is_cancelled = True + self.status = None diff --git a/telemetry/pdf_report.py b/telemetry/pdf_report.py new file mode 100644 index 00000000..d3350665 --- /dev/null +++ b/telemetry/pdf_report.py @@ -0,0 +1,2234 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PDF Report Compilation module for Microsoft 365 Tenant Telemetry data.""" + +import io +import html +from datetime import datetime +from collections import Counter +from matplotlib.figure import Figure +from matplotlib.backends.backend_agg import FigureCanvasAgg +from reportlab.lib.pagesizes import letter +from reportlab.platypus import SimpleDocTemplate, Paragraph, Spacer, Table, TableStyle, Image, PageBreak, KeepTogether +from reportlab.lib.styles import getSampleStyleSheet, ParagraphStyle +from reportlab.lib import colors +from reportlab.pdfgen import canvas + + +class NumberedCanvas(canvas.Canvas): + """Custom canvas to compute total page count and draw running headers, footers and page numbers.""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._saved_page_states = [] + + def showPage(self): + self._saved_page_states.append(dict(self.__dict__)) + self._startPage() + + def save(self): + num_pages = len(self._saved_page_states) + for state in self._saved_page_states: + self.__dict__.update(state) + self.draw_page_decorations(num_pages) + super().showPage() + super().save() + + def draw_page_decorations(self, page_count): + if self._pageNumber == 1: + # Skip page number and decorations on the cover page + return + + self.saveState() + self.setFont("Helvetica-Bold", 8) + self.setFillColor(colors.HexColor("#1E3A8A")) + + # Header + self.drawString(54, 750, "DEAL ASSISTANT") + self.setFont("Helvetica", 8) + self.setFillColor(colors.HexColor("#64748B")) + self.drawRightString(558, 750, "M365 Tenant Telemetry & Audit Report") + + # Header Line + self.setStrokeColor(colors.HexColor("#E2E8F0")) + self.setLineWidth(0.5) + self.line(54, 742, 558, 742) + + # Footer Line + self.line(54, 52, 558, 52) + + # Footer + self.drawString(54, 40, "Confidential - Tenant Audit Assessment") + page_text = f"Page {self._pageNumber} of {page_count}" + self.drawRightString(558, 40, page_text) + + self.restoreState() + + +def format_prepaid_units(item: dict) -> str: + prepaid = item.get("prepaidUnits", {}) + p_str = f"Enabled: {prepaid.get('enabled', 0):,}" + if prepaid.get('warning', 0) > 0: + p_str += f"\nWarn: {prepaid.get('warning'):,}" + if prepaid.get('suspended', 0) > 0: + p_str += f"\nSusp: {prepaid.get('suspended'):,}" + return p_str + + +def generate_trend_chart_bytes(trend_data: dict) -> io.BytesIO: + """Generates the O365 Active User Trend Chart on-the-fly to minimize persistent memory footprint.""" + + dates = trend_data.get("dates", []) + if not dates: + return None + + fig = Figure(figsize=(6.5, 3.2), dpi=150) + ax = fig.add_subplot(111) + fig.patch.set_facecolor("#FFFFFF") + ax.set_facecolor("#FFFFFF") + + # Palette tailored to Match Deal Assistant theme + ax.plot(dates, trend_data.get("office365", []), marker='o', markersize=3, linewidth=1.5, label='Office 365', color="#1E3A8A") + ax.plot(dates, trend_data.get("exchange", []), marker='o', markersize=3, linewidth=1.5, label='Exchange', color="#C2410C") + ax.plot(dates, trend_data.get("onedrive", []), marker='o', markersize=3, linewidth=1.5, label='OneDrive', color="#3B82F6") + ax.plot(dates, trend_data.get("sharepoint", []), marker='o', markersize=3, linewidth=1.5, label='SharePoint', color="#15803D") + ax.plot(dates, trend_data.get("teams", []), marker='o', markersize=3, linewidth=1.5, label='Teams', color="#9333EA") + + ax.set_xlabel("Date", fontsize=8, color="#475569") + ax.set_ylabel("Active Users", fontsize=8, color="#475569") + ax.tick_params(axis='x', colors="#475569", rotation=45, labelsize=7) + ax.tick_params(axis='y', colors="#475569", labelsize=7) + + if len(dates) > 10: + ax.set_xticks(dates[::max(1, len(dates)//10)]) + + for spine in ax.spines.values(): + spine.set_color("#CBD5E1") + + ax.legend(facecolor="#FFFFFF", edgecolor="#CBD5E1", labelcolor="#1E293B", fontsize=8) + fig.tight_layout() + + buf = io.BytesIO() + canvas = FigureCanvasAgg(fig) + canvas.print_png(buf) + buf.seek(0) + return buf + + +def generate_pa_chart_bytes(pa: dict) -> io.BytesIO: + """Generates the Power Automate Flows breakdown bar chart on-the-fly.""" + + counts = pa.get("counts", {}) + if not counts: + return None + + active_counts = pa.get("active_counts", {}) + tier_counts = pa.get("tier_counts", {}) + active_tier_counts = pa.get("active_tier_counts", {}) + complex_flows = pa.get("complex_logic_flows", []) + + fig = Figure(figsize=(6.5, 3.2), dpi=150) + ax = fig.add_subplot(111) + fig.patch.set_facecolor("#FFFFFF") + ax.set_facecolor("#FFFFFF") + + categories = ['Cloud Flows', 'Desktop Flows', 'Personal', 'Enterprise', 'Complex'] + + c_total = counts.get("Cloud Flows", 0) + c_active = active_counts.get("Cloud Flows", 0) + c_inactive = c_total - c_active + + d_total = counts.get("Desktop Flows", 0) + d_active = active_counts.get("Desktop Flows", 0) + d_inactive = d_total - d_active + + p_total = tier_counts.get("Personal Productivity", 0) + p_active = active_tier_counts.get("Personal Productivity", 0) + p_inactive = p_total - p_active + + e_total = tier_counts.get("Enterprise/Departmental", 0) + e_active = active_tier_counts.get("Enterprise/Departmental", 0) + e_inactive = e_total - e_active + + complex_active = sum(1 for f in complex_flows if f.get("Active") == "Yes") + complex_inactive = len(complex_flows) - complex_active + + actives = [c_active, d_active, p_active, e_active, complex_active] + inactives = [c_inactive, d_inactive, p_inactive, e_inactive, complex_inactive] + + x = range(len(categories)) + width = 0.25 + + rects1 = ax.bar(x, actives, width, label='Active', color="#1E3A8A") + rects2 = ax.bar([i + width for i in x], inactives, width, label='Inactive', color="#CBD5E1") + + ax.set_ylabel('Count', color="#1E293B", fontsize=8, fontweight='bold') + ax.set_title('Power Automate Flows Breakdown', color="#1E293B", fontsize=9, fontweight='bold') + ax.set_xticks([i + width/2 for i in x]) + ax.set_xticklabels(categories, color="#1E293B", fontsize=8, fontweight='bold') + ax.legend(facecolor="#FFFFFF", edgecolor="#CBD5E1", labelcolor="#1E293B", prop={'size':8}) + + ax.bar_label(rects1, padding=2, color="#1E293B", fontsize=7) + ax.bar_label(rects2, padding=2, color="#1E293B", fontsize=7) + + for spine in ax.spines.values(): + spine.set_color("#CBD5E1") + + ax.tick_params(axis='y', colors="#1E293B", labelsize=8) + + max_val = max(max(actives), max(inactives)) + ax.set_ylim(0, max(max_val + 3, int(max_val * 1.3))) + + fig.tight_layout() + buf = io.BytesIO() + canvas = FigureCanvasAgg(fig) + canvas.print_png(buf) + buf.seek(0) + return buf + + +def generate_pdf_report(data: dict, filepath: str): + """Generates a beautifully structured PDF document summarizing all tenant telemetry statistics.""" + + # 1. Document Setup + # 54pt margins correspond to 0.75 inches + doc = SimpleDocTemplate( + filepath, + pagesize=letter, + leftMargin=54, + rightMargin=54, + topMargin=64, + bottomMargin=64 + ) + + styles = getSampleStyleSheet() + + # Custom color palette + primary_color = colors.HexColor("#1E3A8A") # Navy Accent + secondary_color = colors.HexColor("#475569") # Slate Secondary + text_color = colors.HexColor("#1E293B") # Charcoal Body Text + outline_color = colors.HexColor("#CBD5E1") # Border light grey + + # Modify default styles in-place + styles['Normal'].textColor = text_color + styles['Normal'].fontSize = 9 + styles['Normal'].leading = 13 + + # Custom styles + title_style = ParagraphStyle( + 'CoverTitle', + parent=styles['Normal'], + fontName='Helvetica-Bold', + fontSize=26, + leading=32, + textColor=primary_color, + spaceAfter=10 + ) + + subtitle_style = ParagraphStyle( + 'CoverSubtitle', + parent=styles['Normal'], + fontName='Helvetica', + fontSize=13, + leading=18, + textColor=secondary_color, + spaceAfter=30 + ) + + h1_style = ParagraphStyle( + 'SectionH1', + parent=styles['Normal'], + fontName='Helvetica-Bold', + fontSize=15, + leading=18, + textColor=primary_color, + spaceBefore=22, + spaceAfter=10, + keepWithNext=True + ) + + h2_style = ParagraphStyle( + 'SectionH2', + parent=styles['Normal'], + fontName='Helvetica-Bold', + fontSize=11, + leading=14, + textColor=secondary_color, + spaceBefore=14, + spaceAfter=6, + keepWithNext=True + ) + + body_style = ParagraphStyle( + 'ReportBody', + parent=styles['Normal'], + fontSize=9, + leading=13, + spaceAfter=6 + ) + + bold_body_style = ParagraphStyle( + 'ReportBodyBold', + parent=body_style, + fontName='Helvetica-Bold' + ) + + table_cell_style = ParagraphStyle( + 'TableCell', + parent=styles['Normal'], + fontSize=8.5, + leading=11 + ) + + table_cell_bold = ParagraphStyle( + 'TableCellBold', + parent=table_cell_style, + fontName='Helvetica-Bold', + textColor=primary_color + ) + + table_cell_header = ParagraphStyle( + 'TableCellHeader', + parent=table_cell_style, + fontName='Helvetica-Bold', + textColor=colors.white + ) + + small_table_cell_style = ParagraphStyle( + 'SmallTableCell', + parent=styles['Normal'], + fontSize=6.0, + leading=7.5 + ) + + small_table_cell_bold = ParagraphStyle( + 'SmallTableCellBold', + parent=small_table_cell_style, + fontName='Helvetica-Bold', + textColor=primary_color + ) + + small_table_cell_header = ParagraphStyle( + 'SmallTableCellHeader', + parent=small_table_cell_style, + fontName='Helvetica-Bold', + textColor=colors.white + ) + + meta_label_style = ParagraphStyle( + 'MetaLabel', + parent=styles['Normal'], + fontName='Helvetica-Bold', + fontSize=10, + textColor=secondary_color + ) + + meta_val_style = ParagraphStyle( + 'MetaValue', + parent=styles['Normal'], + fontSize=10, + textColor=text_color + ) + + story = [] + + # ========================================================================= + # COVER PAGE + # ========================================================================= + story.append(Spacer(1, 120)) + story.append(Paragraph("🤝 Deal Assistant", ParagraphStyle('Branding', parent=styles['Normal'], fontName='Helvetica-Bold', fontSize=18, textColor=primary_color, spaceAfter=20))) + story.append(Paragraph("Microsoft 365 Tenant
Audit & Telemetry Report", title_style)) + story.append(Paragraph("A comprehensive assessment of license allocations, workload adoption patterns, security configurations, and workflow automation.", subtitle_style)) + story.append(Spacer(1, 100)) + + # Metadata Table + meta_data = [ + [Paragraph("Tenant Name / ID:", meta_label_style), Paragraph(data.get("tenant_id", "N/A"), meta_val_style)], + [Paragraph("Report Generated:", meta_label_style), Paragraph(datetime.now().strftime("%B %d, %Y at %I:%M %p"), meta_val_style)], + [Paragraph("Assessment Status:", meta_label_style), Paragraph("🟢 Audit Completed Successfully", ParagraphStyle('StatusStyle', parent=meta_val_style, fontName='Helvetica-Bold', textColor=colors.HexColor("#15803D")))], + [Paragraph("Report Context:", meta_label_style), Paragraph("Usage & Adoption Inventory", meta_val_style)] + ] + + meta_table = Table(meta_data, colWidths=[130, 370]) + meta_table.setStyle(TableStyle([ + ('VALIGN', (0, 0), (-1, -1), 'TOP'), + ('BOTTOMPADDING', (0, 0), (-1, -1), 8), + ('LINEBELOW', (0, 0), (-1, -2), 0.5, colors.HexColor("#F1F5F9")), + ])) + story.append(meta_table) + story.append(PageBreak()) + + # ========================================================================= + # SECTION 1: SUBSCRIBED SKUS INVENTORY + # ========================================================================= + story.append(Paragraph("1. Subscribed SKUs", h1_style)) + story.append(Paragraph("This section outlines the licensing packages (SKUs) currently configured and active in your Microsoft Entra ID tenant scope, displaying total enabled vs. consumed license counts.", body_style)) + story.append(Spacer(1, 8)) + + sku_list = data.get("skus", []) + if not sku_list: + story.append(Paragraph("No subscribed licensing data was discovered or available for this report.", ParagraphStyle('ErrTxt', parent=body_style, textColor=colors.HexColor("#DC2626")))) + else: + # Table columns: SKU, Units, Consumed + sku_table_data = [[ + Paragraph("SKU Part Number", table_cell_header), + Paragraph("Allocated Units Status", table_cell_header), + Paragraph("Consumed Units", table_cell_header) + ]] + + for item in sku_list: + sku_table_data.append([ + Paragraph(item.get("skuPartNumber", "UNKNOWN_SKU"), table_cell_bold), + Paragraph(format_prepaid_units(item).replace("\n", "
"), table_cell_style), + Paragraph(f"{item.get('consumedUnits', 0):,}", table_cell_style) + ]) + + sku_table = Table(sku_table_data, colWidths=[220, 160, 120]) + sku_table.setStyle(TableStyle([ + ('BACKGROUND', (0, 0), (-1, 0), primary_color), + ('ALIGN', (0, 0), (-1, -1), 'LEFT'), + ('VALIGN', (0, 0), (-1, -1), 'MIDDLE'), + ('TOPPADDING', (0, 0), (-1, -1), 6), + ('BOTTOMPADDING', (0, 0), (-1, -1), 6), + ('ROWBACKGROUNDS', (0, 1), (-1, -1), [colors.white, colors.HexColor("#F8FAFC")]), + ('GRID', (0, 0), (-1, -1), 0.5, outline_color), + ])) + story.append(sku_table) + story.append(Spacer(1, 15)) + + # ========================================================================= + # SECTION 1b: DIRECTORY SUMMARY + # ========================================================================= + story.append(Paragraph("1b. Directory Summary", h1_style)) + + dir_data = data.get("directory", {}) + if not dir_data: + story.append(Paragraph("No directory telemetry data was available.", ParagraphStyle('ErrTxt', parent=body_style, textColor=colors.HexColor("#DC2626")))) + else: + # 1. Organization Details Table + story.append(Paragraph("Organization Details", h2_style)) + story.append(Paragraph("This section outlines general configuration parameters, tenant types, sync properties, and active services/plans configured for the tenant organization.", body_style)) + story.append(Spacer(1, 8)) + + org_list = dir_data.get("organization", []) + if not org_list: + story.append(Paragraph("No organization configuration details were available.", ParagraphStyle('ErrTxt', parent=body_style, textColor=colors.HexColor("#DC2626")))) + else: + org = org_list[0] if org_list else {} + plans = org.get("provisionedPlans", []) + plan_services = sorted(list(set(plan.get("service") for plan in plans if plan.get("service")))) + plan_services_str = ", ".join(plan_services) if plan_services else "null" + + def format_pdf_val(v): + return "null" if v is None else str(v) + + org_table_data = [ + [Paragraph("Property", table_cell_header), Paragraph("Value", table_cell_header)], + [Paragraph("displayName", table_cell_bold), Paragraph(format_pdf_val(org.get("displayName")), table_cell_style)], + [Paragraph("isMultipleDataLocationsForServicesEnabled", table_cell_bold), Paragraph(format_pdf_val(org.get("isMultipleDataLocationsForServicesEnabled")), table_cell_style)], + [Paragraph("onPremisesSyncEnabled", table_cell_bold), Paragraph(format_pdf_val(org.get("onPremisesSyncEnabled")), table_cell_style)], + [Paragraph("onPremisesLastSyncDateTime", table_cell_bold), Paragraph(format_pdf_val(org.get("onPremisesLastSyncDateTime")), table_cell_style)], + [Paragraph("partnerTenantType", table_cell_bold), Paragraph(format_pdf_val(org.get("partnerTenantType")), table_cell_style)], + [Paragraph("tenantType", table_cell_bold), Paragraph(format_pdf_val(org.get("tenantType")), table_cell_style)], + [Paragraph("provisionedPlans", table_cell_bold), Paragraph(plan_services_str, table_cell_style)] + ] + + org_table = Table(org_table_data, colWidths=[200, 300]) + org_table.setStyle(TableStyle([ + ('BACKGROUND', (0, 0), (-1, 0), primary_color), + ('ALIGN', (0, 0), (-1, -1), 'LEFT'), + ('VALIGN', (0, 0), (-1, -1), 'MIDDLE'), + ('TOPPADDING', (0, 0), (-1, -1), 5), + ('BOTTOMPADDING', (0, 0), (-1, -1), 5), + ('ROWBACKGROUNDS', (0, 1), (-1, -1), [colors.white, colors.HexColor("#F8FAFC")]), + ('GRID', (0, 0), (-1, -1), 0.5, outline_color), + ])) + story.append(org_table) + story.append(Spacer(1, 4)) + story.append(Paragraph("* If OnPremisesSyncEnabled returns True, on-premises Active Directory is a primary source of truth. If it returns Null or False, the directory is cloud-managed or driven by a 3rd-party application.", body_style)) + + story.append(Spacer(1, 15)) + + # 2. Domains Table + story.append(Paragraph("Domains", h2_style)) + story.append(Paragraph("This section displays the configured internet domains associated with the tenant and their verified statuses.", body_style)) + story.append(Spacer(1, 8)) + + domains = dir_data.get("domains", []) + if not domains: + story.append(Paragraph("No domains discovered in directory scope.", ParagraphStyle('ErrTxt', parent=body_style, textColor=colors.HexColor("#DC2626")))) + else: + domains_table_data = [[ + Paragraph("Domain ID", table_cell_header), + Paragraph("Auth Type", table_cell_header), + Paragraph("Admin Managed", table_cell_header), + Paragraph("Default", table_cell_header), + Paragraph("Verified", table_cell_header), + Paragraph("Supported Services", table_cell_header), + Paragraph("Federation Display Name", table_cell_header), + Paragraph("Federation Issuer URI", table_cell_header) + ]] + for item in domains: + auth_type = item.get("authenticationType", "N/A") or "N/A" + admin_managed = "Yes" if item.get("isAdminManaged") else "No" + is_default = "Yes" if item.get("isDefault") else "No" + is_verified = "Yes" if item.get("isVerified") else "No" + services = item.get("supportedServices", []) + services_str = ", ".join(services) if services else "-" + fed_idp = item.get("federationDisplayName") or "-" + fed_issuer = item.get("federationIssuerUri") or "-" + + domains_table_data.append([ + Paragraph(item.get("id", "-"), table_cell_bold), + Paragraph(auth_type, table_cell_style), + Paragraph(admin_managed, table_cell_style), + Paragraph(is_default, table_cell_style), + Paragraph(is_verified, table_cell_style), + Paragraph(services_str, table_cell_style), + Paragraph(fed_idp, table_cell_style), + Paragraph(fed_issuer, table_cell_style) + ]) + domains_table = Table(domains_table_data, colWidths=[80, 50, 45, 35, 35, 95, 80, 80]) + domains_table.setStyle(TableStyle([ + ('BACKGROUND', (0, 0), (-1, 0), primary_color), + ('ALIGN', (0, 0), (-1, -1), 'LEFT'), + ('VALIGN', (0, 0), (-1, -1), 'MIDDLE'), + ('TOPPADDING', (0, 0), (-1, -1), 5), + ('BOTTOMPADDING', (0, 0), (-1, -1), 5), + ('ROWBACKGROUNDS', (0, 1), (-1, -1), [colors.white, colors.HexColor("#F8FAFC")]), + ('GRID', (0, 0), (-1, -1), 0.5, outline_color), + ])) + story.append(domains_table) + story.append(Spacer(1, 4)) + story.append(Paragraph("* AuthenticationType=Managed indicates a cloud managed domain where Microsoft Entra ID performs user authentication. Federated indicates authentication is federated with an identity provider (eg. AD FS, Okta etc.)", body_style)) + + story.append(Spacer(1, 15)) + + # 2b. User Creation/Deletion Logs Table + story.append(Paragraph("User Creation/Deletion Logs", h2_style)) + story.append(Paragraph("This section displays directory audit logs for user creation and deletion events, indicating who initiated the action and the associated details.", body_style)) + story.append(Spacer(1, 8)) + + user_creation_logs = dir_data.get("user_creation_logs", []) + if not user_creation_logs: + story.append(Paragraph("No user creation or deletion audit logs discovered.", body_style)) + elif user_creation_logs[0].get("activity") == "ERROR": + err_msg = user_creation_logs[0].get("initiatedBy") + story.append(Paragraph(f"Error: {err_msg}", ParagraphStyle('ErrTxtUserCreation', parent=body_style, textColor=colors.HexColor("#DC2626")))) + else: + user_creation_table_data = [[ + Paragraph("Activity", table_cell_header), + Paragraph("Initiated By", table_cell_header) + ]] + + for log in user_creation_logs: + activity = log.get("activity") or "-" + init_by = log.get("initiatedBy") or "-" + + user_creation_table_data.append([ + Paragraph(activity, table_cell_bold), + Paragraph(init_by, table_cell_style) + ]) + + user_creation_table = Table(user_creation_table_data, colWidths=[124, 380]) + user_creation_table.setStyle(TableStyle([ + ('BACKGROUND', (0, 0), (-1, 0), primary_color), + ('ALIGN', (0, 0), (-1, -1), 'LEFT'), + ('VALIGN', (0, 0), (-1, -1), 'MIDDLE'), + ('TOPPADDING', (0, 0), (-1, -1), 5), + ('BOTTOMPADDING', (0, 0), (-1, -1), 5), + ('ROWBACKGROUNDS', (0, 1), (-1, -1), [colors.white, colors.HexColor("#F8FAFC")]), + ('GRID', (0, 0), (-1, -1), 0.5, outline_color), + ])) + story.append(user_creation_table) + story.append(Spacer(1, 4)) + story.append(Paragraph("* Based on sampled data collected from audit logs.", body_style)) + + story.append(Spacer(1, 15)) + + # 2c. Provisioning Logs Table + story.append(Paragraph("Provisioning Logs", h2_style)) + story.append(Paragraph("This section displays directory provisioning audit logs, indicating identity synchronization actions, status info, and target details.", body_style)) + story.append(Spacer(1, 8)) + + provisioning_logs = dir_data.get("provisioning_logs", []) + if not provisioning_logs: + story.append(Paragraph("No provisioning audit logs discovered.", body_style)) + elif provisioning_logs[0].get("initiatedBy") == "ERROR": + err_msg = provisioning_logs[0].get("provisioningAction") + story.append(Paragraph(f"Error: {err_msg}", ParagraphStyle('ErrTxtProvisioning', parent=body_style, textColor=colors.HexColor("#DC2626")))) + else: + prov_table_data = [[ + Paragraph("Initiated By", small_table_cell_header), + Paragraph("Action", small_table_cell_header), + Paragraph("Steps", small_table_cell_header), + Paragraph("Service Principal", small_table_cell_header), + Paragraph("Source System", small_table_cell_header), + Paragraph("Target System", small_table_cell_header), + Paragraph("Tenant ID", small_table_cell_header), + Paragraph("Status Info", small_table_cell_header) + ]] + + for log in provisioning_logs: + initiatedBy = log.get("initiatedBy") or "-" + action = log.get("provisioningAction") or "-" + steps = log.get("provisioningSteps") or "-" + sp = log.get("servicePrincipal") or "-" + src = log.get("sourceSystem") or "-" + tgt = log.get("targetSystem") or "-" + tenant = log.get("tenantId") or "-" + statusInfo = log.get("provisioningStatusInfo") or "-" + + prov_table_data.append([ + Paragraph(initiatedBy, small_table_cell_style), + Paragraph(action, small_table_cell_bold), + Paragraph(steps, small_table_cell_style), + Paragraph(sp, small_table_cell_style), + Paragraph(src, small_table_cell_style), + Paragraph(tgt, small_table_cell_style), + Paragraph(tenant, small_table_cell_style), + Paragraph(statusInfo, small_table_cell_style) + ]) + + prov_table = Table(prov_table_data, colWidths=[64, 60, 80, 60, 40, 40, 40, 120]) + prov_table.setStyle(TableStyle([ + ('BACKGROUND', (0, 0), (-1, 0), primary_color), + ('ALIGN', (0, 0), (-1, -1), 'LEFT'), + ('VALIGN', (0, 0), (-1, -1), 'TOP'), + ('TOPPADDING', (0, 0), (-1, -1), 4), + ('BOTTOMPADDING', (0, 0), (-1, -1), 4), + ('LEFTPADDING', (0, 0), (-1, -1), 3), + ('RIGHTPADDING', (0, 0), (-1, -1), 3), + ('ROWBACKGROUNDS', (0, 1), (-1, -1), [colors.white, colors.HexColor("#F8FAFC")]), + ('GRID', (0, 0), (-1, -1), 0.5, outline_color), + ])) + story.append(prov_table) + story.append(Spacer(1, 4)) + story.append(Paragraph("* Based on sampled data collected from audit logs.", body_style)) + + story.append(Spacer(1, 15)) + + # 3. Groups & Users Table + story.append(Paragraph("Groups & Users", h2_style)) + story.append(Paragraph("This section displays counts of different directory user and group categories configured in Microsoft Entra ID.", body_style)) + story.append(Spacer(1, 8)) + + group_counts = dir_data.get("group_counts", {}) + user_counts = dir_data.get("user_counts", {}) + + dir_table_data = [[ + Paragraph("Category", table_cell_header), + Paragraph("Count", table_cell_header) + ]] + + rows_spec = [ + # User statistics + ("Total Users", user_counts.get("total", 0), True), + ("Enabled Users", user_counts.get("enabled", 0), False), + ("Disabled Users", user_counts.get("disabled", 0), False), + ("Member Users", user_counts.get("member", 0), False), + ("Guest Users", user_counts.get("guest", 0), False), + # Spacing placeholder + ("", "", False), + # Group statistics + ("Total Groups", group_counts.get("total", 0), True), + ("Microsoft 365 Groups (Unified)", group_counts.get("m365", 0), False), + ("Security Groups (Static, non-mail-enabled)", group_counts.get("security", 0), False), + ("Mail-enabled Security Groups", group_counts.get("mail_enabled_security", 0), False), + ("Distribution Groups", group_counts.get("distribution", 0), False), + ("Dynamic Groups (Dynamic Membership)", group_counts.get("dynamic", 0), False) + ] + + row_backgrounds = [] + for idx, item in enumerate(rows_spec, start=1): + metric_name, val, is_bold = item + if metric_name == "": + dir_table_data.append([Paragraph("", table_cell_style), Paragraph("", table_cell_style)]) + # Divider background color + row_backgrounds.append((idx, colors.HexColor("#CBD5E1"))) + continue + + cell_bold = table_cell_bold if is_bold else table_cell_style + dir_table_data.append([ + Paragraph(metric_name, cell_bold), + Paragraph(f"{val:,}", table_cell_style) + ]) + # Alternate row background + bg = colors.white if idx % 2 == 0 else colors.HexColor("#F8FAFC") + row_backgrounds.append((idx, bg)) + + dir_table_style = [ + ('BACKGROUND', (0, 0), (-1, 0), primary_color), + ('ALIGN', (0, 0), (-1, -1), 'LEFT'), + ('VALIGN', (0, 0), (-1, -1), 'MIDDLE'), + ('TOPPADDING', (0, 0), (-1, -1), 5), + ('BOTTOMPADDING', (0, 0), (-1, -1), 5), + ('GRID', (0, 0), (-1, -1), 0.5, outline_color), + ] + + for r_idx, bg_color in row_backgrounds: + dir_table_style.append(('BACKGROUND', (0, r_idx), (-1, r_idx), bg_color)) + + dir_table = Table(dir_table_data, colWidths=[300, 200]) + dir_table.setStyle(TableStyle(dir_table_style)) + story.append(dir_table) + story.append(Spacer(1, 15)) + + + # ========================================================================= + # SECTION 2: APP USAGE SUMMARY + # ========================================================================= + story.append(Paragraph("2. App Usage Summary", h1_style)) + story.append(Paragraph("Active Users Usage", h2_style)) + story.append(Paragraph("A breakdown of user activity across major Microsoft 365 services over the last 30, 90, and 180 days, representing actual adoption levels.", body_style)) + story.append(Spacer(1, 8)) + + o365_usage = data.get("o365_usage", []) + if not o365_usage: + story.append(Paragraph("No active user usage report data was available.", ParagraphStyle('ErrTxt', parent=body_style, textColor=colors.HexColor("#DC2626")))) + else: + usage_table_data = [[ + Paragraph("Service / License", table_cell_header), + Paragraph("30 Days Active", table_cell_header), + Paragraph("90 Days Active", table_cell_header), + Paragraph("180 Days Active", table_cell_header) + ]] + + for row in o365_usage: + usage_table_data.append([ + Paragraph(str(row[0]), table_cell_bold), + Paragraph(f"{row[1]:,}", table_cell_style), + Paragraph(f"{row[2]:,}", table_cell_style), + Paragraph(f"{row[3]:,}", table_cell_style) + ]) + + usage_table = Table(usage_table_data, colWidths=[200, 100, 100, 100]) + usage_table.setStyle(TableStyle([ + ('BACKGROUND', (0, 0), (-1, 0), primary_color), + ('ALIGN', (0, 0), (-1, -1), 'LEFT'), + ('VALIGN', (0, 0), (-1, -1), 'MIDDLE'), + ('TOPPADDING', (0, 0), (-1, -1), 6), + ('BOTTOMPADDING', (0, 0), (-1, -1), 6), + ('ROWBACKGROUNDS', (0, 1), (-1, -1), [colors.white, colors.HexColor("#F8FAFC")]), + ('GRID', (0, 0), (-1, -1), 0.5, outline_color), + ])) + story.append(usage_table) + + # 30-Day Trend Chart - Generated on the fly + o365_trend = data.get("o365_trend", {}) + if o365_trend and o365_trend.get("dates"): + try: + chart_bytes = generate_trend_chart_bytes(o365_trend) + if chart_bytes: + story.append(Spacer(1, 15)) + story.append(Paragraph("O365 30-Day Active User Trend", h2_style)) + chart_flow = Image(chart_bytes, width=450, height=210) + story.append(chart_flow) + except Exception as chart_ex: + print(f"Failed to generate active user trend chart for PDF: {chart_ex}") + + story.append(PageBreak()) + + # M365 Apps Usage + story.append(Paragraph("Microsoft 365 Client Applications Usage (180 Days)", h2_style)) + story.append(Paragraph("Displays the unique counts of active users on client applications (Outlook, Word, Excel, PowerPoint, OneNote, Teams) segmented by system platforms.", body_style)) + story.append(Spacer(1, 8)) + + m365_apps = data.get("m365_apps", []) + if not m365_apps: + story.append(Paragraph("No client application telemetry data was available.", ParagraphStyle('ErrTxt', parent=body_style, textColor=colors.HexColor("#DC2626")))) + else: + # Format 4-columns layout matching the UI table + app_table_data = [[ + Paragraph("App / Platform", table_cell_header), + Paragraph("Active Users", table_cell_header), + Paragraph("App / Platform", table_cell_header), + Paragraph("Active Users", table_cell_header) + ]] + + half = (len(m365_apps) + 1) // 2 + left_col = m365_apps[:half] + right_col = m365_apps[half:] + + for r_idx in range(half): + l_name = left_col[r_idx][0] if r_idx < len(left_col) else "" + l_val = f"{left_col[r_idx][1]:,}" if r_idx < len(left_col) else "" + r_name = right_col[r_idx][0] if r_idx < len(right_col) else "" + r_val = f"{right_col[r_idx][1]:,}" if r_idx < len(right_col) else "" + + app_table_data.append([ + Paragraph(l_name, table_cell_bold if l_name else table_cell_style), + Paragraph(l_val, table_cell_style), + Paragraph(r_name, table_cell_bold if r_name else table_cell_style), + Paragraph(r_val, table_cell_style) + ]) + + app_table = Table(app_table_data, colWidths=[150, 100, 150, 100]) + app_table.setStyle(TableStyle([ + ('BACKGROUND', (0, 0), (-1, 0), primary_color), + ('ALIGN', (0, 0), (-1, -1), 'LEFT'), + ('VALIGN', (0, 0), (-1, -1), 'MIDDLE'), + ('TOPPADDING', (0, 0), (-1, -1), 5), + ('BOTTOMPADDING', (0, 0), (-1, -1), 5), + ('ROWBACKGROUNDS', (0, 1), (-1, -1), [colors.white, colors.HexColor("#F8FAFC")]), + ('GRID', (0, 0), (-1, -1), 0.5, outline_color), + ])) + story.append(app_table) + + story.append(Spacer(1, 15)) + + # ========================================================================= + # SECTION 3: WORKLOAD STORAGE & METRICS + # ========================================================================= + story.append(Paragraph("3. Workload Storage & Environmental Telemetry", h1_style)) + story.append(Paragraph("A compiled summary of storage consumption, item counts, and device statistics across Exchange Online, SharePoint, and OneDrive workloads.", body_style)) + story.append(Spacer(1, 8)) + + # 3.1 Exchange Mailbox & Calendar Telemetry + story.append(Paragraph("Exchange Online Mailbox & Resource Configurations", h2_style)) + + mailbox = data.get("mailbox", {}) + calendar = data.get("calendar", {}) + + # Let's check for warning/errors + pw_warn = [] + if mailbox.get("powershell_error"): + pw_warn.append(f"Mailbox: {mailbox['powershell_error']}") + if calendar.get("powershell_error"): + pw_warn.append(f"Calendar: {calendar['powershell_error']}") + + if pw_warn: + story.append(Paragraph(f"⚠️ Warning: PowerShell metrics are restricted or incomplete ({'; '.join(pw_warn)})", ParagraphStyle('WarnTxt', parent=body_style, textColor=colors.HexColor("#D97706"), fontName="Helvetica-Bold"))) + story.append(Spacer(1, 4)) + + workload_table_data = [[ + Paragraph("Metric / Telemetry Property", table_cell_header), + Paragraph("Exchange Mailbox Value", table_cell_header) + ]] + + # Compile rows from mailbox & calendar + exchange_rows = [ + ("Total Mailboxes Analyzed", f"{mailbox.get('total_mailboxes', 0):,} Mailboxes"), + ("Total Size of All Mailboxes", mailbox.get("total_storage_formatted", "0.00 Bytes")), + ("Average Mailbox Size", mailbox.get("average_mailbox_size_formatted", "0.00 Bytes")), + ("Total Emails Volume", f"{mailbox.get('total_emails', 0):,} Emails"), + ("Average Emails per Mailbox", f"{mailbox.get('average_emails', 0.0):,.0f} Emails"), + ] + + s_count = mailbox.get('shared_mailboxes_count') + s_count_str = f"{s_count:,} Shared Mailboxes" if s_count is not None else "Error/Unavailable" + s_size_str = mailbox.get("shared_mailboxes_total_formatted", "Error/Unavailable") + + pf_count = mailbox.get('public_folders_count') + pf_count_str = f"{pf_count:,} Public Folders" if pf_count is not None else "Error/Unavailable" + + mail_pf_count = mailbox.get('mail_public_folders_count') + mail_pf_count_str = f"{mail_pf_count:,} Public Folders" if mail_pf_count is not None else "Error/Unavailable" + + pf_size_str = mailbox.get("public_folders_total_formatted", "Error/Unavailable") + + exchange_rows += [ + ("Shared Mailboxes Count", s_count_str), + ("Total Shared Mailbox Size", s_size_str), + ("Public Folders Count", pf_count_str), + ("Mail-enabled Public Folders Count", mail_pf_count_str), + ("Total Public Folder Size", pf_size_str), + ] + + # Add calendar properties + reserve_val = calendar.get("CanUsersReserveRooms") + if isinstance(reserve_val, bool): reserve_val = "Yes" if reserve_val else "No" + + att_val = calendar.get("CanShareAttachments") + if isinstance(att_val, bool): att_val = "Yes" if att_val else "No" + + exchange_rows += [ + ("Room & Resource Reservation Enabled", str(reserve_val)), + ("Calendar Resource Pools (Rooms/Devices)", calendar.get("NamingConvention") or "None found"), + ("Calendar Attachment Link Permissions", str(att_val)), + ] + + for label, val in exchange_rows: + workload_table_data.append([ + Paragraph(label, table_cell_bold), + Paragraph(val, table_cell_style) + ]) + + ex_table = Table(workload_table_data, colWidths=[260, 240]) + ex_table.setStyle(TableStyle([ + ('BACKGROUND', (0, 0), (-1, 0), primary_color), + ('ALIGN', (0, 0), (-1, -1), 'LEFT'), + ('VALIGN', (0, 0), (-1, -1), 'MIDDLE'), + ('TOPPADDING', (0, 0), (-1, -1), 5), + ('BOTTOMPADDING', (0, 0), (-1, -1), 5), + ('ROWBACKGROUNDS', (0, 1), (-1, -1), [colors.white, colors.HexColor("#F8FAFC")]), + ('GRID', (0, 0), (-1, -1), 0.5, outline_color), + ])) + story.append(ex_table) + story.append(Spacer(1, 15)) + + # 3.1b Integrated Apps + story.append(Paragraph("Integrated Apps", h2_style)) + story.append(Paragraph("This section lists all organization-wide apps deployed in Exchange Online by administrators and their enabled status.", body_style)) + story.append(Spacer(1, 8)) + + org_apps = calendar.get("OrganizationApps", []) + apps_error = calendar.get("AppsError") + + if apps_error: + story.append(Paragraph(f"Error querying organization apps: {apps_error}", ParagraphStyle('ErrTxt', parent=body_style, textColor=colors.HexColor("#DC2626")))) + elif not org_apps: + story.append(Paragraph("No organization-wide apps found in Exchange Online.", body_style)) + else: + apps_table_data = [[ + Paragraph("App Display Name", table_cell_header), + Paragraph("Status", table_cell_header), + Paragraph("App Display Name", table_cell_header), + Paragraph("Status", table_cell_header) + ]] + half = (len(org_apps) + 1) // 2 + left_col = org_apps[:half] + right_col = org_apps[half:] + + for r_idx in range(half): + row_items = [] + if r_idx < len(left_col): + app = left_col[r_idx] + enabled_str = "Enabled" if app.get("Enabled") else "Disabled" + row_items.extend([app.get("DisplayName", "-"), enabled_str]) + else: + row_items.extend(["", ""]) + + if r_idx < len(right_col): + app = right_col[r_idx] + enabled_str = "Enabled" if app.get("Enabled") else "Disabled" + row_items.extend([app.get("DisplayName", "-"), enabled_str]) + else: + row_items.extend(["", ""]) + + apps_table_data.append([ + Paragraph(row_items[0], table_cell_bold if row_items[0] else table_cell_style), + Paragraph(row_items[1], table_cell_style), + Paragraph(row_items[2], table_cell_bold if row_items[2] else table_cell_style), + Paragraph(row_items[3], table_cell_style) + ]) + + apps_table = Table(apps_table_data, colWidths=[180, 70, 180, 70]) + apps_table.setStyle(TableStyle([ + ('BACKGROUND', (0, 0), (-1, 0), primary_color), + ('ALIGN', (0, 0), (-1, -1), 'LEFT'), + ('VALIGN', (0, 0), (-1, -1), 'MIDDLE'), + ('TOPPADDING', (0, 0), (-1, -1), 5), + ('BOTTOMPADDING', (0, 0), (-1, -1), 5), + ('ROWBACKGROUNDS', (0, 1), (-1, -1), [colors.white, colors.HexColor("#F8FAFC")]), + ('GRID', (0, 0), (-1, -1), 0.5, outline_color), + ])) + story.append(apps_table) + + story.append(Spacer(1, 15)) + + # 3.1c Exchange Connectors + story.append(Paragraph("Exchange Connectors", h2_style)) + story.append(Paragraph("This section displays mail routing connectors configured in Exchange Online.", body_style)) + story.append(Spacer(1, 8)) + + connectors = data.get("exchange_connectors", []) + if not connectors: + story.append(Paragraph("No Exchange connectors configured.", body_style)) + else: + conn_table_data = [[ + Paragraph("Direction", table_cell_header), + Paragraph("Connector Name", table_cell_header), + Paragraph("Status", table_cell_header), + Paragraph("Domains", table_cell_header), + Paragraph("Routing Config", table_cell_header) + ]] + for conn in connectors: + routing_txt = conn.get("Routing", "-").replace("\n", "
") + conn_table_data.append([ + Paragraph(conn.get("Direction", "-"), table_cell_style), + Paragraph(conn.get("Name", "-"), table_cell_bold), + Paragraph(conn.get("Status", "-"), table_cell_style), + Paragraph(conn.get("Domains", "-"), table_cell_style), + Paragraph(routing_txt, table_cell_style) + ]) + conn_table = Table(conn_table_data, colWidths=[70, 120, 60, 100, 154]) + conn_table.setStyle(TableStyle([ + ('BACKGROUND', (0, 0), (-1, 0), primary_color), + ('ALIGN', (0, 0), (-1, -1), 'LEFT'), + ('VALIGN', (0, 0), (-1, -1), 'MIDDLE'), + ('TOPPADDING', (0, 0), (-1, -1), 5), + ('BOTTOMPADDING', (0, 0), (-1, -1), 5), + ('ROWBACKGROUNDS', (0, 1), (-1, -1), [colors.white, colors.HexColor("#F8FAFC")]), + ('GRID', (0, 0), (-1, -1), 0.5, outline_color), + ])) + story.append(conn_table) + story.append(Spacer(1, 15)) + + # 3.1d Email Clients & PST Files + story.append(Paragraph("Email Clients & PST Environment", h2_style)) + story.append(Paragraph("Overview of email client adoption and PST configuration.", body_style)) + story.append(Spacer(1, 8)) + + email_clients = data.get("email_clients", {}) + if not email_clients: + story.append(Paragraph("No email client telemetry data available.", body_style)) + else: + ec_table_data = [[ + Paragraph("Client Type", table_cell_header), + Paragraph("Active Users", table_cell_header) + ]] + rows = [ + ("Outlook on the Web (OWA)", email_clients.get("client_browser", 0)), + ("Outlook for Windows", email_clients.get("client_win_outlook", 0)), + ("Outlook for Mac", email_clients.get("client_mac_outlook", 0)), + ("Apple Mail (macOS)", email_clients.get("client_mac_mail", 0)), + ("Other Desktop Apps", email_clients.get("client_desktop_other", 0)), + ("Outlook Mobile (iOS/Android)", email_clients.get("client_mobile_outlook", 0)), + ("Native / Other Mobile Apps", email_clients.get("client_mobile_other", 0)), + ("IMAP4 Apps", email_clients.get("client_imap", 0)), + ("POP3 Apps", email_clients.get("client_pop", 0)), + ("SMTP Apps", email_clients.get("client_smtp", 0)) + ] + for label, val in rows: + ec_table_data.append([ + Paragraph(label, table_cell_bold), + Paragraph(f"{val:,} Users" if 'SMTP' not in label else f"{val:,} Accounts", table_cell_style) + ]) + ec_table = Table(ec_table_data, colWidths=[250, 150]) + ec_table.setStyle(TableStyle([ + ('BACKGROUND', (0, 0), (-1, 0), primary_color), + ('ALIGN', (0, 0), (-1, -1), 'LEFT'), + ('VALIGN', (0, 0), (-1, -1), 'MIDDLE'), + ('TOPPADDING', (0, 0), (-1, -1), 5), + ('BOTTOMPADDING', (0, 0), (-1, -1), 5), + ('ROWBACKGROUNDS', (0, 1), (-1, -1), [colors.white, colors.HexColor("#F8FAFC")]), + ('GRID', (0, 0), (-1, -1), 0.5, outline_color), + ])) + story.append(ec_table) + story.append(Spacer(1, 10)) + + pst_files = data.get("pst_files", {}) + if pst_files: + pst_table_data = [[Paragraph("PST Metric", table_cell_header), Paragraph("Value", table_cell_header)]] + pst_cloud = pst_files.get("pst_cloud_data", {}) + cloud_count = 0 + cloud_bytes = 0 + if pst_cloud and "value" in pst_cloud: + for item in pst_cloud.get("value", []): + for hc in item.get("hitsContainers", []): + cloud_count += hc.get("total", 0) + for hit in hc.get("hits", []): + cloud_bytes += int(hit.get("resource", {}).get("size", 0)) + + def format_bytes(size): + for unit in ['Bytes', 'KB', 'MB', 'GB', 'TB']: + if size < 1024.0: return f"{size:.2f} {unit}" + size /= 1024.0 + return f"{size:.2f} PB" + + cloud_size_str = f" ({format_bytes(cloud_bytes)})" if cloud_bytes > 0 else "" + cloud_str = f"{cloud_count:,} Files{cloud_size_str}" if cloud_count > 0 else "None Detected" + + pst_table_data.append([Paragraph("Cloud (SharePoint & OneDrive)", table_cell_bold), Paragraph(cloud_str, table_cell_style)]) + + pst_table = Table(pst_table_data, colWidths=[250, 150]) + pst_table.setStyle(TableStyle([ + ('BACKGROUND', (0, 0), (-1, 0), primary_color), + ('ALIGN', (0, 0), (-1, -1), 'LEFT'), + ('VALIGN', (0, 0), (-1, -1), 'MIDDLE'), + ('TOPPADDING', (0, 0), (-1, -1), 5), + ('BOTTOMPADDING', (0, 0), (-1, -1), 5), + ('ROWBACKGROUNDS', (0, 1), (-1, -1), [colors.white, colors.HexColor("#F8FAFC")]), + ('GRID', (0, 0), (-1, -1), 0.5, outline_color), + ])) + story.append(pst_table) + + story.append(Spacer(1, 15)) + story.append(PageBreak()) + + # 3.2 SharePoint & OneDrive Storage + story.append(Paragraph("SharePoint & OneDrive Environment Telemetry", h2_style)) + story.append(Paragraph("A comparison of file volume, storage consumption, site activity, and active synchronization clients.", body_style)) + story.append(Spacer(1, 8)) + + sp = data.get("sharepoint", {}) + od = data.get("onedrive", {}) + + files_table_data = [[ + Paragraph("Metric Property Description", table_cell_header), + Paragraph("SharePoint Sites (180d)", table_cell_header), + Paragraph("OneDrive Personal (180d)", table_cell_header) + ]] + + files_rows = [ + ("Total Scope Count (Sites / Accounts)", f"{sp.get('total_sites', 0):,} Sites", f"{od.get('total_accounts', 0):,} Accounts"), + ("Total Storage Consumed", sp.get("total_storage_formatted", "0.00 Bytes"), od.get("total_storage_formatted", "0.00 Bytes")), + ("Total Stored File Count", f"{sp.get('total_files', 0):,} Files", f"{od.get('total_files', 0):,} Files"), + ("Active Files Count (Active %)", f"{sp.get('active_files', 0):,} ({sp.get('active_files_pct', 0.0):.1f}%)", f"{od.get('active_files', 0):,} ({od.get('active_files_pct', 0.0):.1f}%)"), + ("Users with Sync Client Active", "N/A (SharePoint level)", f"{od.get('sync_users', 0):,} Users ({od.get('sync_users_pct', 0.0):.1f}%)"), + ("Active OneNote Users", "N/A (SharePoint level)", f"{od.get('onenote_users', 0):,} Users"), + ] + + for label, sp_val, od_val in files_rows: + files_table_data.append([ + Paragraph(label, table_cell_bold), + Paragraph(sp_val, table_cell_style), + Paragraph(od_val, table_cell_style) + ]) + + files_table = Table(files_table_data, colWidths=[200, 150, 150]) + files_table.setStyle(TableStyle([ + ('BACKGROUND', (0, 0), (-1, 0), primary_color), + ('ALIGN', (0, 0), (-1, -1), 'LEFT'), + ('VALIGN', (0, 0), (-1, -1), 'MIDDLE'), + ('TOPPADDING', (0, 0), (-1, -1), 6), + ('BOTTOMPADDING', (0, 0), (-1, -1), 6), + ('ROWBACKGROUNDS', (0, 1), (-1, -1), [colors.white, colors.HexColor("#F8FAFC")]), + ('GRID', (0, 0), (-1, -1), 0.5, outline_color), + ])) + story.append(files_table) + story.append(Spacer(1, 15)) + + sp_data_types = data.get("sharepoint_data_types", {}) + if sp_data_types: + story.append(Paragraph("SharePoint Data Types (Tenant Wide)", h2_style)) + story.append(Paragraph("A global count of major SharePoint components across the tenant.", body_style)) + story.append(Spacer(1, 8)) + + sp_dt_table_data = [[ + Paragraph("Data Type", table_cell_header), + Paragraph("Count", table_cell_header) + ]] + + for k, v in [("Document Libraries", sp_data_types.get("Document Libraries", 0)), + ("Lists", sp_data_types.get("Lists", 0)), + ("Web Pages", sp_data_types.get("Web Pages", 0))]: + sp_dt_table_data.append([ + Paragraph(k, table_cell_bold), + Paragraph(f"{v:,}", table_cell_style) + ]) + + sp_dt_table = Table(sp_dt_table_data, colWidths=[250, 250]) + sp_dt_table.setStyle(TableStyle([ + ('BACKGROUND', (0, 0), (-1, 0), primary_color), + ('ALIGN', (0, 0), (-1, -1), 'LEFT'), + ('VALIGN', (0, 0), (-1, -1), 'MIDDLE'), + ('TOPPADDING', (0, 0), (-1, -1), 5), + ('BOTTOMPADDING', (0, 0), (-1, -1), 5), + ('ROWBACKGROUNDS', (0, 1), (-1, -1), [colors.white, colors.HexColor("#F8FAFC")]), + ('GRID', (0, 0), (-1, -1), 0.5, outline_color), + ])) + story.append(sp_dt_table) + story.append(Spacer(1, 15)) + + # 3.2b Microsoft Teams Overview + story.append(Paragraph("Microsoft Teams Overview", h2_style)) + story.append(Paragraph("A summary of Microsoft Teams activity, including active users, guests, and meetings organized over the last 180 days.", body_style)) + story.append(Spacer(1, 8)) + + msteams_data = data.get("msteams_activity", []) + if not msteams_data: + story.append(Paragraph("No Microsoft Teams activity data was available.", ParagraphStyle('ErrTxt', parent=body_style, textColor=colors.HexColor("#DC2626")))) + else: + teams_table_data = [[ + Paragraph("Team Name", table_cell_header), + Paragraph("Last Activity", table_cell_header), + Paragraph("Active Users", table_cell_header), + Paragraph("Guests", table_cell_header), + Paragraph("Meetings", table_cell_header), + Paragraph("Messages", table_cell_header) + ]] + + for row in msteams_data[:20]: + teams_table_data.append([ + Paragraph(row.get("Team Name", "-"), table_cell_bold), + Paragraph(row.get("Last Activity Date", "-"), table_cell_style), + Paragraph(row.get("Active Users", "0"), table_cell_style), + Paragraph(row.get("Guests", "0"), table_cell_style), + Paragraph(row.get("Meetings Organized", "0"), table_cell_style), + Paragraph(row.get("Channel Messages", "0"), table_cell_style) + ]) + + teams_table = Table(teams_table_data, colWidths=[120, 70, 70, 50, 70, 70]) + teams_table.setStyle(TableStyle([ + ('BACKGROUND', (0, 0), (-1, 0), primary_color), + ('ALIGN', (0, 0), (-1, -1), 'LEFT'), + ('VALIGN', (0, 0), (-1, -1), 'MIDDLE'), + ('TOPPADDING', (0, 0), (-1, -1), 5), + ('BOTTOMPADDING', (0, 0), (-1, -1), 5), + ('ROWBACKGROUNDS', (0, 1), (-1, -1), [colors.white, colors.HexColor("#F8FAFC")]), + ('GRID', (0, 0), (-1, -1), 0.5, outline_color), + ])) + story.append(teams_table) + story.append(Spacer(1, 15)) + + # 3.3 Microsoft Entra Data + story.append(Paragraph("Microsoft Entra Data", h2_style)) + story.append(Paragraph("This section outlines application sign-in metrics and authentication methods configuration summaries.", body_style)) + story.append(Spacer(1, 8)) + + entra_data = data.get("devices_apps", {}) + + # 3.3.2 App Sign Ins + story.append(Paragraph("App Sign Ins", body_style)) + story.append(Spacer(1, 4)) + + app_signins = entra_data.get("app_signins", []) + if not app_signins: + story.append(Paragraph("No Azure AD application sign-in logs were discovered or permission restricted.", ParagraphStyle('ErrTxtAppSignins', parent=body_style, textColor=colors.HexColor("#DC2626")))) + else: + app_signins_table_data = [[ + Paragraph("App Name", table_cell_header), + Paragraph("Successful Sign Ins", table_cell_header) + ]] + + for app, success in app_signins: + app_signins_table_data.append([ + Paragraph(app, table_cell_bold), + Paragraph(success, table_cell_style) + ]) + + app_signins_table = Table(app_signins_table_data, colWidths=[250, 254]) + app_signins_table.setStyle(TableStyle([ + ('BACKGROUND', (0, 0), (-1, 0), primary_color), + ('ALIGN', (0, 0), (-1, -1), 'LEFT'), + ('VALIGN', (0, 0), (-1, -1), 'MIDDLE'), + ('TOPPADDING', (0, 0), (-1, -1), 5), + ('BOTTOMPADDING', (0, 0), (-1, -1), 5), + ('ROWBACKGROUNDS', (0, 1), (-1, -1), [colors.white, colors.HexColor("#F8FAFC")]), + ('GRID', (0, 0), (-1, -1), 0.5, outline_color), + ])) + story.append(app_signins_table) + + story.append(Spacer(1, 15)) + + # 3.3.3 App Registrations + story.append(Paragraph("App Registrations", body_style)) + story.append(Spacer(1, 4)) + + app_registrations = entra_data.get("app_registrations", []) + if not app_registrations: + story.append(Paragraph("No Azure AD app registrations were discovered or permission restricted.", ParagraphStyle('ErrTxtAppRegs', parent=body_style, textColor=colors.HexColor("#DC2626")))) + else: + app_regs_table_data = [[ + Paragraph("App Name", table_cell_header), + Paragraph("Application ID", table_cell_header), + Paragraph("Created Date", table_cell_header), + Paragraph("Sign In Audience", table_cell_header), + Paragraph("Credentials", table_cell_header) + ]] + + for name, app_id, created, audience, creds in app_registrations: + formatted_created = created[:10] if created else "" + app_regs_table_data.append([ + Paragraph(name, table_cell_bold), + Paragraph(app_id, table_cell_style), + Paragraph(formatted_created, table_cell_style), + Paragraph(audience, table_cell_style), + Paragraph(creds, table_cell_style) + ]) + + app_regs_table = Table(app_regs_table_data, colWidths=[120, 110, 70, 110, 94]) + app_regs_table.setStyle(TableStyle([ + ('BACKGROUND', (0, 0), (-1, 0), primary_color), + ('ALIGN', (0, 0), (-1, -1), 'LEFT'), + ('VALIGN', (0, 0), (-1, -1), 'MIDDLE'), + ('TOPPADDING', (0, 0), (-1, -1), 5), + ('BOTTOMPADDING', (0, 0), (-1, -1), 5), + ('ROWBACKGROUNDS', (0, 1), (-1, -1), [colors.white, colors.HexColor("#F8FAFC")]), + ('GRID', (0, 0), (-1, -1), 0.5, outline_color), + ])) + story.append(app_regs_table) + + story.append(Spacer(1, 15)) + + # 3.3.4 User Sign-Ins + story.append(Paragraph("User Sign-Ins", body_style)) + story.append(Spacer(1, 4)) + + user_signins = entra_data.get("user_signins", {}) + if not user_signins or (not user_signins.get("apps") and not user_signins.get("os") and not user_signins.get("browsers")): + story.append(Paragraph("No successful user sign-in logs were discovered or permission restricted.", ParagraphStyle('ErrTxtUserSignins', parent=body_style, textColor=colors.HexColor("#DC2626")))) + else: + user_signins_table_data = [ + [Paragraph("Sign-in Attribute", table_cell_header), Paragraph("Successful Unique Values", table_cell_header)], + [Paragraph("App Display Names", table_cell_bold), Paragraph(", ".join(user_signins.get("apps", [])) or "None", table_cell_style)], + [Paragraph("Operating Systems", table_cell_bold), Paragraph(", ".join(user_signins.get("os", [])) or "None", table_cell_style)], + [Paragraph("Browsers", table_cell_bold), Paragraph(", ".join(user_signins.get("browsers", [])) or "None", table_cell_style)] + ] + user_table = Table(user_signins_table_data, colWidths=[150, 354]) + user_table.setStyle(TableStyle([ + ('BACKGROUND', (0, 0), (-1, 0), primary_color), + ('ALIGN', (0, 0), (-1, -1), 'LEFT'), + ('VALIGN', (0, 0), (-1, -1), 'MIDDLE'), + ('TOPPADDING', (0, 0), (-1, -1), 5), + ('BOTTOMPADDING', (0, 0), (-1, -1), 5), + ('ROWBACKGROUNDS', (0, 1), (-1, -1), [colors.white, colors.HexColor("#F8FAFC")]), + ('GRID', (0, 0), (-1, -1), 0.5, outline_color), + ])) + story.append(user_table) + story.append(Spacer(1, 4)) + story.append(Paragraph("* Based on sample data collected from signins.", body_style)) + + story.append(Spacer(1, 15)) + + # 3.3.5 Authentication Methods + story.append(Paragraph("Authentication Methods", body_style)) + story.append(Spacer(1, 4)) + + auth_methods = entra_data.get("auth_methods", []) + if not auth_methods: + story.append(Paragraph("No authentication methods logs were discovered or permission restricted.", ParagraphStyle('ErrTxtAuthMethods', parent=body_style, textColor=colors.HexColor("#DC2626")))) + else: + auth_period = entra_data.get("auth_methods_period", "D7") + period_str = auth_period + if period_str.startswith("D"): + period_str = f"{period_str[1:]} days" + auth_table_data = [[ + Paragraph("Authentication Method", table_cell_header), + Paragraph(f"Success Activity Count ({period_str})", table_cell_header) + ]] + + for method, activity in auth_methods: + auth_table_data.append([ + Paragraph(method, table_cell_bold), + Paragraph(activity, table_cell_style) + ]) + + auth_table = Table(auth_table_data, colWidths=[250, 254]) + auth_table.setStyle(TableStyle([ + ('BACKGROUND', (0, 0), (-1, 0), primary_color), + ('ALIGN', (0, 0), (-1, -1), 'LEFT'), + ('VALIGN', (0, 0), (-1, -1), 'MIDDLE'), + ('TOPPADDING', (0, 0), (-1, -1), 5), + ('BOTTOMPADDING', (0, 0), (-1, -1), 5), + ('ROWBACKGROUNDS', (0, 1), (-1, -1), [colors.white, colors.HexColor("#F8FAFC")]), + ('GRID', (0, 0), (-1, -1), 0.5, outline_color), + ])) + story.append(auth_table) + + story.append(Spacer(1, 15)) + + # 3.4 Microsoft Intune Data + story.append(Paragraph("Microsoft Intune Data", h2_style)) + story.append(Paragraph("This section contains mobile applications and device configuration policies managed and distributed via Microsoft Intune.", body_style)) + story.append(Spacer(1, 8)) + + intune_data = data.get("intune", {}) + mobile_apps = intune_data.get("mobile_apps", []) + table_rows = intune_data.get("table_rows", []) + + # Render Mobile Apps + story.append(Paragraph("Managed Mobile Apps:", body_style)) + apps_text = ", ".join(mobile_apps) if mobile_apps else "No mobile apps discovered or permission restricted." + story.append(Paragraph(apps_text, body_style)) + story.append(Spacer(1, 10)) + + # Render Managed Devices Table (Top 10) + story.append(Paragraph("Managed Devices (Top 10)", body_style)) + story.append(Spacer(1, 6)) + + managed_devices = intune_data.get("managed_devices", []) + if not managed_devices: + story.append(Paragraph("No managed devices were discovered or permission restricted.", ParagraphStyle('ErrTxtMngDev', parent=body_style, textColor=colors.HexColor("#DC2626")))) + else: + dev_table_data = [[ + Paragraph("User ID", table_cell_header), + Paragraph("Device Name", table_cell_header), + Paragraph("OS", table_cell_header), + Paragraph("Agent", table_cell_header), + Paragraph("State", table_cell_header), + Paragraph("Model", table_cell_header), + Paragraph("Manufacturer", table_cell_header) + ]] + + for dev in managed_devices[:10]: + dev_table_data.append([ + Paragraph(dev.get("userId", "N/A"), table_cell_bold), + Paragraph(dev.get("deviceName", "N/A"), table_cell_style), + Paragraph(dev.get("operatingSystem", "N/A"), table_cell_style), + Paragraph(dev.get("managementAgent", "unknown"), table_cell_style), + Paragraph(dev.get("deviceRegistrationState", "unknown"), table_cell_style), + Paragraph(dev.get("model", "N/A"), table_cell_style), + Paragraph(dev.get("manufacturer", "N/A"), table_cell_style) + ]) + + dev_table = Table(dev_table_data, colWidths=[90, 80, 60, 70, 70, 64, 70]) + dev_table.setStyle(TableStyle([ + ('BACKGROUND', (0, 0), (-1, 0), primary_color), + ('ALIGN', (0, 0), (-1, -1), 'LEFT'), + ('VALIGN', (0, 0), (-1, -1), 'MIDDLE'), + ('TOPPADDING', (0, 0), (-1, -1), 4), + ('BOTTOMPADDING', (0, 0), (-1, -1), 4), + ('ROWBACKGROUNDS', (0, 1), (-1, -1), [colors.white, colors.HexColor("#F8FAFC")]), + ('GRID', (0, 0), (-1, -1), 0.5, outline_color), + ])) + story.append(dev_table) + story.append(Spacer(1, 10)) + + # Render VC Devices Table (Top 10) + story.append(Paragraph("Video Conferencing (VC) Devices (Top 10)", body_style)) + story.append(Spacer(1, 6)) + + vc_devices = intune_data.get("vc_devices", []) + if not vc_devices: + story.append(Paragraph("No Video Conferencing (VC) devices were discovered or matched against room mailboxes.", ParagraphStyle('ErrTxtVCDev', parent=body_style, textColor=colors.HexColor("#DC2626")))) + else: + vc_table_data = [[ + Paragraph("User ID", table_cell_header), + Paragraph("Device Name", table_cell_header), + Paragraph("OS", table_cell_header), + Paragraph("Agent", table_cell_header), + Paragraph("State", table_cell_header), + Paragraph("Model", table_cell_header), + Paragraph("Manufacturer", table_cell_header) + ]] + + for dev in vc_devices[:10]: + vc_table_data.append([ + Paragraph(dev.get("userId", "N/A"), table_cell_bold), + Paragraph(dev.get("deviceName", "N/A"), table_cell_style), + Paragraph(dev.get("operatingSystem", "N/A"), table_cell_style), + Paragraph(dev.get("managementAgent", "unknown"), table_cell_style), + Paragraph(dev.get("deviceRegistrationState", "unknown"), table_cell_style), + Paragraph(dev.get("model", "N/A"), table_cell_style), + Paragraph(dev.get("manufacturer", "N/A"), table_cell_style) + ]) + + vc_table = Table(vc_table_data, colWidths=[90, 80, 60, 70, 70, 64, 70]) + vc_table.setStyle(TableStyle([ + ('BACKGROUND', (0, 0), (-1, 0), primary_color), + ('ALIGN', (0, 0), (-1, -1), 'LEFT'), + ('VALIGN', (0, 0), (-1, -1), 'MIDDLE'), + ('TOPPADDING', (0, 0), (-1, -1), 4), + ('BOTTOMPADDING', (0, 0), (-1, -1), 4), + ('ROWBACKGROUNDS', (0, 1), (-1, -1), [colors.white, colors.HexColor("#F8FAFC")]), + ('GRID', (0, 0), (-1, -1), 0.5, outline_color), + ])) + story.append(vc_table) + story.append(Spacer(1, 10)) + + # Render Device Configurations Table + story.append(Paragraph("Device Configurations", body_style)) + story.append(Spacer(1, 6)) + + if not table_rows: + story.append(Paragraph("No device configuration policies were discovered or permission restricted.", ParagraphStyle('ErrTxtIntune', parent=body_style, textColor=colors.HexColor("#DC2626")))) + else: + intune_table_data = [[ + Paragraph("Platform", table_cell_header), + Paragraph("Policy Type", table_cell_header), + Paragraph("Number of Policies", table_cell_header) + ]] + + for platform, p_type, count in table_rows: + intune_table_data.append([ + Paragraph(platform, table_cell_bold), + Paragraph(p_type, table_cell_style), + Paragraph(count, table_cell_style) + ]) + + intune_table = Table(intune_table_data, colWidths=[150, 200, 154]) + intune_table.setStyle(TableStyle([ + ('BACKGROUND', (0, 0), (-1, 0), primary_color), + ('ALIGN', (0, 0), (-1, -1), 'LEFT'), + ('VALIGN', (0, 0), (-1, -1), 'MIDDLE'), + ('TOPPADDING', (0, 0), (-1, -1), 5), + ('BOTTOMPADDING', (0, 0), (-1, -1), 5), + ('ROWBACKGROUNDS', (0, 1), (-1, -1), [colors.white, colors.HexColor("#F8FAFC")]), + ('GRID', (0, 0), (-1, -1), 0.5, outline_color), + ])) + story.append(intune_table) + story.append(Spacer(1, 4)) + story.append(Paragraph("* Based on sample data collected from Intune.", body_style)) + story.append(Spacer(1, 10)) + + # Render Mobile Device Compliance Policies Section + story.append(Paragraph("Mobile Device Compliance Policies", body_style)) + story.append(Spacer(1, 6)) + + # 1. Android Devices Table + story.append(Paragraph("Android Devices (Top 10)", body_style)) + story.append(Spacer(1, 4)) + android_compliance = intune_data.get("android_compliance", []) + if not android_compliance: + story.append(Paragraph("No Android device compliance policies were discovered or permission restricted.", ParagraphStyle('ErrTxtAndCompliance', parent=body_style, textColor=colors.HexColor("#DC2626")))) + else: + compliance_table_data = [[ + Paragraph("Display Name", table_cell_header), + Paragraph("Description", table_cell_header), + Paragraph("Created Time", table_cell_header), + Paragraph("Last Modified", table_cell_header), + Paragraph("Version", table_cell_header) + ]] + for policy in android_compliance[:10]: + compliance_table_data.append([ + Paragraph(policy.get("displayName", "N/A"), table_cell_bold), + Paragraph(policy.get("description", "N/A"), table_cell_style), + Paragraph(policy.get("createdDateTime", "N/A"), table_cell_style), + Paragraph(policy.get("lastModifiedDateTime", "N/A"), table_cell_style), + Paragraph(str(policy.get("version", 0)), table_cell_style) + ]) + compliance_table = Table(compliance_table_data, colWidths=[120, 150, 100, 100, 34]) + compliance_table.setStyle(TableStyle([ + ('BACKGROUND', (0, 0), (-1, 0), primary_color), + ('ALIGN', (0, 0), (-1, -1), 'LEFT'), + ('VALIGN', (0, 0), (-1, -1), 'MIDDLE'), + ('TOPPADDING', (0, 0), (-1, -1), 5), + ('BOTTOMPADDING', (0, 0), (-1, -1), 5), + ('ROWBACKGROUNDS', (0, 1), (-1, -1), [colors.white, colors.HexColor("#F8FAFC")]), + ('GRID', (0, 0), (-1, -1), 0.5, outline_color), + ])) + story.append(compliance_table) + story.append(Spacer(1, 10)) + + # 2. iOS Devices Table + story.append(Paragraph("iOS Devices (Top 10)", body_style)) + story.append(Spacer(1, 4)) + ios_compliance = intune_data.get("ios_compliance", []) + if not ios_compliance: + story.append(Paragraph("No iOS device compliance policies were discovered or permission restricted.", ParagraphStyle('ErrTxtIosCompliance', parent=body_style, textColor=colors.HexColor("#DC2626")))) + else: + compliance_table_data = [[ + Paragraph("Display Name", table_cell_header), + Paragraph("Description", table_cell_header), + Paragraph("Created Time", table_cell_header), + Paragraph("Last Modified", table_cell_header), + Paragraph("Version", table_cell_header) + ]] + for policy in ios_compliance[:10]: + compliance_table_data.append([ + Paragraph(policy.get("displayName", "N/A"), table_cell_bold), + Paragraph(policy.get("description", "N/A"), table_cell_style), + Paragraph(policy.get("createdDateTime", "N/A"), table_cell_style), + Paragraph(policy.get("lastModifiedDateTime", "N/A"), table_cell_style), + Paragraph(str(policy.get("version", 0)), table_cell_style) + ]) + compliance_table = Table(compliance_table_data, colWidths=[120, 150, 100, 100, 34]) + compliance_table.setStyle(TableStyle([ + ('BACKGROUND', (0, 0), (-1, 0), primary_color), + ('ALIGN', (0, 0), (-1, -1), 'LEFT'), + ('VALIGN', (0, 0), (-1, -1), 'MIDDLE'), + ('TOPPADDING', (0, 0), (-1, -1), 5), + ('BOTTOMPADDING', (0, 0), (-1, -1), 5), + ('ROWBACKGROUNDS', (0, 1), (-1, -1), [colors.white, colors.HexColor("#F8FAFC")]), + ('GRID', (0, 0), (-1, -1), 0.5, outline_color), + ])) + story.append(compliance_table) + story.append(Spacer(1, 10)) + + # Render Mobile BYOD Configurations (Top 10) + story.append(Paragraph("Mobile BYOD Configurations (Top 10)", body_style)) + story.append(Spacer(1, 4)) + byod_configs = intune_data.get("byod_configs", []) + if not byod_configs: + story.append(Paragraph("No Mobile BYOD configurations were discovered or permission restricted.", ParagraphStyle('ErrTxtByod', parent=body_style, textColor=colors.HexColor("#DC2626")))) + else: + byod_table_data = [[ + Paragraph("Display Name", table_cell_header), + Paragraph("Description", table_cell_header), + Paragraph("Priority", table_cell_header), + Paragraph("Last Modified", table_cell_header), + Paragraph("iOS Restrictions", table_cell_header), + Paragraph("Windows Mobile", table_cell_header), + Paragraph("Android Restrictions", table_cell_header) + ]] + for config in byod_configs[:10]: + byod_table_data.append([ + Paragraph(config.get("displayName", "N/A"), table_cell_bold), + Paragraph(config.get("description", "N/A"), table_cell_style), + Paragraph(str(config.get("priority", 0)), table_cell_style), + Paragraph(config.get("lastModifiedDateTime", "N/A"), table_cell_style), + Paragraph(config.get("iosRestrictions", "N/A"), table_cell_style), + Paragraph(config.get("windowsMobileRestrictions", "N/A"), table_cell_style), + Paragraph(config.get("androidRestrictions", "N/A"), table_cell_style) + ]) + byod_table = Table(byod_table_data, colWidths=[70, 75, 34, 65, 90, 90, 90]) + byod_table.setStyle(TableStyle([ + ('BACKGROUND', (0, 0), (-1, 0), primary_color), + ('ALIGN', (0, 0), (-1, -1), 'LEFT'), + ('VALIGN', (0, 0), (-1, -1), 'MIDDLE'), + ('TOPPADDING', (0, 0), (-1, -1), 4), + ('BOTTOMPADDING', (0, 0), (-1, -1), 4), + ('ROWBACKGROUNDS', (0, 1), (-1, -1), [colors.white, colors.HexColor("#F8FAFC")]), + ('GRID', (0, 0), (-1, -1), 0.5, outline_color), + ])) + story.append(byod_table) + story.append(Spacer(1, 10)) + + # Render Mobile Device Management Policies (Top 10) + story.append(Paragraph("Mobile Device Management Policies (Top 10)", body_style)) + story.append(Spacer(1, 6)) + + mdm_policies = intune_data.get("mdm_policies", []) + if not mdm_policies: + story.append(Paragraph("No Mobile Device Management (MDM) policies were discovered or permission restricted.", ParagraphStyle('ErrTxtMdmPolicies', parent=body_style, textColor=colors.HexColor("#DC2626")))) + else: + mdm_table_data = [[ + Paragraph("Display Name", table_cell_header), + Paragraph("Description", table_cell_header), + Paragraph("Applies To", table_cell_header), + Paragraph("Discovery URL", table_cell_header), + Paragraph("Terms of Use", table_cell_header), + Paragraph("Compliance", table_cell_header) + ]] + for policy in mdm_policies[:10]: + mdm_table_data.append([ + Paragraph(policy.get("displayName", "N/A"), table_cell_bold), + Paragraph(policy.get("description", "N/A"), table_cell_style), + Paragraph(policy.get("appliesTo", "None"), table_cell_style), + Paragraph(policy.get("discoveryUrl", "N/A"), table_cell_style), + Paragraph(policy.get("termsOfUseUrl", "N/A"), table_cell_style), + Paragraph(policy.get("complianceUrl", "N/A"), table_cell_style) + ]) + mdm_table = Table(mdm_table_data, colWidths=[90, 100, 64, 86, 86, 86]) + mdm_table.setStyle(TableStyle([ + ('BACKGROUND', (0, 0), (-1, 0), primary_color), + ('ALIGN', (0, 0), (-1, -1), 'LEFT'), + ('VALIGN', (0, 0), (-1, -1), 'MIDDLE'), + ('TOPPADDING', (0, 0), (-1, -1), 4), + ('BOTTOMPADDING', (0, 0), (-1, -1), 4), + ('ROWBACKGROUNDS', (0, 1), (-1, -1), [colors.white, colors.HexColor("#F8FAFC")]), + ('GRID', (0, 0), (-1, -1), 0.5, outline_color), + ])) + story.append(mdm_table) + story.append(Spacer(1, 10)) + + # Render Detected Apps Table (first 10 items) + story.append(Spacer(1, 10)) + story.append(Paragraph("Detected Apps (Top 10 Discovered)", body_style)) + story.append(Spacer(1, 6)) + + detected_apps = intune_data.get("detected_apps", []) + if not detected_apps: + story.append(Paragraph("No detected apps were discovered or permission restricted.", ParagraphStyle('ErrTxtDetApps', parent=body_style, textColor=colors.HexColor("#DC2626")))) + else: + det_table_data = [[ + Paragraph("App Name", table_cell_header), + Paragraph("Version", table_cell_header), + Paragraph("Publisher", table_cell_header), + Paragraph("Platform", table_cell_header) + ]] + + for app in detected_apps[:10]: + det_table_data.append([ + Paragraph(app.get("displayName", "N/A"), table_cell_bold), + Paragraph(app.get("version", "N/A"), table_cell_style), + Paragraph(app.get("publisher", "N/A"), table_cell_style), + Paragraph(app.get("platform", "unknown"), table_cell_style) + ]) + + det_table = Table(det_table_data, colWidths=[150, 100, 150, 104]) + det_table.setStyle(TableStyle([ + ('BACKGROUND', (0, 0), (-1, 0), primary_color), + ('ALIGN', (0, 0), (-1, -1), 'LEFT'), + ('VALIGN', (0, 0), (-1, -1), 'MIDDLE'), + ('TOPPADDING', (0, 0), (-1, -1), 5), + ('BOTTOMPADDING', (0, 0), (-1, -1), 5), + ('ROWBACKGROUNDS', (0, 1), (-1, -1), [colors.white, colors.HexColor("#F8FAFC")]), + ('GRID', (0, 0), (-1, -1), 0.5, outline_color), + ])) + story.append(det_table) + story.append(Spacer(1, 4)) + story.append(Paragraph("* Showing top 10 detected apps. The full inventory list of up to 10,000 apps is available in the exported CSV report.", body_style)) + + story.append(Spacer(1, 15)) + + # ========================================================================= + # SECTION 4: NETWORK SECURITY + # ========================================================================= + story.append(Paragraph("4. Network Security", h1_style)) + story.append(Paragraph("A summary of Entra Global Secure Access filtering policies, Conditional Access exclusions, and Intune Firewall and Proxy configurations.", body_style)) + story.append(Spacer(1, 10)) + + net_sec = data.get("network_security", {}) + filtering_policies = net_sec.get("filtering_policies", []) + ca_policies = net_sec.get("conditional_access", []) + fw_policies = net_sec.get("firewall_policies", []) + + # 4.1 Filtering Policies (GSA) + story.append(Paragraph("Filtering Policies (Global Secure Access)", h2_style)) + if not filtering_policies: + story.append(Paragraph("No Global Secure Access filtering policies configured or permission restricted.", ParagraphStyle('ErrTxtNetSec', parent=body_style, textColor=colors.HexColor("#DC2626")))) + else: + table_data = [[ + Paragraph("Policy Name", table_cell_header), + Paragraph("Description", table_cell_header), + Paragraph("Version", table_cell_header), + Paragraph("Action", table_cell_header), + Paragraph("Rules", table_cell_header) + ]] + for item in filtering_policies: + table_data.append([ + Paragraph(item.get("name", "N/A"), table_cell_bold), + Paragraph(item.get("description", "N/A"), table_cell_style), + Paragraph(item.get("version", "N/A"), table_cell_style), + Paragraph(item.get("action", "N/A"), table_cell_style), + Paragraph(item.get("rules_count", "0"), table_cell_style) + ]) + t = Table(table_data, colWidths=[120, 180, 70, 70, 64]) + t.setStyle(TableStyle([ + ('BACKGROUND', (0, 0), (-1, 0), primary_color), + ('ALIGN', (0, 0), (-1, -1), 'LEFT'), + ('VALIGN', (0, 0), (-1, -1), 'MIDDLE'), + ('TOPPADDING', (0, 0), (-1, -1), 4), + ('BOTTOMPADDING', (0, 0), (-1, -1), 4), + ('ROWBACKGROUNDS', (0, 1), (-1, -1), [colors.white, colors.HexColor("#F8FAFC")]), + ('GRID', (0, 0), (-1, -1), 0.5, outline_color), + ])) + story.append(t) + + story.append(Spacer(1, 12)) + + # 4.2 Conditional Access policies + story.append(Paragraph("Conditional Access (Network Exclusions & Scope)", h2_style)) + if not ca_policies: + story.append(Paragraph("No Conditional Access policies configured or permission restricted.", ParagraphStyle('ErrTxtNetSecCA', parent=body_style, textColor=colors.HexColor("#DC2626")))) + else: + table_data = [[ + Paragraph("Policy Name", table_cell_header), + Paragraph("State", table_cell_header), + Paragraph("Target Users", table_cell_header), + Paragraph("Target Apps", table_cell_header), + Paragraph("Grant Controls", table_cell_header) + ]] + for item in ca_policies: + table_data.append([ + Paragraph(item.get("name", "N/A"), table_cell_bold), + Paragraph(item.get("state", "N/A"), table_cell_style), + Paragraph(item.get("target_users", "N/A"), table_cell_style), + Paragraph(item.get("target_apps", "N/A"), table_cell_style), + Paragraph(item.get("controls", "N/A"), table_cell_style) + ]) + t = Table(table_data, colWidths=[130, 60, 100, 100, 114]) + t.setStyle(TableStyle([ + ('BACKGROUND', (0, 0), (-1, 0), primary_color), + ('ALIGN', (0, 0), (-1, -1), 'LEFT'), + ('VALIGN', (0, 0), (-1, -1), 'MIDDLE'), + ('TOPPADDING', (0, 0), (-1, -1), 4), + ('BOTTOMPADDING', (0, 0), (-1, -1), 4), + ('ROWBACKGROUNDS', (0, 1), (-1, -1), [colors.white, colors.HexColor("#F8FAFC")]), + ('GRID', (0, 0), (-1, -1), 0.5, outline_color), + ])) + story.append(t) + + story.append(Spacer(1, 12)) + + # 4.3 Firewall/Proxy Policies + story.append(Paragraph("Firewall and Proxy Configurations", h2_style)) + if not fw_policies: + story.append(Paragraph("No Firewall or Proxy configurations discovered in Intune policies.", ParagraphStyle('ErrTxtNetSecFW', parent=body_style, textColor=colors.HexColor("#DC2626")))) + else: + table_data = [[ + Paragraph("Configuration Name", table_cell_header), + Paragraph("Policy Type", table_cell_header), + Paragraph("Firewall Status", table_cell_header), + Paragraph("Proxy Status", table_cell_header) + ]] + for item in fw_policies: + table_data.append([ + Paragraph(item.get("name", "N/A"), table_cell_bold), + Paragraph(item.get("policy_type", "N/A"), table_cell_style), + Paragraph(item.get("firewall_status", "N/A"), table_cell_style), + Paragraph(item.get("proxy_status", "N/A"), table_cell_style) + ]) + t = Table(table_data, colWidths=[150, 150, 100, 104]) + t.setStyle(TableStyle([ + ('BACKGROUND', (0, 0), (-1, 0), primary_color), + ('ALIGN', (0, 0), (-1, -1), 'LEFT'), + ('VALIGN', (0, 0), (-1, -1), 'MIDDLE'), + ('TOPPADDING', (0, 0), (-1, -1), 4), + ('BOTTOMPADDING', (0, 0), (-1, -1), 4), + ('ROWBACKGROUNDS', (0, 1), (-1, -1), [colors.white, colors.HexColor("#F8FAFC")]), + ('GRID', (0, 0), (-1, -1), 0.5, outline_color), + ])) + story.append(t) + + story.append(Spacer(1, 15)) + story.append(PageBreak()) + + # ========================================================================= + # SECTION 5: DATA SECURITY & GOVERNANCE + # ========================================================================= + story.append(Paragraph("5. Data Security, Governance & Compliance", h1_style)) + story.append(Paragraph("A summary of classification sensitivity labels and data retention lifecycle policies configured within Microsoft Purview to protect corporate properties.", body_style)) + + # 4.1 Sensitivity Labels + story.append(Paragraph("Microsoft Purview Sensitivity Labels", h2_style)) + labels = data.get("security_labels", []) + if not labels: + story.append(Paragraph("No Purview Sensitivity Labels configured or permission restricted.", ParagraphStyle('ErrTxt', parent=body_style, textColor=colors.HexColor("#DC2626")))) + else: + labels_table_data = [[ + Paragraph("Sensitivity Label", table_cell_header), + Paragraph("Description", table_cell_header), + Paragraph("Shield", table_cell_header), + Paragraph("Mode", table_cell_header), + Paragraph("Priority", table_cell_header), + Paragraph("Status", table_cell_header) + ]] + + # Flatten parent labels and sublabels for the PDF table + flattened_labels = [] + for parent in labels: + flattened_labels.append({ + "name": parent.get("name", "N/A"), + "description": parent.get("description", "") or parent.get("toolTip", "") or "N/A", + "hasProtection": parent.get("hasProtection", False), + "applicationMode": parent.get("applicationMode", "N/A") or "N/A", + "priority": parent.get("priority", 0), + "isEnabled": parent.get("isEnabled", True), + "is_sub": False + }) + for sub in parent.get("sublabels", []): + flattened_labels.append({ + "name": f" L_ {sub.get('name', 'N/A')}", + "description": sub.get("description", "") or sub.get("toolTip", "") or "N/A", + "hasProtection": sub.get("hasProtection", False), + "applicationMode": sub.get("applicationMode", "N/A") or "N/A", + "priority": sub.get("priority", 0), + "isEnabled": sub.get("isEnabled", True), + "is_sub": True + }) + + for item in flattened_labels: + bg_bold_s = table_cell_bold if not item["is_sub"] else table_cell_style + protection_str = "Yes" if item["hasProtection"] else "No" + status_str = "Enabled" if item["isEnabled"] else "Disabled" + + labels_table_data.append([ + Paragraph(item["name"], bg_bold_s), + Paragraph(item["description"], table_cell_style), + Paragraph(protection_str, table_cell_style), + Paragraph(str(item["applicationMode"]).capitalize(), table_cell_style), + Paragraph(str(item["priority"]), table_cell_style), + Paragraph(status_str, table_cell_style) + ]) + + labels_table = Table(labels_table_data, colWidths=[120, 160, 50, 60, 50, 60]) + labels_table.setStyle(TableStyle([ + ('BACKGROUND', (0, 0), (-1, 0), primary_color), + ('ALIGN', (0, 0), (-1, -1), 'LEFT'), + ('VALIGN', (0, 0), (-1, -1), 'MIDDLE'), + ('TOPPADDING', (0, 0), (-1, -1), 5), + ('BOTTOMPADDING', (0, 0), (-1, -1), 5), + ('ROWBACKGROUNDS', (0, 1), (-1, -1), [colors.white, colors.HexColor("#F8FAFC")]), + ('GRID', (0, 0), (-1, -1), 0.5, outline_color), + ])) + story.append(labels_table) + + story.append(PageBreak()) + + # 4.2 Retention Policies + story.append(Paragraph("Microsoft Purview Retention Compliance Policies", h2_style)) + policies = data.get("retention_policies", []) + if not policies: + story.append(Paragraph("No Purview Retention compliance policies discovered or permission restricted.", ParagraphStyle('ErrTxt', parent=body_style, textColor=colors.HexColor("#DC2626")))) + else: + ret_table_data = [[ + Paragraph("Policy Name", table_cell_header), + Paragraph("Workloads Involved", table_cell_header), + Paragraph("Retention Duration Basis", table_cell_header), + Paragraph("Distribution Status", table_cell_header), + Paragraph("Status", table_cell_header) + ]] + + policies_list = policies if isinstance(policies, list) else [policies] + for policy in policies_list: + duration_val = str(policy.get("Duration", "N/A")) + duration_str = duration_val + if duration_val.lower() == "unlimited": + duration_str = "Keep Forever" + elif duration_val.isdigit(): + days = int(duration_val) + if days >= 365: + years = days / 365.0 + duration_str = f"{int(years)} Years ({days} days)" if years.is_integer() else f"{years:.1f} Years ({days} days)" + else: + duration_str = f"{days} days" + + trigger_val = policy.get("RetentionTrigger", "N/A") + if trigger_val and trigger_val != "N/A": + trigger_map = {"DateCreated": "created date", "DateModified": "last modified date", "DateLabeled": "labeled date"} + duration_str += f"
(from {trigger_map.get(trigger_val, trigger_val)})" + + enabled_val = policy.get("Enabled", True) + is_enabled = enabled_val.lower() == "true" if isinstance(enabled_val, str) else bool(enabled_val) + status_str = "Enabled" if is_enabled else "Disabled" + + ret_table_data.append([ + Paragraph(policy.get("Name", "N/A"), table_cell_bold), + Paragraph(policy.get("Workload", "N/A"), table_cell_style), + Paragraph(duration_str, table_cell_style), + Paragraph(policy.get("DistributionStatus", "Success"), table_cell_style), + Paragraph(status_str, table_cell_style) + ]) + + ret_table = Table(ret_table_data, colWidths=[130, 110, 110, 90, 60]) + ret_table.setStyle(TableStyle([ + ('BACKGROUND', (0, 0), (-1, 0), primary_color), + ('ALIGN', (0, 0), (-1, -1), 'LEFT'), + ('VALIGN', (0, 0), (-1, -1), 'MIDDLE'), + ('TOPPADDING', (0, 0), (-1, -1), 5), + ('BOTTOMPADDING', (0, 0), (-1, -1), 5), + ('ROWBACKGROUNDS', (0, 1), (-1, -1), [colors.white, colors.HexColor("#F8FAFC")]), + ('GRID', (0, 0), (-1, -1), 0.5, outline_color), + ])) + story.append(ret_table) + story.append(Spacer(1, 15)) + + # 4.3 Data Loss Prevention Policies + story.append(Paragraph("4.3 Data Loss Prevention Policies", h2_style)) + story.append(Paragraph("This section outlines DLP policies configured in Microsoft Purview to prevent accidental data leaks.", body_style)) + story.append(Spacer(1, 8)) + + dlp_policies = data.get("dlp_policies", []) + if not dlp_policies: + story.append(Paragraph("No Purview Data Loss Prevention policies discovered.", ParagraphStyle('ErrTxt', parent=body_style, textColor=colors.HexColor("#DC2626")))) + else: + dlp_table_data = [[ + Paragraph("Policy Name", table_cell_header), + Paragraph("Mode", table_cell_header), + Paragraph("Workload", table_cell_header), + Paragraph("State", table_cell_header), + Paragraph("Actions", table_cell_header), + Paragraph("Created By", table_cell_header) + ]] + for dlp in dlp_policies: + en_val = str(dlp.get("Enabled", "")).lower() + state_str = "Enabled" if en_val in ("true", "1", "yes") else "Disabled" + + dlp_table_data.append([ + Paragraph(dlp.get("Name", "-"), table_cell_bold), + Paragraph(dlp.get("Mode", "-"), table_cell_style), + Paragraph(dlp.get("Workload", "-"), table_cell_style), + Paragraph(state_str, table_cell_style), + Paragraph(dlp.get("Actions", "-"), table_cell_style), + Paragraph(dlp.get("CreatedBy", "-"), table_cell_style) + ]) + dlp_table = Table(dlp_table_data, colWidths=[110, 50, 110, 60, 90, 80]) + dlp_table.setStyle(TableStyle([ + ('BACKGROUND', (0, 0), (-1, 0), primary_color), + ('ALIGN', (0, 0), (-1, -1), 'LEFT'), + ('VALIGN', (0, 0), (-1, -1), 'MIDDLE'), + ('TOPPADDING', (0, 0), (-1, -1), 5), + ('BOTTOMPADDING', (0, 0), (-1, -1), 5), + ('ROWBACKGROUNDS', (0, 1), (-1, -1), [colors.white, colors.HexColor("#F8FAFC")]), + ('GRID', (0, 0), (-1, -1), 0.5, outline_color), + ])) + story.append(dlp_table) + story.append(Spacer(1, 15)) + + # 4.4 Sensitive Information Types + story.append(Paragraph("4.4 Sensitive Information Types", h2_style)) + story.append(Paragraph("This section outlines custom and built-in sensitive information types active in the environment.", body_style)) + story.append(Spacer(1, 8)) + + sit_types = data.get("sensitive_info_types", []) + if not sit_types: + story.append(Paragraph("No Sensitive Information Types discovered.", ParagraphStyle('ErrTxt', parent=body_style, textColor=colors.HexColor("#DC2626")))) + else: + sit_table_data = [[ + Paragraph("Name", table_cell_header), + Paragraph("Type", table_cell_header), + Paragraph("Confidence", table_cell_header) + ]] + for sit in sit_types: + sit_table_data.append([ + Paragraph(sit.get("Name", "-"), table_cell_bold), + Paragraph(sit.get("Type", "-"), table_cell_style), + Paragraph(str(sit.get("RecommendedConfidence", "-")), table_cell_style) + ]) + sit_table = Table(sit_table_data, colWidths=[280, 100, 120]) + sit_table.setStyle(TableStyle([ + ('BACKGROUND', (0, 0), (-1, 0), primary_color), + ('ALIGN', (0, 0), (-1, -1), 'LEFT'), + ('VALIGN', (0, 0), (-1, -1), 'MIDDLE'), + ('TOPPADDING', (0, 0), (-1, -1), 5), + ('BOTTOMPADDING', (0, 0), (-1, -1), 5), + ('ROWBACKGROUNDS', (0, 1), (-1, -1), [colors.white, colors.HexColor("#F8FAFC")]), + ('GRID', (0, 0), (-1, -1), 0.5, outline_color), + ])) + story.append(sit_table) + story.append(Spacer(1, 15)) + + # ========================================================================= + + # 4.5 Mail Security + story.append(Paragraph("4.5 Mail Security (Exchange)", h2_style)) + story.append(Paragraph("This section displays configured email filtering and threat protection policies.", body_style)) + story.append(Spacer(1, 8)) + + mail_sec = data.get("mail_security", {}) + if not mail_sec or (not mail_sec.get("defender", {}).get("skus") and not mail_sec.get("eop", {}).get("skus")): + story.append(Paragraph("No mail security SKUs detected.", ParagraphStyle('ErrTxt', parent=body_style, textColor=colors.HexColor("#DC2626")))) + else: + ms_table_data = [[ + Paragraph("Mail Security Configuration", table_cell_header), + Paragraph("Detected SKUs", table_cell_header), + Paragraph("Affected Users", table_cell_header) + ]] + + defender_data = mail_sec.get("defender", {}) + eop_data = mail_sec.get("eop", {}) + + if defender_data.get("skus"): + ms_table_data.append([ + Paragraph("Microsoft Defender for Office 365", table_cell_bold), + Paragraph(", ".join(defender_data.get("skus", [])), table_cell_style), + Paragraph(f"{defender_data.get('users', 0):,} Users", table_cell_style) + ]) + + if eop_data.get("skus"): + ms_table_data.append([ + Paragraph("Exchange Online Protection (Baseline)", table_cell_bold), + Paragraph(", ".join(eop_data.get("skus", [])), table_cell_style), + Paragraph(f"{eop_data.get('users', 0):,} Users", table_cell_style) + ]) + + ms_table = Table(ms_table_data, colWidths=[200, 200, 100]) + ms_table.setStyle(TableStyle([ + ('BACKGROUND', (0, 0), (-1, 0), primary_color), + ('ALIGN', (0, 0), (-1, -1), 'LEFT'), + ('VALIGN', (0, 0), (-1, -1), 'MIDDLE'), + ('TOPPADDING', (0, 0), (-1, -1), 5), + ('BOTTOMPADDING', (0, 0), (-1, -1), 5), + ('ROWBACKGROUNDS', (0, 1), (-1, -1), [colors.white, colors.HexColor("#F8FAFC")]), + ('GRID', (0, 0), (-1, -1), 0.5, outline_color), + ])) + story.append(ms_table) + story.append(Spacer(1, 15)) + + # 4.6 Exchange Transport Rules + story.append(Paragraph("4.6 Exchange Transport Rules", h2_style)) + story.append(Paragraph("This section displays mail flow rules configured in Exchange Online.", body_style)) + story.append(Spacer(1, 8)) + + transport_rules = data.get("transport_rules", []) + if not transport_rules: + story.append(Paragraph("No Exchange Transport Rules discovered.", ParagraphStyle('ErrTxt', parent=body_style, textColor=colors.HexColor("#DC2626")))) + else: + rules_table_data = [[ + Paragraph("Rule Name", table_cell_header), + Paragraph("State", table_cell_header), + Paragraph("Priority", table_cell_header), + Paragraph("Mode", table_cell_header), + Paragraph("Rule Logic", table_cell_header) + ]] + + display_rules = transport_rules + for rule in display_rules: + desc_text = rule.get("Description") or "N/A" + safe_desc = html.escape(str(desc_text)).replace('\n', '
') + safe_name = html.escape(str(rule.get("Name", "-"))) + rules_table_data.append([ + Paragraph(safe_name, table_cell_bold), + Paragraph(html.escape(str(rule.get("State", "-"))), table_cell_style), + Paragraph(html.escape(str(rule.get("Priority", "-"))), table_cell_style), + Paragraph(html.escape(str(rule.get("Mode", "-"))), table_cell_style), + Paragraph(safe_desc, small_table_cell_style) + ]) + + rules_table = Table(rules_table_data, colWidths=[120, 50, 40, 60, 234]) + rules_table.setStyle(TableStyle([ + ('BACKGROUND', (0, 0), (-1, 0), primary_color), + ('ALIGN', (0, 0), (-1, -1), 'LEFT'), + ('VALIGN', (0, 0), (-1, -1), 'MIDDLE'), + ('TOPPADDING', (0, 0), (-1, -1), 5), + ('BOTTOMPADDING', (0, 0), (-1, -1), 5), + ('ROWBACKGROUNDS', (0, 1), (-1, -1), [colors.white, colors.HexColor("#F8FAFC")]), + ('GRID', (0, 0), (-1, -1), 0.5, outline_color), + ])) + story.append(rules_table) + story.append(Spacer(1, 15)) + + # 4.7 SSO Service Principals + story.append(Paragraph("4.7 Enterprise SAML SSO Apps", h2_style)) + story.append(Paragraph("This section displays Enterprise Applications configured for SAML Single Sign-On.", body_style)) + story.append(Spacer(1, 8)) + + sso_apps = data.get("service_principals_sso", []) + if not sso_apps: + story.append(Paragraph("No SAML SSO applications discovered.", ParagraphStyle('ErrTxt', parent=body_style, textColor=colors.HexColor("#DC2626")))) + else: + mode_counts = Counter() + for app in sso_apps: + mode = app.get("preferredSingleSignOnMode", "").strip() or "None" + mode_counts[mode] += 1 + + sso_table_data = [[ + Paragraph("SSO Mode", table_cell_header), + Paragraph("Number of Applications", table_cell_header) + ]] + + for mode, count in sorted(mode_counts.items(), key=lambda x: x[1], reverse=True): + sso_table_data.append([ + Paragraph(mode, table_cell_bold), + Paragraph(f"{count:,} Apps", table_cell_style) + ]) + + sso_table = Table(sso_table_data, colWidths=[300, 200]) + sso_table.setStyle(TableStyle([ + ('BACKGROUND', (0, 0), (-1, 0), primary_color), + ('ALIGN', (0, 0), (-1, -1), 'LEFT'), + ('VALIGN', (0, 0), (-1, -1), 'MIDDLE'), + ('TOPPADDING', (0, 0), (-1, -1), 5), + ('BOTTOMPADDING', (0, 0), (-1, -1), 5), + ('ROWBACKGROUNDS', (0, 1), (-1, -1), [colors.white, colors.HexColor("#F8FAFC")]), + ('GRID', (0, 0), (-1, -1), 0.5, outline_color), + ])) + story.append(sso_table) + story.append(Spacer(1, 15)) + + # 4.8 Conditional Access Policies + story.append(Paragraph("4.8 Conditional Access Policies", h2_style)) + story.append(Paragraph("This section displays Azure AD Auth Policies governing conditional access.", body_style)) + story.append(Spacer(1, 8)) + + ca_policies = data.get("conditional_access", []) + if not ca_policies: + story.append(Paragraph("No conditional access policies discovered.", ParagraphStyle('ErrTxt', parent=body_style, textColor=colors.HexColor("#DC2626")))) + else: + ca_table_data = [[ + Paragraph("Policy Name", table_cell_header), + Paragraph("State", table_cell_header), + Paragraph("Controls", table_cell_header) + ]] + for cap in ca_policies: + ca_table_data.append([ + Paragraph(cap.get("name", "-"), table_cell_bold), + Paragraph(cap.get("state", "-"), table_cell_style), + Paragraph(cap.get("controls", "-"), table_cell_style) + ]) + ca_table = Table(ca_table_data, colWidths=[250, 100, 150]) + ca_table.setStyle(TableStyle([ + ('BACKGROUND', (0, 0), (-1, 0), primary_color), + ('ALIGN', (0, 0), (-1, -1), 'LEFT'), + ('VALIGN', (0, 0), (-1, -1), 'MIDDLE'), + ('TOPPADDING', (0, 0), (-1, -1), 5), + ('BOTTOMPADDING', (0, 0), (-1, -1), 5), + ('ROWBACKGROUNDS', (0, 1), (-1, -1), [colors.white, colors.HexColor("#F8FAFC")]), + ('GRID', (0, 0), (-1, -1), 0.5, outline_color), + ])) + story.append(ca_table) + story.append(Spacer(1, 15)) + story.append(PageBreak()) + + # 4.9 eDiscovery Cases + story.append(Paragraph("4.9 Microsoft Purview eDiscovery Cases", h2_style)) + story.append(Paragraph("This section lists the active and closed eDiscovery cases across the tenant, providing visibility into compliance and legal discovery workloads.", body_style)) + story.append(Spacer(1, 8)) + + ediscovery_cases = data.get("ediscovery_cases", []) + if not ediscovery_cases: + story.append(Paragraph("No eDiscovery cases were discovered or Delegated Authentication was not used.", ParagraphStyle('ErrTxt', parent=body_style, textColor=colors.HexColor("#DC2626")))) + else: + edisc_table_data = [[ + Paragraph("Display Name", table_cell_header), + Paragraph("Status", table_cell_header), + Paragraph("Created Date", table_cell_header), + Paragraph("Closed By", table_cell_header) + ]] + for case in ediscovery_cases[:10]: + created_date = str(case.get("createdDateTime", "-")).split("T")[0] + edisc_table_data.append([ + Paragraph(case.get("displayName", "-"), table_cell_bold), + Paragraph(case.get("status", "-"), table_cell_style), + Paragraph(created_date, table_cell_style), + Paragraph(case.get("closedBy", "-"), table_cell_style) + ]) + + edisc_table = Table(edisc_table_data, colWidths=[200, 80, 100, 120]) + edisc_table.setStyle(TableStyle([ + ('BACKGROUND', (0, 0), (-1, 0), primary_color), + ('ALIGN', (0, 0), (-1, -1), 'LEFT'), + ('VALIGN', (0, 0), (-1, -1), 'MIDDLE'), + ('TOPPADDING', (0, 0), (-1, -1), 5), + ('BOTTOMPADDING', (0, 0), (-1, -1), 5), + ('ROWBACKGROUNDS', (0, 1), (-1, -1), [colors.white, colors.HexColor("#F8FAFC")]), + ('GRID', (0, 0), (-1, -1), 0.5, outline_color), + ])) + story.append(edisc_table) + if len(ediscovery_cases) > 10: + story.append(Paragraph(f"...and {len(ediscovery_cases) - 10} more. See generated CSV reports for full details.", ParagraphStyle('Ital', parent=body_style, fontName='Helvetica-Oblique', textColor=secondary_color))) + story.append(Spacer(1, 15)) + story.append(PageBreak()) + + # SECTION 6: POWER AUTOMATE + # ========================================================================= + story.append(Paragraph("6. Power Platform & Automate Flows Analytics", h1_style)) + story.append(Paragraph("An analysis of low-code cloud and desktop workflows configured inside the tenant environments, identifying complex workflows and premium connectors.", body_style)) + story.append(Spacer(1, 8)) + + pa = data.get("power_automate", {}) + if not pa: + story.append(Paragraph("No Power Platform or Power Automate telemetry scan data was available.", ParagraphStyle('ErrTxt', parent=body_style, textColor=colors.HexColor("#DC2626")))) + else: + counts = pa.get("counts", {}) + total_flows = counts.get("Cloud Flows", 0) + counts.get("Desktop Flows", 0) + premium_conns = pa.get("premium_connectors", []) + custom_conns = pa.get("custom_connectors", []) + + prem_str = ", ".join(premium_conns) if premium_conns else "0" + cust_str = ", ".join(custom_conns) if custom_conns else "0" + + pa_table_data = [[ + Paragraph("Power Platform Telemetry Property", table_cell_header), + Paragraph("Scanned Value", table_cell_header) + ]] + + pa_rows = [ + ("Total Environments Scanned", str(pa.get("total_environments", 0))), + ("Total Flows (Active + Inactive)", f"{total_flows:,} Flows"), + ("Active Cloud Flows Count", f"{pa.get('active_counts', {}).get('Cloud Flows', 0):,} Cloud Flows"), + ("Active Desktop Flows Count", f"{pa.get('active_counts', {}).get('Desktop Flows', 0):,} Desktop Flows"), + ("Premium Connectors In Use", prem_str), + ("Custom Connectors In Use", cust_str), + ("Complex Business-Logic Flows Identified", f"{len(pa.get('complex_logic_flows', [])):,} Flows"), + ] + + for label, val in pa_rows: + pa_table_data.append([ + Paragraph(label, table_cell_bold), + Paragraph(val, table_cell_style) + ]) + + pa_table = Table(pa_table_data, colWidths=[220, 280]) + pa_table.setStyle(TableStyle([ + ('BACKGROUND', (0, 0), (-1, 0), primary_color), + ('ALIGN', (0, 0), (-1, -1), 'LEFT'), + ('VALIGN', (0, 0), (-1, -1), 'MIDDLE'), + ('TOPPADDING', (0, 0), (-1, -1), 5), + ('BOTTOMPADDING', (0, 0), (-1, -1), 5), + ('ROWBACKGROUNDS', (0, 1), (-1, -1), [colors.white, colors.HexColor("#F8FAFC")]), + ('GRID', (0, 0), (-1, -1), 0.5, outline_color), + ])) + story.append(pa_table) + + # Power Automate Breakdown Chart - Generated on the fly + if counts: + try: + pa_chart_bytes = generate_pa_chart_bytes(pa) + if pa_chart_bytes: + story.append(Spacer(1, 15)) + story.append(Paragraph("Power Automate Flows Breakdown Chart", h2_style)) + pa_chart = Image(pa_chart_bytes, width=450, height=210) + story.append(pa_chart) + except Exception as chart_ex: + print(f"Failed to generate Power Automate chart for PDF: {chart_ex}") + + # 4. Build Document + doc.build(story, canvasmaker=NumberedCanvas) diff --git a/telemetry/power_automate.py b/telemetry/power_automate.py new file mode 100644 index 00000000..884491db --- /dev/null +++ b/telemetry/power_automate.py @@ -0,0 +1,768 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Modular Power Automate telemetry scanner, analysis pipelines, and visual interfaces.""" + +import os +import csv +import shutil +import time +import json +import logging +import threading +import requests +import pandas as pd +from datetime import datetime +from concurrent.futures import ThreadPoolExecutor, as_completed +from tkinter import filedialog, messagebox +from typing import Any, Dict, List +import customtkinter as ctk +import webbrowser + +# Safely import matplotlib to embed plots in Tkinter +try: + from matplotlib.figure import Figure + from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg + MATPLOTLIB_AVAILABLE = True +except ImportError: + MATPLOTLIB_AVAILABLE = False + +# Import shared styles +from telemetry.styles import * + +# Bind to the async logger initialized in m365_telemetry.py +usage_logger = logging.getLogger("M365TelemetryAsyncLogger") + + +def run_power_automate_pipeline(client_id: str, client_secret: str, tenant_id: str) -> dict: + """Helper entrypoint matching pipeline naming patterns.""" + scanner = PowerAutomateScanner(tenant_id, client_id, client_secret) + return scanner.scan_flows() + + +class PowerAutomateScanner: + def __init__(self, tenant_id, client_id, client_secret): + self.tenant_id = tenant_id + self.client_id = client_id + self.client_secret = client_secret + + self._setup_logger() + self.access_token = None + + def _setup_logger(self): + """Configures logging to propagate to the central M365TelemetryAsyncLogger.""" + self.logger = logging.getLogger("M365TelemetryAsyncLogger.PowerAutomateScanner") + + def _get_access_token(self, scope): + """Fetches OAuth 2.0 token for a given scope.""" + self.logger.info(f"Step Start: Fetching access token for scope {scope} from Microsoft Identity Platform.") + url = f"https://login.microsoftonline.com/{self.tenant_id}/oauth2/v2.0/token" + headers = {'Content-Type': 'application/x-www-form-urlencoded'} + payload = { + 'grant_type': 'client_credentials', + 'client_id': self.client_id, + 'client_secret': self.client_secret, + 'scope': scope + } + + try: + response = requests.post(url, headers=headers, data=payload) + response.raise_for_status() + token = response.json().get('access_token') + self.logger.info(f"Step End: Successfully retrieved access token for {scope}.") + return token + except Exception as e: + self.logger.error(f"Step Error: Failed to fetch access token for {scope}. Error: {str(e)}") + return None + + def fetch_all_pages(self, url, headers, context_name="API"): + """Helper to cleanly handle Power Platform API pagination & Throttling limits.""" + results = [] + while url: + res = requests.get(url, headers=headers) + + if res.status_code == 429: + retry_after = int(res.headers.get("Retry-After", 2)) + self.logger.warning(f"[!] Rate limited on {context_name}. Waiting {retry_after} seconds...") + time.sleep(retry_after) + continue + + if res.status_code == 200: + try: + data = res.json() + except Exception as e: + self.logger.error(f"[X] JSON Decode Error on {context_name} | HTTP {res.status_code}: {res.text[:100]}") + raise e + results.extend(data.get("value", [])) + url = data.get("nextLink") or data.get("@odata.nextLink") + else: + self.logger.error(f"[X] {context_name} Request Failed | HTTP {res.status_code}: {res.text}") + res.raise_for_status() + + return results + + def fetch_single_resource(self, url, headers, context_name="API"): + """Helper to cleanly fetch a single resource handling Throttling limits for loop logic.""" + while True: + res = requests.get(url, headers=headers) + if res.status_code == 429: + retry_after = int(res.headers.get("Retry-After", 2)) + self.logger.warning(f"[!] Rate limited on {context_name}. Waiting {retry_after} seconds...") + time.sleep(retry_after) + continue + if res.status_code == 200: + try: + return res.json() + except Exception as e: + self.logger.error(f"[X] JSON Decode Error on {context_name} | HTTP {res.status_code}: {res.text[:100]}") + raise e + self.logger.error(f"[X] {context_name} Request Failed | HTTP {res.status_code}: {res.text}") + res.raise_for_status() + + def scan_flows(self): + """Scans Power Automate flows across all environments in the tenant.""" + self.logger.info("Main Process Start: Initiating Power Automate Flow Scan.") + + try: + bap_token = self._get_access_token("https://api.bap.microsoft.com/.default") + flow_token = self._get_access_token("https://service.flow.microsoft.com/.default") + except Exception as e: + self.logger.error(f"Auth Error: {e}") + raise e + + if not bap_token or not flow_token: + self.logger.error("Main Process Failure: Aborting scan due to missing access tokens.") + raise Exception("Authentication failed: Missing access tokens.") + + bap_headers = {"Authorization": f"Bearer {bap_token}", "Accept": "application/json"} + flow_headers = {"Authorization": f"Bearer {flow_token}", "Accept": "application/json"} + + self.logger.info("Step Start: Fetching all environments in the tenant.") + env_api_url = "https://api.bap.microsoft.com/providers/Microsoft.BusinessAppPlatform/scopes/admin/environments?api-version=2023-06-01" + environments = self.fetch_all_pages(env_api_url, bap_headers, context_name="Environment Discovery") + + if not environments: + self.logger.error("[X] No environments found or failed to fetch.") + return None + + self.logger.info(f"[+] Successfully retrieved {len(environments)} environments.") + + counts = {"Cloud Flows": 0, "Desktop Flows": 0} + active_counts = {"Cloud Flows": 0, "Desktop Flows": 0} + tier_counts = {"Personal Productivity": 0, "Enterprise/Departmental": 0} + active_tier_counts = {"Personal Productivity": 0, "Enterprise/Departmental": 0} + premium_connectors_found = set() + custom_connectors_found = set() + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + reports_dir = os.path.join(script_dir, "reports", f"{self.tenant_id}_{self.client_id}") + os.makedirs(reports_dir, exist_ok=True) + complex_logic_flows_path = os.path.join(reports_dir, "power_automate_complex_flows.csv") + + # Initialize CSV and header + with open(complex_logic_flows_path, 'w', encoding='utf-8', newline='') as f: + writer = csv.writer(f) + writer.writerow(["Environment", "Name", "Type", "Tier", "Active", "Reason"]) + + complex_flows_lock = threading.Lock() + PREMIUM_KEYWORDS = ['shared_sql', 'shared_httpaction', 'shared_salesforce', 'shared_oracle', 'shared_sap'] + + for env in environments: + env_name = env.get("name") + env_props = env.get("properties", {}) + env_display = env_props.get("displayName", env_name) + is_default = env_props.get("isDefault", False) + + self.logger.info(f"[*] Scanning Environment: {env_display}") + + # ========================================== + # 1. FETCH CLOUD FLOWS + # ========================================== + flows_url = f"https://api.flow.microsoft.com/providers/Microsoft.ProcessSimple/scopes/admin/environments/{env_name}/v2/flows?api-version=2016-11-01" + cloud_flows = self.fetch_all_pages(flows_url, flow_headers, context_name="Cloud Flows Admin API (V2)") + + counts["Cloud Flows"] += len(cloud_flows) + active_cloud_flows = [f for f in cloud_flows if f.get("properties", {}).get("state") == "Started"] + active_counts["Cloud Flows"] += len(active_cloud_flows) + + self.logger.info(f" -> Cloud Flows: {len(cloud_flows)} total found, {len(active_cloud_flows)} are currently active.") + + # Fetch details in parallel using ThreadPoolExecutor + + def fetch_and_process_flow_detail(flow_summary): + flow_id = flow_summary.get("name") + state = flow_summary.get("properties", {}).get("state") + is_active = (state == "Started") + + detail_url = f"https://api.flow.microsoft.com/providers/Microsoft.ProcessSimple/scopes/admin/environments/{env_name}/flows/{flow_id}?api-version=2016-11-01" + try: + flow_detail = self.fetch_single_resource(detail_url, flow_headers, context_name=f"Get Flow Details ({flow_id})") + if not flow_detail: + return None + return (flow_summary, flow_detail, is_active) + except Exception as ex: + self.logger.warning(f"Failed to fetch flow details for flow {flow_id}: {ex}") + return None + + with ThreadPoolExecutor(max_workers=15) as executor: + futures = {executor.submit(fetch_and_process_flow_detail, f): f for f in cloud_flows} + for future in as_completed(futures): + res = future.result() + if not res: + continue + + flow_summary, flow_detail, is_active = res + props = flow_detail.get("properties", {}) + name = props.get("displayName", "Unnamed Flow") + + is_managed = "workflowEntityId" in props or not is_default + if is_managed: + tier_counts["Enterprise/Departmental"] += 1 + tier = "Enterprise" + if is_active: + active_tier_counts["Enterprise/Departmental"] += 1 + else: + tier_counts["Personal Productivity"] += 1 + tier = "Personal" + if is_active: + active_tier_counts["Personal Productivity"] += 1 + + conn_refs = props.get("connectionReferences", {}) + for conn_key, conn_val in conn_refs.items(): + api_obj = conn_val.get("api", {}) + api_id = api_obj.get("id", "") + conn_name = api_id.split("/")[-1] if "/" in api_id else api_id + + if api_obj.get("tier") == "Premium" or any(kw in conn_name.lower() for kw in PREMIUM_KEYWORDS): + premium_connectors_found.add(conn_name) + self.logger.info(f" [!] Premium connector found: {conn_name} in flow {name}") + if "custom" in api_id.lower() or api_obj.get("type") == "Microsoft.PowerApps/apis/custom": + custom_connectors_found.add(conn_name) + self.logger.info(f" [!] Custom connector found: {conn_name} in flow {name}") + + actions_str = json.dumps(props) + has_nested_loops = actions_str.count('"type": "Foreach"') > 0 or actions_str.count('"type": "Until"') > 0 + has_multi_approvals = "shared_approvals" in actions_str or "Approval" in actions_str + has_advanced_expressions = "@" in actions_str and any(exp in actions_str for exp in ["concat(", "split(", "base64("]) + + if has_nested_loops or has_multi_approvals or has_advanced_expressions: + self.logger.info(f" [!] Complex logic detected in Cloud Flow: {name}") + reasons = [] + if has_nested_loops: reasons.append("Nested Loops") + if has_multi_approvals: reasons.append("Multi Approvals") + if has_advanced_expressions: reasons.append("Advanced Expressions") + + flow_dict = { + "Environment": env_display, "Name": name, "Type": "Cloud Flow", "Tier": tier, + "Active": "Yes" if is_active else "No", + "Reason": ", ".join(reasons) + } + with complex_flows_lock: + with open(complex_logic_flows_path, 'a', encoding='utf-8', newline='') as cf_f: + csv.writer(cf_f).writerow([ + flow_dict.get("Environment", ""), + flow_dict.get("Name", ""), + flow_dict.get("Type", ""), + flow_dict.get("Tier", ""), + flow_dict.get("Active", ""), + flow_dict.get("Reason", "") + ]) + + del flow_detail + del res + + # ========================================== + # 2. FETCH DESKTOP FLOWS + # ========================================== + instance_url = env_props.get("linkedEnvironmentMetadata", {}).get("instanceApiUrl") + + if instance_url: + dv_url = instance_url.rstrip("/") + try: + dv_token = self._get_access_token(f"{dv_url}/.default") + if not dv_token: + self.logger.warning(f" [X] Failed to acquire token for Dataverse instance {dv_url}") + continue + + headers_dv = { + "Authorization": f"Bearer {dv_token}", + "Accept": "application/json", + "OData-MaxVersion": "4.0", "OData-Version": "4.0" + } + + dv_api_url = f"{dv_url}/api/data/v9.2/workflows?$filter=category eq 6&$select=name,clientdata,ismanaged,statecode,_ownerid_value&$expand=ownerid" + desktop_flows = self.fetch_all_pages(dv_api_url, headers_dv, context_name="Dataverse Desktop Flows API") + + counts["Desktop Flows"] += len(desktop_flows) + active_desktop_flows = [f for f in desktop_flows if f.get("statecode") == 1] + active_counts["Desktop Flows"] += len(active_desktop_flows) + + self.logger.info(f" -> Desktop Flows: {len(desktop_flows)} total found, {len(active_desktop_flows)} are active.") + + for flow in desktop_flows: + name = flow.get("name", "Unnamed Desktop Flow") + is_managed = flow.get("ismanaged", False) + owner_name = flow.get("ownerid", {}).get("fullname", "Unknown / System") + statecode = flow.get("statecode") + is_active = (statecode == 1) + + if is_managed or "system" in owner_name.lower() or not is_default: + tier_counts["Enterprise/Departmental"] += 1 + tier = "Enterprise" + if is_active: + active_tier_counts["Enterprise/Departmental"] += 1 + else: + tier_counts["Personal Productivity"] += 1 + tier = "Personal" + if is_active: + active_tier_counts["Personal Productivity"] += 1 + + client_data_str = flow.get("clientdata", "") + if client_data_str: + try: + has_nested_loops = client_data_str.lower().count("foreach") > 1 + has_multi_approvals = client_data_str.count("shared_approvals") > 1 + has_advanced_expressions = "@" in client_data_str and any(exp in client_data_str for exp in ["concat(", "split(", "base64("]) + + if has_nested_loops or has_multi_approvals or has_advanced_expressions: + self.logger.info(f" [!] Complex logic detected in Desktop Flow: {name}") + reasons = [] + if has_nested_loops: reasons.append("Nested Loops") + if has_multi_approvals: reasons.append("Multi Approvals") + if has_advanced_expressions: reasons.append("Advanced Expressions") + + flow_dict = { + "Environment": env_display, "Name": name, "Type": "Desktop Flow", "Tier": tier, + "Active": "Yes" if is_active else "No", + "Reason": ", ".join(reasons) + } + with complex_flows_lock: + with open(complex_logic_flows_path, 'a', encoding='utf-8', newline='') as cf_f: + csv.writer(cf_f).writerow([ + flow_dict.get("Environment", ""), + flow_dict.get("Name", ""), + flow_dict.get("Type", ""), + flow_dict.get("Tier", ""), + flow_dict.get("Active", ""), + flow_dict.get("Reason", "") + ]) + except Exception: + pass + except Exception as e: + self.logger.error(f" [X] Failed to authenticate against Dataverse instance {dv_url}: {e}") + + complex_active_count = 0 + complex_inactive_count = 0 + with open(complex_logic_flows_path, 'r', encoding='utf-8') as f_cf: + reader = csv.DictReader(f_cf) + for row in reader: + if row.get("Active") == "Yes": + complex_active_count += 1 + else: + complex_inactive_count += 1 + + results = { + "total_environments": len(environments), + "counts": counts, + "active_counts": active_counts, + "tier_counts": tier_counts, + "active_tier_counts": active_tier_counts, + "premium_connectors": list(premium_connectors_found), + "custom_connectors": list(custom_connectors_found), + "complex_logic_flows_path": complex_logic_flows_path, + "complex_active_count": complex_active_count, + "complex_inactive_count": complex_inactive_count + } + + self.logger.info("Step End: Analysis complete.") + + self.logger.info("Main Process End: Power Automate Telemetry scan finished.") + return results + + +# ================================================================================= +# MODULAR UI COMPONENT +# ================================================================================= + +class PowerAutomateUsageFrame(ctk.CTkFrame): + def update_loading_text(self, text_msg): + if hasattr(self, 'loading_label') and self.loading_label.winfo_exists(): + self.loading_label.configure(text=f"⏳ {text_msg}") + """Self-contained component wrapping Power Automate UI and export controls.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, **kwargs): + self.semaphore = kwargs.pop("concurrency_semaphore", None) + super().__init__(master, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=12, **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.status = None # 'loading', 'success', 'error', None + self.last_complex_flows = [] + self.last_results = {} + + self.build_ui() + + def build_ui(self): + self.pack(fill="x", expand=True, pady=10) + + self.inner_pad = ctk.CTkFrame(self, fg_color="transparent") + self.inner_pad.pack(fill="both", expand=True, padx=20, pady=20) + + self.pa_header = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.pa_header.pack(fill="x", pady=(0, 10)) + + self.header = ctk.CTkFrame(self.pa_header, fg_color="transparent") + self.header.pack(fill="x", pady=(0, 10)) + ctk.CTkLabel(self.header, text="Power Automate", font=FONT_HEADER_SMALL, text_color=COLOR_TEXT_MAIN).pack(side="left") + self.admin_center_link = ctk.CTkLabel( + self.header, + text="Open Power Platform Admin Center ↗", + font=FONT_BODY_BOLD, + text_color=COLOR_PRIMARY, + cursor="hand2" + ) + self.admin_center_link.pack(side="left", padx=(15, 0)) + self.admin_center_link.bind("", lambda e: webbrowser.open("https://admin.powerplatform.microsoft.com")) + self.admin_center_link.bind("", lambda e: self.admin_center_link.configure(text_color=COLOR_PRIMARY_HOVER)) + self.admin_center_link.bind("", lambda e: self.admin_center_link.configure(text_color=COLOR_PRIMARY)) + self.reload_btn = ctk.CTkButton( + self.header, + state="disabled", text="↻ Reload", + width=80, + height=24, + font=__import__("customtkinter").CTkFont(family="Segoe UI", size=12), + fg_color="transparent", + border_width=1, + text_color="#2563EB", + hover_color="#DBEAFE", + command=self._retry_fetch + ) + self.reload_btn.pack(side="right", padx=(10, 15)) + + self.btn_export_pa = ctk.CTkButton( + self.header, text="Export Complex Flows", width=160, height=32, corner_radius=16, + font=FONT_BODY_BOLD, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, + command=self.export_complex_flows, state="disabled" + ) + self.btn_export_pa.pack(side="right") + + self.state_frame = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.grid_frame = ctk.CTkFrame(self.inner_pad, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + + # Height control slider for Power Automate Chart (packed above the chart dynamically) + self.pa_height_var = ctk.DoubleVar(value=400) + self.pa_slider_frame = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + + self.slider_pa_height = ctk.CTkSlider( + self.pa_slider_frame, from_=200, to=800, number_of_steps=60, + variable=self.pa_height_var, width=120, height=16, + command=self._on_pa_height_slider_change + ) + self.slider_pa_height.pack(side="right") + + self.lbl_pa_height = ctk.CTkLabel(self.pa_slider_frame, text="Height: 400px", font=FONT_BODY_SMALL, text_color=COLOR_TEXT_SUB) + self.lbl_pa_height.pack(side="right", padx=(0, 10)) + + self.pa_chart_container = ctk.CTkFrame( + self.inner_pad, fg_color=COLOR_SURFACE, + border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8, + height=400 + ) + self.pa_chart_container.pack_propagate(False) + + self.reset_view() + + def reset_view(self): + self.pack_forget() + self.state_frame.pack_forget() + self.grid_frame.pack_forget() + self.pa_slider_frame.pack_forget() + self.pa_chart_container.pack_forget() + self.btn_export_pa.configure(state="disabled") + self.last_complex_flows = [] + self.last_results = {} + + for w in self.state_frame.winfo_children(): + w.destroy() + for w in self.grid_frame.winfo_children(): + w.destroy() + for w in self.pa_chart_container.winfo_children(): + w.destroy() + + def _set_state_loading(self, msg="Loading..."): + for w in self.state_frame.winfo_children(): + w.destroy() + self.loading_label = __import__("customtkinter").CTkLabel(self.state_frame, text=f"⏳ {msg}", text_color="#6b7280", font=__import__("customtkinter").CTkFont(family="Segoe UI", size=13)) + self.loading_label.pack(pady=(20, 5)) + pb = __import__("customtkinter").CTkProgressBar(self.state_frame, mode="indeterminate", width=250, fg_color="#F3F4F6", progress_color="#2563EB") + pb.pack(pady=(0, 20)) + pb.start() + self.state_frame.pack(fill="x", expand=True) + + def _set_state_error(self, error_msg): + for w in self.state_frame.winfo_children(): + w.destroy() + + display_msg = error_msg + if "401" in error_msg or "403" in error_msg or "unauthorized" in error_msg.lower() or "forbidden" in error_msg.lower(): + try: + tenant, clients, secrets = self.get_credentials() + tenant_id = tenant if tenant else "" + client_id = clients[0] if clients else "" + except Exception: + tenant_id = "" + client_id = "" + + display_msg = ( + "Power Platform Admin / Dataverse Permissions required.\n\n" + "1. Register the App Registration as a Management App via PowerShell (Global/Power Platform Admin):\n" + " Install-Module -Name Microsoft.PowerApps.Administration.PowerShell -Force\n" + f" Add-PowerAppsAccount -Endpoint prod -TenantID \"{tenant_id}\"\n" + f" New-PowerAppManagementApp -ApplicationId \"{client_id}\"\n\n" + "2. Assign the App Registration the 'System Administrator' security role in target Dataverse environments." + ) + + ctk.CTkLabel(self.state_frame, text=f"✖ {display_msg}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="left", wraplength=700).pack(pady=(20, 15)) + ctk.CTkButton(self.state_frame, text="Try Again", command=self._retry_fetch, width=120, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 20)) + self.state_frame.pack(fill="x", expand=True) + + def _retry_fetch(self): + if hasattr(self, 'reload_btn') and self.reload_btn.winfo_exists(): + self.reload_btn.configure(state="disabled") + tenant, clients, secrets = self.get_credentials() + if tenant: + self.trigger_fetch(tenant, clients[0], secrets[0]) + + def trigger_fetch(self, tenant, client_id, client_secret): + usage_logger.info("Power Automate trigger_fetch called. Spawning background worker thread...") + self.status = "loading" + self.on_status_change() + + self.pack(fill="x", expand=True, pady=10) + self.grid_frame.pack_forget() + self.pa_slider_frame.pack_forget() + self.pa_chart_container.pack_forget() + + self._set_state_loading("Scanning Power Automate flows...") + + threading.Thread( + target=self._execute_worker, + args=(tenant, client_id, client_secret), + daemon=True + ).start() + + def _execute_worker(self, tenant: str, client_id: str, client_secret: str): + if self.semaphore: + self.semaphore.acquire() + try: + scanner = PowerAutomateScanner(tenant, client_id, client_secret) + results = scanner.scan_flows() + usage_logger.info("Successfully completed Power Automate scan.") + self.after(0, self._render_success, results) + except Exception as e: + usage_logger.error("Exception caught in PowerAutomateUsage worker.", exc_info=True) + self.after(0, self._render_error, str(e)) + finally: + if self.semaphore: + self.semaphore.release() + + def _render_success(self, results: dict): + self.last_results = results + if hasattr(self, 'reload_btn') and self.reload_btn.winfo_exists(): + self.reload_btn.configure(state="normal") + self.state_frame.pack_forget() + for w in self.grid_frame.winfo_children(): + w.destroy() + + self.grid_frame.pack(fill="x", expand=True) + + if not results: + empty_cell = ctk.CTkFrame(self.grid_frame, fg_color="transparent") + empty_cell.pack(fill="x", expand=True, pady=15) + ctk.CTkLabel(empty_cell, text="No Power Automate data found.", text_color=COLOR_TEXT_SUB).pack() + self.status = "success" + self.on_status_change() + return + + total_envs = results.get("total_environments", 0) + counts = results.get("counts", {}) + active_counts = results.get("active_counts", {}) + tier_counts = results.get("tier_counts", {}) + active_tier_counts = results.get("active_tier_counts", {}) + premium_conns = results.get("premium_connectors", []) + custom_conns = results.get("custom_connectors", []) + complex_flows = results.get("complex_logic_flows", []) + + total_flows = counts.get("Cloud Flows", 0) + counts.get("Desktop Flows", 0) + + summary_frame = ctk.CTkFrame(self.grid_frame, fg_color=COLOR_OUTLINE_LIGHT, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + summary_frame.pack(fill="x", pady=20) + + for i in range(2): + summary_frame.grid_columnconfigure(i, weight=1) + + headers_pa = ["Metric", "Value"] + for col_idx, head_text in enumerate(headers_pa): + cell = ctk.CTkFrame(summary_frame, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + prem_str = ", ".join(premium_conns) if premium_conns else "0" + cust_str = ", ".join(custom_conns) if custom_conns else "0" + + mapping = [ + ("Total Environments Scanned", total_envs), + ("Total Flows (Active + Inactive)", total_flows), + ("Premium Connectors In Use", prem_str), + ("Custom Connectors In Use", cust_str), + ] + + r_idx = 1 + for label, val in mapping: + bg_style = COLOR_SURFACE if r_idx % 2 == 0 else COLOR_SURFACE_VARIANT + + c0 = ctk.CTkFrame(summary_frame, fg_color=bg_style, corner_radius=0) + c0.grid(row=r_idx, column=0, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c0, text=label, font=FONT_BODY_BOLD, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=6, anchor="nw") + + c1 = ctk.CTkFrame(summary_frame, fg_color=bg_style, corner_radius=0) + c1.grid(row=r_idx, column=1, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c1, text=str(val), font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN, wraplength=400).pack(padx=10, pady=6, anchor="nw") + + r_idx += 1 + + complex_flows_path = results.get("complex_logic_flows_path") + self.last_complex_flows_path = complex_flows_path + if complex_flows_path and os.path.exists(complex_flows_path) and os.path.getsize(complex_flows_path) > 0: + self.btn_export_pa.configure(state="normal") + else: + self.btn_export_pa.configure(state="disabled") + + if total_flows > 0: + self.pa_slider_frame.pack(fill="x", pady=(10, 0)) + self.pa_chart_container.pack(fill="x", pady=(5, 20)) + for w in self.pa_chart_container.winfo_children(): + w.destroy() + + if not MATPLOTLIB_AVAILABLE: + ctk.CTkLabel(self.pa_chart_container, text="Matplotlib is required to render charts.\nPlease install it using 'pip install matplotlib'.", text_color=COLOR_ERROR).pack(pady=15) + else: + try: + fig = Figure(figsize=(10, 4), dpi=100) + ax = fig.add_subplot(111) + fig.patch.set_facecolor(COLOR_SURFACE) + ax.set_facecolor(COLOR_SURFACE) + + categories = ['Cloud Flows', 'Desktop Flows', 'Personal Flows', 'Enterprise Flows', 'Complex Flows'] + + c_total = counts.get("Cloud Flows", 0) + c_active = active_counts.get("Cloud Flows", 0) + c_inactive = c_total - c_active + + d_total = counts.get("Desktop Flows", 0) + d_active = active_counts.get("Desktop Flows", 0) + d_inactive = d_total - d_active + + p_total = tier_counts.get("Personal Productivity", 0) + p_active = active_tier_counts.get("Personal Productivity", 0) + p_inactive = p_total - p_active + + e_total = tier_counts.get("Enterprise/Departmental", 0) + e_active = active_tier_counts.get("Enterprise/Departmental", 0) + e_inactive = e_total - e_active + + complex_active = results.get("complex_active_count", 0) + complex_inactive = results.get("complex_inactive_count", 0) + + actives = [c_active, d_active, p_active, e_active, complex_active] + inactives = [c_inactive, d_inactive, p_inactive, e_inactive, complex_inactive] + + x = range(len(categories)) + width = 0.15 + + color_active = COLOR_PRIMARY + color_inactive = COLOR_TONAL_BG + + rects1 = ax.bar(x, actives, width, label='Active', color=color_active) + rects2 = ax.bar([i + width for i in x], inactives, width, label='Inactive', color=color_inactive) + + ax.set_ylabel('Count', color=COLOR_TEXT_MAIN, fontsize=10, fontweight='bold') + ax.set_title('Power Automate Flows Breakdown', color=COLOR_TEXT_MAIN, fontsize=12, fontweight='bold') + ax.set_xticks([i + width/2 for i in x]) + ax.set_xticklabels(categories, color=COLOR_TEXT_MAIN, fontsize=10, fontweight='bold') + ax.legend(facecolor=COLOR_SURFACE, edgecolor=COLOR_OUTLINE_LIGHT, labelcolor=COLOR_TEXT_MAIN, prop={'weight':'bold', 'size':9}) + + ax.bar_label(rects1, padding=3, color=COLOR_TEXT_MAIN, fontsize=9, fontweight='bold') + ax.bar_label(rects2, padding=3, color=COLOR_TEXT_MAIN, fontsize=9, fontweight='bold') + + for spine in ax.spines.values(): + spine.set_color(COLOR_OUTLINE_LIGHT) + + ax.tick_params(axis='y', colors=COLOR_TEXT_MAIN, labelsize=9) + for label in ax.get_yticklabels(): + label.set_fontweight('bold') + + max_val = max(max(actives), max(inactives)) + ax.set_ylim(0, max(max_val + 3, int(max_val * 1.3))) + + fig.tight_layout() + + canvas = FigureCanvasTkAgg(fig, master=self.pa_chart_container) + canvas.draw() + canvas.get_tk_widget().pack(fill="both", expand=True, padx=20, pady=10) + + except Exception as e: + usage_logger.error(f"Error drawing Power Automate charts: {e}", exc_info=True) + + self.status = "success" + self.on_status_change() + + def _render_error(self, err_msg): + usage_logger.warning(f"Power Automate fetch failed: {err_msg}") + if hasattr(self, 'reload_btn') and self.reload_btn.winfo_exists(): + self.reload_btn.configure(state="normal") + self._set_state_error(err_msg) + self.status = "error" + self.on_status_change() + + def export_complex_flows(self): + usage_logger.info("Exporting complex flows to local spreadsheet requested.") + if not hasattr(self, "last_complex_flows_path") or not self.last_complex_flows_path: + return + + ts = datetime.now().strftime("%Y%m%d_%H%M%S") + f = filedialog.asksaveasfilename( + initialfile=f"complex_flows_{ts}.csv", + defaultextension=".csv", + filetypes=[("CSV Spreadsheet", "*.csv")] + ) + + if not f: + return + + headers = ["Environment", "Name", "Type", "Tier", "Active", "Reason"] + + try: + shutil.copyfile(self.last_complex_flows_path, f) + usage_logger.info("Complex flows exported successfully.") + messagebox.showinfo("Export Successful", f"Complex flows successfully saved to:\n{f}", parent=self) + except Exception as e: + usage_logger.error("Failed writing export spreadsheet to disk.", exc_info=True) + messagebox.showerror("Export Error", f"Failed to save file:\n{e}", parent=self) + + def _on_pa_height_slider_change(self, val): + height_val = int(val) + self.lbl_pa_height.configure(text=f"Height: {height_val}px") + self.pa_chart_container.configure(height=height_val) diff --git a/telemetry/security/__init__.py b/telemetry/security/__init__.py new file mode 100644 index 00000000..7d556d45 --- /dev/null +++ b/telemetry/security/__init__.py @@ -0,0 +1,225 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Consolidated UI container for Data Security & Governance telemetry.""" + +import logging +import customtkinter as ctk +import webbrowser + +from telemetry.styles import * +from telemetry.ediscovery_ui import EDiscoveryFrame +from telemetry.security.sensitivity_labels import SensitivityLabelsSubFrame +from telemetry.security.retention_policies import RetentionPoliciesSubFrame +from telemetry.security.dlp_policies import DLPPoliciesSubFrame +from telemetry.security.sensitive_info_types import SensitiveInfoTypesSubFrame +from telemetry.security.authentication import AuthenticationSubFrame +from telemetry.security.service_principals_sso import ServicePrincipalsSsoSubFrame + +class DataSecurityGovernanceFrame(ctk.CTkFrame): + """Card container enclosing all Data Security & Governance subframes.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, **kwargs): + self.semaphore = kwargs.pop("concurrency_semaphore", None) + self.get_delegated_auth = kwargs.pop("delegated_auth_callback", None) + + super().__init__(master, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=12, **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.status = None + + self.build_ui() + + def build_ui(self): + self.pack(fill="x", expand=True, pady=10) + + self.inner_pad = ctk.CTkFrame(self, fg_color="transparent") + self.inner_pad.pack(fill="both", expand=True, padx=20, pady=20) + + # Main Title Header + self.header = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.header.pack(fill="x", pady=(0, 10)) + ctk.CTkLabel(self.header, text="Data Security & Governance", font=FONT_HEADER_SMALL, text_color=COLOR_TEXT_MAIN).pack(side="left") + + # 1. Sensitivity Labels SubFrame + self.sensitivity_frame = SensitivityLabelsSubFrame( + self.inner_pad, self.log_msg, self.get_credentials, self._check_overall_status, semaphore=self.semaphore + ) + + # 2. Retention Compliance Policies SubFrame + self.retention_frame = RetentionPoliciesSubFrame( + self.inner_pad, self.log_msg, self.get_credentials, self._check_overall_status, semaphore=self.semaphore + ) + + # 3. DLP Policies SubFrame + self.dlp_frame = DLPPoliciesSubFrame( + self.inner_pad, self.log_msg, self.get_credentials, self._check_overall_status, semaphore=self.semaphore + ) + + # 4. Sensitive Information Types (SIT) SubFrame + self.sit_frame = SensitiveInfoTypesSubFrame( + self.inner_pad, self.log_msg, self.get_credentials, self._check_overall_status, semaphore=self.semaphore + ) + + # 5. Authentication Mechanics (Conditional Access) SubFrame + self.auth_frame = AuthenticationSubFrame( + self.inner_pad, self.log_msg, self.get_credentials, self._check_overall_status, semaphore=self.semaphore + ) + + # 6. Service Principals SSO Modes SubFrame + self.sso_frame = ServicePrincipalsSsoSubFrame( + self.inner_pad, self.log_msg, self.get_credentials, self._check_overall_status, semaphore=self.semaphore + ) + + # 7. eDiscovery Cases Section (Conditional based on Delegated Auth setting) + self.ediscovery_header_frame = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + ctk.CTkLabel( + self.ediscovery_header_frame, text="eDiscovery Cases", font=FONT_HEADER_SMALL, text_color=COLOR_TEXT_MAIN + ).pack(side="left", anchor="w") + + self.ediscovery_body_frame = ctk.CTkFrame( + self.inner_pad, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8 + ) + self.ediscovery_content = ctk.CTkFrame(self.ediscovery_body_frame, fg_color="transparent") + + lbl_inst1 = ctk.CTkLabel( + self.ediscovery_content, + text="eDiscovery cases cannot be scanned directly under standard Application permissions. To view your active cases, please navigate to Microsoft Purview, or enable Delegated Authentication on the Connection screen to view them directly here.", + font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN, justify="left", wraplength=700 + ) + lbl_inst1.pack(anchor="w", pady=(0, 8)) + + lbl_cases_link = ctk.CTkLabel( + self.ediscovery_content, text="🔗 Open Purview eDiscovery Cases Portal", + font=FONT_BODY_BOLD, text_color=COLOR_PRIMARY, cursor="hand2" + ) + lbl_cases_link.pack(anchor="w", pady=(0, 15)) + lbl_cases_link.bind("", lambda e: webbrowser.open("https://purview.microsoft.com/ediscovery/casespage")) + lbl_cases_link.bind("", lambda e: lbl_cases_link.configure(text_color=COLOR_PRIMARY_HOVER)) + lbl_cases_link.bind("", lambda e: lbl_cases_link.configure(text_color=COLOR_PRIMARY)) + + lbl_inst2 = ctk.CTkLabel( + self.ediscovery_content, + text="Note: Accessing eDiscovery cases requires your administrator account to have the eDiscovery Manager role assigned in the tenant permissions page:", + font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_SUB, justify="left", wraplength=700 + ) + lbl_inst2.pack(anchor="w", pady=(0, 8)) + + lbl_roles_link = ctk.CTkLabel( + self.ediscovery_content, text="🔗 Assign eDiscovery Manager Role in Purview Settings", + font=FONT_BODY_BOLD, text_color=COLOR_PRIMARY, cursor="hand2" + ) + lbl_roles_link.pack(anchor="w") + lbl_roles_link.bind("", lambda e: webbrowser.open("https://purview.microsoft.com/settings/purviewpermissions")) + lbl_roles_link.bind("", lambda e: lbl_roles_link.configure(text_color=COLOR_PRIMARY_HOVER)) + lbl_roles_link.bind("", lambda e: lbl_roles_link.configure(text_color=COLOR_PRIMARY)) + + self.ediscovery_ui_view = EDiscoveryFrame( + master=self.ediscovery_body_frame, + log_callback=self.log_msg, + credentials_callback=self.get_credentials, + status_change_callback=self._check_overall_status, + concurrency_semaphore=self.semaphore, + delegated_auth_callback=self.get_delegated_auth + ) + + self.reset_view() + + def reset_view(self): + self.pack_forget() + self.sensitivity_frame.pack_forget() + self.retention_frame.pack_forget() + self.dlp_frame.pack_forget() + self.sit_frame.pack_forget() + self.auth_frame.pack_forget() + self.sso_frame.pack_forget() + self.ediscovery_header_frame.pack_forget() + self.ediscovery_body_frame.pack_forget() + + self.sensitivity_frame.reset_view() + self.retention_frame.reset_view() + self.dlp_frame.reset_view() + self.sit_frame.reset_view() + self.auth_frame.reset_view() + self.sso_frame.reset_view() + self.ediscovery_ui_view.reset_view() + self.status = None + + def trigger_fetch(self, tenant, client_id, client_secret): + self.status = "loading" + self.on_status_change() + + self.pack(fill="x", expand=True, pady=(20, 5)) + + # Pack and trigger each subframe + self.sensitivity_frame.pack(fill="x", pady=(0, 15)) + self.sensitivity_frame.trigger_fetch(tenant, client_id, client_secret) + + self.retention_frame.pack(fill="x", pady=(20, 15)) + self.retention_frame.trigger_fetch(tenant, client_id, client_secret) + + self.dlp_frame.pack(fill="x", pady=(20, 15)) + self.dlp_frame.trigger_fetch(tenant, client_id, client_secret) + + self.sit_frame.pack(fill="x", pady=(20, 15)) + self.sit_frame.trigger_fetch(tenant, client_id, client_secret) + + self.auth_frame.pack(fill="x", pady=(20, 15)) + self.auth_frame.trigger_fetch(tenant, client_id, client_secret) + + self.sso_frame.pack(fill="x", pady=(20, 15)) + self.sso_frame.trigger_fetch(tenant, client_id, client_secret) + + # Draw eDiscovery block + self.ediscovery_header_frame.pack(fill="x", pady=(20, 5)) + self.ediscovery_body_frame.pack(fill="x", pady=(0, 15)) + + use_delegated = self.get_delegated_auth() if self.get_delegated_auth else False + if use_delegated: + self.ediscovery_content.pack_forget() + self.ediscovery_ui_view.pack(fill="x", expand=True) + self.ediscovery_ui_view.trigger_fetch(tenant, client_id, client_secret, use_delegated_auth=True) + else: + self.ediscovery_ui_view.pack_forget() + self.ediscovery_content.pack(fill="x", padx=20, pady=20) + + def _check_overall_status(self): + statuses = [ + self.sensitivity_frame.status, + self.retention_frame.status, + self.dlp_frame.status, + self.sit_frame.status, + self.auth_frame.status, + self.sso_frame.status, + self.ediscovery_ui_view.status + ] + + if "loading" in statuses: + self.status = "loading" + elif all(s == "error" for s in statuses if s is not None): + self.status = "error" + else: + self.status = "success" + self.on_status_change() + + def cancel(self): + self.sensitivity_frame.cancel() + self.retention_frame.cancel() + self.dlp_frame.cancel() + self.sit_frame.cancel() + self.auth_frame.cancel() + self.sso_frame.cancel() + self.ediscovery_ui_view.reset_view() + self.status = None diff --git a/telemetry/security/authentication.py b/telemetry/security/authentication.py new file mode 100644 index 00000000..9c23ea57 --- /dev/null +++ b/telemetry/security/authentication.py @@ -0,0 +1,284 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""UI component for Conditional Access Policies telemetry.""" + +import os +import time +import logging +import threading +import csv +import shutil +from datetime import datetime +import customtkinter as ctk +from tkinter import filedialog, messagebox + +from core.graph.security.authentication import run_authentication_pipeline +from telemetry.styles import * + +usage_logger = logging.getLogger("M365TelemetryAsyncLogger.AuthUI") + +class AuthenticationSubFrame(ctk.CTkFrame): + """Sub-frame for Conditional Access Policies.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, semaphore=None, **kwargs): + super().__init__(master, fg_color="transparent", **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.semaphore = semaphore + self.status = None + self.page = 0 + self.ITEMS_PER_PAGE = 5 + self.csv_path = None + self.is_cancelled = False + + self.build_ui() + + def build_ui(self): + self.header_frame = ctk.CTkFrame(self, fg_color="transparent") + self.header_frame.pack(fill="x", pady=(0, 10)) + + ctk.CTkLabel(self.header_frame, text="Authentication Mechanics (Conditional Access)", font=FONT_BODY_BOLD, text_color=COLOR_TEXT_MAIN).pack(side="left") + + self.lbl_link = ctk.CTkLabel( + self.header_frame, text="Open Microsoft Entra Conditional Access Portal ↗", + font=FONT_BODY_BOLD, text_color=COLOR_PRIMARY, cursor="hand2" + ) + self.lbl_link.pack(side="left", anchor="w", padx=(15, 0)) + self.lbl_link.bind("", lambda e: __import__("webbrowser").open("https://portal.azure.com/#view/Microsoft_AAD_IAM/ConditionalAccessBlade/~/Policies")) + self.lbl_link.bind("", lambda e: self.lbl_link.configure(text_color=COLOR_PRIMARY_HOVER)) + self.lbl_link.bind("", lambda e: self.lbl_link.configure(text_color=COLOR_PRIMARY)) + + self.btn_reload = ctk.CTkButton( + self.header_frame, state="disabled", text="↻ Reload", width=80, height=24, + font=ctk.CTkFont(family="Segoe UI", size=12), + fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color="#2563EB", hover_color="#DBEAFE", + command=self.trigger_fetch_individual + ) + self.btn_reload.pack(side="right", padx=(10, 0)) + + self.btn_export = ctk.CTkButton( + self.header_frame, text="Export CA Policies", width=180, height=32, corner_radius=16, + font=FONT_BODY_BOLD, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, + command=self._export, state="disabled" + ) + self.btn_export.pack(side="right") + + self.state_frame = ctk.CTkFrame(self, fg_color="transparent") + + self.grid_frame = ctk.CTkFrame(self, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + + self.grid_frame.grid_columnconfigure(0, weight=1) + self.grid_frame.grid_columnconfigure(1, weight=1) + self.grid_frame.grid_columnconfigure(2, weight=1) + self.grid_frame.grid_columnconfigure(3, weight=1) + self.grid_frame.grid_columnconfigure(4, weight=1) + + headers = ["Policy Name", "State", "Target Users", "Target Apps", "Enforced Controls"] + for col_idx, head_text in enumerate(headers): + cell = ctk.CTkFrame(self.grid_frame, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + self.reset_view() + + def reset_view(self): + self.status = None + self.is_cancelled = False + self.csv_path = None + self.page = 0 + self.state_frame.pack_forget() + self.grid_frame.pack_forget() + self.btn_export.configure(state="disabled") + for w in self.state_frame.winfo_children(): w.destroy() + for w in self.grid_frame.winfo_children(): + info = w.grid_info() + if "row" in info and int(info["row"]) > 0: w.destroy() + + def _set_loading_state(self, msg): + for w in self.state_frame.winfo_children(): w.destroy() + self.state_frame.pack(fill="x", expand=True) + ctk.CTkLabel(self.state_frame, text=f"⏳ {msg}", text_color=COLOR_TEXT_SUB, font=FONT_BODY_MEDIUM).pack(pady=(15, 5)) + pb = ctk.CTkProgressBar(self.state_frame, mode="indeterminate", width=200, fg_color=COLOR_SURFACE_VARIANT, progress_color=COLOR_PRIMARY) + pb.pack(pady=(0, 15)) + pb.start() + + def trigger_fetch_individual(self): + tenant, clients, secrets = self.get_credentials() + if tenant: + self.trigger_fetch(tenant, clients[0], secrets[0]) + + def trigger_fetch(self, tenant, client_id, client_secret): + self.status = "loading" + self.is_cancelled = False + self.page = 0 + self.btn_reload.configure(state="disabled") + self.btn_export.configure(state="disabled") + self.grid_frame.pack_forget() + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + if os.path.basename(script_dir) == "security": + script_dir = os.path.dirname(script_dir) + reports_dir = os.path.join(script_dir, "reports", f"{tenant}_{client_id}") + self.csv_path = os.path.join(reports_dir, "auth_policies.csv") + + self._set_loading_state("Scanning Conditional Access policies...") + self.on_status_change() + + threading.Thread(target=self._execute_worker, args=(tenant, client_id, client_secret), daemon=True).start() + + def _execute_worker(self, tenant, client_id, client_secret): + if self.semaphore: self.semaphore.acquire() + try: + run_authentication_pipeline( + client_id=client_id, + client_secret=client_secret, + tenant_id=tenant, + csv_path=self.csv_path, + is_cancelled_callback=lambda: self.is_cancelled + ) + self.status = "success" + self.after(0, self._render_success) + except Exception as e: + usage_logger.error(f"CA policies fetch error: {e}", exc_info=True) + self.status = "error" + self.after(0, self._render_error, str(e)) + finally: + if self.semaphore: self.semaphore.release() + self.after(0, self.on_status_change) + + def _render_success(self): + self.btn_reload.configure(state="normal") + self.btn_export.configure(state="normal") + self.state_frame.pack_forget() + self.grid_frame.pack(fill="x") + + self._update_grid() + + def _render_error(self, err_msg): + self.btn_reload.configure(state="normal") + self.grid_frame.pack_forget() + for w in self.state_frame.winfo_children(): w.destroy() + self.state_frame.pack(fill="x", expand=True) + display_msg = err_msg + if "401" in err_msg or "403" in err_msg or "permission" in err_msg.lower() or "unauthorized" in err_msg.lower() or "forbidden" in err_msg.lower() or "policy.read" in err_msg.lower(): + display_msg = "Conditional Access telemetry permission required.\nPlease grant the 'Policy.Read.All' (or 'Policy.Read') application permission to your App Registration in Microsoft Entra ID." + ctk.CTkLabel(self.state_frame, text=f"✖ {display_msg}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center", wraplength=700).pack(pady=(15, 5)) + ctk.CTkButton(self.state_frame, text="Try Again", command=self.trigger_fetch_individual, width=100, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 15)) + + def _load_page(self): + if not self.csv_path or not os.path.exists(self.csv_path): return [], 0 + items = [] + total_count = 0 + try: + with open(self.csv_path, 'r', encoding='utf-8') as f: + reader = csv.DictReader(f) + all_rows = list(reader) + total_count = len(all_rows) + start_idx = self.page * self.ITEMS_PER_PAGE + end_idx = start_idx + self.ITEMS_PER_PAGE + items = all_rows[start_idx:end_idx] + except Exception as e: + usage_logger.error(f"Error reading CA policies CSV: {e}") + return items, total_count + + def _update_grid(self): + for w in self.grid_frame.winfo_children(): + info = w.grid_info() + if "row" in info and int(info["row"]) > 0: w.destroy() + + page_data, total_count = self._load_page() + total_pages = (total_count - 1) // self.ITEMS_PER_PAGE + 1 if total_count > 0 else 1 + + for offset, policy in enumerate(page_data, start=1): + r_idx = offset + bg_style = COLOR_SURFACE if r_idx % 2 != 0 else COLOR_SURFACE_VARIANT + + name = policy.get("name", "N/A") + state = policy.get("state", "N/A") + target_users = policy.get("target_users", "N/A") + target_apps = policy.get("target_apps", "N/A") + controls = policy.get("controls", "N/A") + + vals = [name, state, target_users, target_apps, controls] + for col_idx, val in enumerate(vals): + cell = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + cell.grid(row=r_idx, column=col_idx, sticky="nsew", padx=0, pady=(0, 1)) + lbl = ctk.CTkLabel(cell, text=str(val), font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN, justify="left", wraplength=180) + lbl.pack(padx=10, pady=12, anchor="nw") + + # Pagination controls row + control_frame = ctk.CTkFrame(self.grid_frame, fg_color="transparent") + control_frame.grid(row=self.ITEMS_PER_PAGE + 1, column=0, columnspan=5, pady=0, sticky="ew") + + center_container = ctk.CTkFrame(control_frame, fg_color="transparent") + center_container.pack(pady=(5, 10)) + + prev_state = "normal" if self.page > 0 else "disabled" + ctk.CTkButton( + center_container, text="◀ Prev", width=70, height=22, corner_radius=6, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, state=prev_state, + command=lambda: self._change_page(-1) + ).pack(side="left", padx=5) + + ctk.CTkLabel(center_container, text=f"Page {self.page + 1} of {total_pages}", font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_SUB).pack(side="left", padx=15) + + next_state = "normal" if self.page < total_pages - 1 else "disabled" + ctk.CTkButton( + center_container, text="Next ▶", width=70, height=22, corner_radius=6, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, state=next_state, + command=lambda: self._change_page(1) + ).pack(side="left", padx=5) + + def _change_page(self, delta): + self.page += delta + self._update_grid() + + def _export(self): + if not self.csv_path or not os.path.exists(self.csv_path): + messagebox.showinfo("No Data", "There is no CA policy data to export.", parent=self) + return + + ts = datetime.now().strftime("%Y%m%d_%H%M%S") + f = filedialog.asksaveasfilename( + initialfile=f"auth_policies_{ts}.csv", + defaultextension=".csv", + filetypes=[("CSV Files", "*.csv")], + parent=self + ) + if not f: return + try: + shutil.copyfile(self.csv_path, f) + messagebox.showinfo("Export Successful", f"Conditional Access policies exported to:\n{f}", parent=self) + except Exception as e: + messagebox.showerror("Export Failed", f"Error: {e}", parent=self) + + @property + def last_data(self): + if not self.csv_path or not os.path.exists(self.csv_path): return [] + try: + with open(self.csv_path, 'r', encoding='utf-8') as f: + reader = csv.DictReader(f) + return list(reader) + except Exception: + return [] + + def cancel(self): + self.is_cancelled = True + self.status = None diff --git a/telemetry/security/dlp_policies.py b/telemetry/security/dlp_policies.py new file mode 100644 index 00000000..f87016b9 --- /dev/null +++ b/telemetry/security/dlp_policies.py @@ -0,0 +1,306 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""UI component for Data Loss Prevention (DLP) Policies.""" + +import os +import time +import logging +import threading +import csv +import shutil +from datetime import datetime +import customtkinter as ctk +from tkinter import filedialog, messagebox + +from core.graph.security.dlp_policies import run_dlp_policies_pipeline +from telemetry.styles import * + +usage_logger = logging.getLogger("M365TelemetryAsyncLogger.DLPUI") + +class DLPPoliciesSubFrame(ctk.CTkFrame): + """Sub-frame for Data Loss Prevention Policies.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, semaphore=None, **kwargs): + super().__init__(master, fg_color="transparent", **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.semaphore = semaphore + self.status = None + self.page = 0 + self.ITEMS_PER_PAGE = 5 + self.csv_path = None + self.is_cancelled = False + + self.build_ui() + + def build_ui(self): + self.header_frame = ctk.CTkFrame(self, fg_color="transparent") + self.header_frame.pack(fill="x", pady=(0, 10)) + + ctk.CTkLabel(self.header_frame, text="Data Loss Prevention (DLP) Policies", font=FONT_BODY_BOLD, text_color=COLOR_TEXT_MAIN).pack(side="left") + + self.lbl_link = ctk.CTkLabel( + self.header_frame, text="Open Purview DLP Portal ↗", + font=FONT_BODY_BOLD, text_color=COLOR_PRIMARY, cursor="hand2" + ) + self.lbl_link.pack(side="left", anchor="w", padx=(15, 0)) + self.lbl_link.bind("", lambda e: __import__("webbrowser").open("https://purview.microsoft.com/datalossprevention/policies")) + self.lbl_link.bind("", lambda e: self.lbl_link.configure(text_color=COLOR_PRIMARY_HOVER)) + self.lbl_link.bind("", lambda e: self.lbl_link.configure(text_color=COLOR_PRIMARY)) + + self.btn_reload = ctk.CTkButton( + self.header_frame, state="disabled", text="↻ Reload", width=80, height=24, + font=ctk.CTkFont(family="Segoe UI", size=12), + fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color="#2563EB", hover_color="#DBEAFE", + command=self.trigger_fetch_individual + ) + self.btn_reload.pack(side="right", padx=(10, 0)) + + self.btn_export = ctk.CTkButton( + self.header_frame, text="Export DLP Policies", width=180, height=32, corner_radius=16, + font=FONT_BODY_BOLD, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, + command=self._export, state="disabled" + ) + self.btn_export.pack(side="right") + + self.state_frame = ctk.CTkFrame(self, fg_color="transparent") + + self.grid_frame = ctk.CTkFrame(self, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + + self.grid_frame.grid_columnconfigure(0, weight=3) + self.grid_frame.grid_columnconfigure(1, weight=1) + self.grid_frame.grid_columnconfigure(2, weight=2) + self.grid_frame.grid_columnconfigure(3, weight=1) + self.grid_frame.grid_columnconfigure(4, weight=2) + self.grid_frame.grid_columnconfigure(5, weight=2) + + headers = ["Policy Name", "Mode", "Workload", "State", "Actions", "Created By"] + for col_idx, head_text in enumerate(headers): + cell = ctk.CTkFrame(self.grid_frame, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + self.reset_view() + + def reset_view(self): + self.status = None + self.is_cancelled = False + self.csv_path = None + self.page = 0 + self.state_frame.pack_forget() + self.grid_frame.pack_forget() + self.btn_export.configure(state="disabled") + for w in self.state_frame.winfo_children(): w.destroy() + for w in self.grid_frame.winfo_children(): + info = w.grid_info() + if "row" in info and int(info["row"]) > 0: w.destroy() + + def _set_loading_state(self, msg): + for w in self.state_frame.winfo_children(): w.destroy() + self.state_frame.pack(fill="x", expand=True) + ctk.CTkLabel(self.state_frame, text=f"⏳ {msg}", text_color=COLOR_TEXT_SUB, font=FONT_BODY_MEDIUM).pack(pady=(15, 5)) + pb = ctk.CTkProgressBar(self.state_frame, mode="indeterminate", width=200, fg_color=COLOR_SURFACE_VARIANT, progress_color=COLOR_PRIMARY) + pb.pack(pady=(0, 15)) + pb.start() + + def trigger_fetch_individual(self): + tenant, clients, secrets = self.get_credentials() + if tenant: + self.trigger_fetch(tenant, clients[0], secrets[0]) + + def trigger_fetch(self, tenant, client_id, client_secret): + self.status = "loading" + self.is_cancelled = False + self.page = 0 + self.btn_reload.configure(state="disabled") + self.btn_export.configure(state="disabled") + self.grid_frame.pack_forget() + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + if os.path.basename(script_dir) == "security": + script_dir = os.path.dirname(script_dir) + reports_dir = os.path.join(script_dir, "reports", f"{tenant}_{client_id}") + self.csv_path = os.path.join(reports_dir, "dlp_policies.csv") + + self._set_loading_state("Scanning DLP policies...") + self.on_status_change() + + threading.Thread(target=self._execute_worker, args=(tenant, client_id, client_secret), daemon=True).start() + + def _execute_worker(self, tenant, client_id, client_secret): + if self.semaphore: self.semaphore.acquire() + try: + policies = run_dlp_policies_pipeline( + client_id=client_id, + client_secret=client_secret, + tenant_id=tenant + ) + + # Stream to CSV + if isinstance(policies, dict) and "value" in policies: + policies = policies["value"] + policies_list = policies if isinstance(policies, list) else [policies] + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + if os.path.basename(script_dir) == "security": + script_dir = os.path.dirname(script_dir) + reports_dir = os.path.join(script_dir, "reports", f"{tenant}_{client_id}") + os.makedirs(reports_dir, exist_ok=True) + self.csv_path = os.path.join(reports_dir, "dlp_policies.csv") + + with open(self.csv_path, 'w', encoding='utf-8', newline='') as f: + if policies_list: + writer = csv.DictWriter(f, fieldnames=policies_list[0].keys()) + writer.writeheader() + writer.writerows(policies_list) + + self.status = "success" + self.after(0, self._render_success) + except Exception as e: + usage_logger.error(f"DLP policies fetch error: {e}", exc_info=True) + self.status = "error" + self.after(0, self._render_error, str(e)) + finally: + if self.semaphore: self.semaphore.release() + self.after(0, self.on_status_change) + + def _render_success(self): + self.btn_reload.configure(state="normal") + self.btn_export.configure(state="normal") + self.state_frame.pack_forget() + self.grid_frame.pack(fill="x") + + self._update_grid() + + def _render_error(self, err_msg): + self.btn_reload.configure(state="normal") + self.grid_frame.pack_forget() + for w in self.state_frame.winfo_children(): w.destroy() + self.state_frame.pack(fill="x", expand=True) + display_msg = err_msg + if "401" in err_msg or "403" in err_msg or "permission" in err_msg.lower(): + display_msg = "DLP policies telemetry permission required.\nPlease grant required application permissions to your App Registration in Microsoft Entra ID." + ctk.CTkLabel(self.state_frame, text=f"✖ {display_msg}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center", wraplength=700).pack(pady=(15, 5)) + ctk.CTkButton(self.state_frame, text="Try Again", command=self.trigger_fetch_individual, width=100, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 15)) + + def _load_page(self): + if not self.csv_path or not os.path.exists(self.csv_path): return [], 0 + items = [] + total_count = 0 + try: + with open(self.csv_path, 'r', encoding='utf-8') as f: + reader = csv.DictReader(f) + all_rows = list(reader) + total_count = len(all_rows) + start_idx = self.page * self.ITEMS_PER_PAGE + end_idx = start_idx + self.ITEMS_PER_PAGE + items = all_rows[start_idx:end_idx] + except Exception as e: + usage_logger.error(f"Error reading DLP CSV: {e}") + return items, total_count + + def _update_grid(self): + for w in self.grid_frame.winfo_children(): + info = w.grid_info() + if "row" in info and int(info["row"]) > 0: w.destroy() + + page_data, total_count = self._load_page() + total_pages = (total_count - 1) // self.ITEMS_PER_PAGE + 1 if total_count > 0 else 1 + + for offset, row_item in enumerate(page_data, start=1): + r_idx = offset + bg_style = COLOR_SURFACE if r_idx % 2 == 0 else COLOR_SURFACE_VARIANT + + name = row_item.get("Name", "N/A") + mode = row_item.get("Mode", "N/A") + workloads = row_item.get("Workload", "N/A") + + en_val = str(row_item.get("Enabled", "")).lower() + enabled = "🟢 Enabled" if en_val in ("true", "1", "yes") else "🔴 Disabled" + + actions = row_item.get("Actions", "None") + created_by = row_item.get("CreatedBy", "N/A") + + vals = [name, mode, workloads, enabled, actions, created_by] + for col_idx, val in enumerate(vals): + cell = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + cell.grid(row=r_idx, column=col_idx, sticky="nsew", padx=0, pady=(0, 1)) + lbl = ctk.CTkLabel(cell, text=str(val), font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN, justify="left", wraplength=180) + lbl.pack(padx=10, pady=12, anchor="nw") + + # Pagination controls row + control_frame = ctk.CTkFrame(self.grid_frame, fg_color="transparent") + control_frame.grid(row=self.ITEMS_PER_PAGE + 1, column=0, columnspan=6, pady=0, sticky="ew") + + center_container = ctk.CTkFrame(control_frame, fg_color="transparent") + center_container.pack(pady=(5, 10)) + + prev_state = "normal" if self.page > 0 else "disabled" + ctk.CTkButton( + center_container, text="◀ Prev", width=70, height=22, corner_radius=6, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, state=prev_state, + command=lambda: self._change_page(-1) + ).pack(side="left", padx=5) + + ctk.CTkLabel(center_container, text=f"Page {self.page + 1} of {total_pages}", font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_SUB).pack(side="left", padx=15) + + next_state = "normal" if self.page < total_pages - 1 else "disabled" + ctk.CTkButton( + center_container, text="Next ▶", width=70, height=22, corner_radius=6, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, state=next_state, + command=lambda: self._change_page(1) + ).pack(side="left", padx=5) + + def _change_page(self, delta): + self.page += delta + self._update_grid() + + def _export(self): + if not self.csv_path or not os.path.exists(self.csv_path): + messagebox.showinfo("No Data", "There is no DLP policy data to export.", parent=self) + return + + ts = datetime.now().strftime("%Y%m%d_%H%M%S") + f = filedialog.asksaveasfilename( + initialfile=f"dlp_policies_{ts}.csv", + defaultextension=".csv", + filetypes=[("CSV Files", "*.csv")], + parent=self + ) + if not f: return + try: + shutil.copyfile(self.csv_path, f) + messagebox.showinfo("Export Successful", f"DLP policies exported to:\n{f}", parent=self) + except Exception as e: + messagebox.showerror("Export Failed", f"Error: {e}", parent=self) + + @property + def last_data(self): + if not self.csv_path or not os.path.exists(self.csv_path): return [] + try: + with open(self.csv_path, 'r', encoding='utf-8') as f: + reader = csv.DictReader(f) + return list(reader) + except Exception: + return [] + + def cancel(self): + self.is_cancelled = True + self.status = None diff --git a/telemetry/security/retention_policies.py b/telemetry/security/retention_policies.py new file mode 100644 index 00000000..f20ba1fa --- /dev/null +++ b/telemetry/security/retention_policies.py @@ -0,0 +1,348 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""UI component for Retention Compliance Policies.""" + +import os +import time +import logging +import threading +import csv +import shutil +from datetime import datetime +import customtkinter as ctk +from tkinter import filedialog, messagebox + +from core.graph.security.retention_policies import run_retention_policies_pipeline +from telemetry.styles import * + +usage_logger = logging.getLogger("M365TelemetryAsyncLogger.RetentionUI") + +class RetentionPoliciesSubFrame(ctk.CTkFrame): + """Sub-frame for Retention Compliance Policies.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, semaphore=None, **kwargs): + super().__init__(master, fg_color="transparent", **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.semaphore = semaphore + self.status = None + self.page = 0 + self.ITEMS_PER_PAGE = 5 + self.csv_path = None + self.is_cancelled = False + + self.build_ui() + + def build_ui(self): + self.header_frame = ctk.CTkFrame(self, fg_color="transparent") + self.header_frame.pack(fill="x", pady=(0, 10)) + + ctk.CTkLabel(self.header_frame, text="Retention Compliance Policies", font=FONT_BODY_BOLD, text_color=COLOR_TEXT_MAIN).pack(side="left") + + self.lbl_link = ctk.CTkLabel( + self.header_frame, text="Open Purview Retention Policy Portal ↗", + font=FONT_BODY_BOLD, text_color=COLOR_PRIMARY, cursor="hand2" + ) + self.lbl_link.pack(side="left", anchor="w", padx=(15, 0)) + self.lbl_link.bind("", lambda e: __import__("webbrowser").open("https://purview.microsoft.com/datalifecyclemanagement/retention")) + self.lbl_link.bind("", lambda e: self.lbl_link.configure(text_color=COLOR_PRIMARY_HOVER)) + self.lbl_link.bind("", lambda e: self.lbl_link.configure(text_color=COLOR_PRIMARY)) + + self.btn_reload = ctk.CTkButton( + self.header_frame, state="disabled", text="↻ Reload", width=80, height=24, + font=ctk.CTkFont(family="Segoe UI", size=12), + fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color="#2563EB", hover_color="#DBEAFE", + command=self.trigger_fetch_individual + ) + self.btn_reload.pack(side="right", padx=(10, 0)) + + self.btn_export = ctk.CTkButton( + self.header_frame, text="Export Retention Policies", width=180, height=32, corner_radius=16, + font=FONT_BODY_BOLD, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, + command=self._export, state="disabled" + ) + self.btn_export.pack(side="right") + + self.state_frame = ctk.CTkFrame(self, fg_color="transparent") + + self.grid_frame = ctk.CTkFrame(self, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + + self.grid_frame.grid_columnconfigure(0, weight=3) + self.grid_frame.grid_columnconfigure(1, weight=3) + self.grid_frame.grid_columnconfigure(2, weight=2) + self.grid_frame.grid_columnconfigure(3, weight=1) + self.grid_frame.grid_columnconfigure(4, weight=1) + + headers = ["Policy Name", "Workloads", "Duration", "Distribution", "Status"] + for col_idx, head_text in enumerate(headers): + cell = ctk.CTkFrame(self.grid_frame, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=1, pady=1) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + self.reset_view() + + def reset_view(self): + self.status = None + self.is_cancelled = False + self.csv_path = None + self.page = 0 + self.state_frame.pack_forget() + self.grid_frame.pack_forget() + self.btn_export.configure(state="disabled") + for w in self.state_frame.winfo_children(): w.destroy() + for w in self.grid_frame.winfo_children(): + info = w.grid_info() + if "row" in info and int(info["row"]) > 0: w.destroy() + + def _set_loading_state(self, msg): + for w in self.state_frame.winfo_children(): w.destroy() + self.state_frame.pack(fill="x", expand=True) + ctk.CTkLabel(self.state_frame, text=f"⏳ {msg}", text_color=COLOR_TEXT_SUB, font=FONT_BODY_MEDIUM).pack(pady=(15, 5)) + pb = ctk.CTkProgressBar(self.state_frame, mode="indeterminate", width=200, fg_color=COLOR_SURFACE_VARIANT, progress_color=COLOR_PRIMARY) + pb.pack(pady=(0, 15)) + pb.start() + + def trigger_fetch_individual(self): + tenant, clients, secrets = self.get_credentials() + if tenant: + self.trigger_fetch(tenant, clients[0], secrets[0]) + + def trigger_fetch(self, tenant, client_id, client_secret): + self.status = "loading" + self.is_cancelled = False + self.page = 0 + self.btn_reload.configure(state="disabled") + self.btn_export.configure(state="disabled") + self.grid_frame.pack_forget() + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + if os.path.basename(script_dir) == "security": + script_dir = os.path.dirname(script_dir) + reports_dir = os.path.join(script_dir, "reports", f"{tenant}_{client_id}") + self.csv_path = os.path.join(reports_dir, "retention_policies.csv") + + self._set_loading_state("Scanning Retention Compliance policies...") + self.on_status_change() + + threading.Thread(target=self._execute_worker, args=(tenant, client_id, client_secret), daemon=True).start() + + def _execute_worker(self, tenant, client_id, client_secret): + if self.semaphore: self.semaphore.acquire() + try: + policies = run_retention_policies_pipeline( + client_id=client_id, + client_secret=client_secret, + tenant_id=tenant + ) + + # Stream to CSV + if isinstance(policies, dict) and "value" in policies: + policies = policies["value"] + policies_list = policies if isinstance(policies, list) else [policies] + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + if os.path.basename(script_dir) == "security": + script_dir = os.path.dirname(script_dir) + reports_dir = os.path.join(script_dir, "reports", f"{tenant}_{client_id}") + os.makedirs(reports_dir, exist_ok=True) + self.csv_path = os.path.join(reports_dir, "retention_policies.csv") + + with open(self.csv_path, 'w', encoding='utf-8', newline='') as f: + if policies_list: + writer = csv.DictWriter(f, fieldnames=policies_list[0].keys()) + writer.writeheader() + writer.writerows(policies_list) + + self.status = "success" + self.after(0, self._render_success) + except Exception as e: + usage_logger.error(f"Retention fetch error: {e}", exc_info=True) + self.status = "error" + self.after(0, self._render_error, str(e)) + finally: + if self.semaphore: self.semaphore.release() + self.after(0, self.on_status_change) + + def _render_success(self): + self.btn_reload.configure(state="normal") + self.btn_export.configure(state="normal") + self.state_frame.pack_forget() + self.grid_frame.pack(fill="x") + + self._update_grid() + + def _render_error(self, err_msg): + self.btn_reload.configure(state="normal") + self.grid_frame.pack_forget() + for w in self.state_frame.winfo_children(): w.destroy() + self.state_frame.pack(fill="x", expand=True) + display_msg = err_msg + if "pwsh" in err_msg.lower() or "is not installed" in err_msg.lower(): + display_msg = "PowerShell Core ('pwsh') is not installed or configured on this machine." + elif "exchangeonlinemanagement" in err_msg.lower(): + display_msg = "ExchangeOnlineManagement PowerShell module is missing.\nPlease run: Install-Module -Name ExchangeOnlineManagement -Scope CurrentUser" + ctk.CTkLabel(self.state_frame, text=f"✖ {display_msg}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center", wraplength=700).pack(pady=(15, 5)) + ctk.CTkButton(self.state_frame, text="Try Again", command=self.trigger_fetch_individual, width=100, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 15)) + + def _load_page(self): + if not self.csv_path or not os.path.exists(self.csv_path): return [], 0 + items = [] + total_count = 0 + try: + with open(self.csv_path, 'r', encoding='utf-8') as f: + reader = csv.DictReader(f) + all_rows = list(reader) + total_count = len(all_rows) + start_idx = self.page * self.ITEMS_PER_PAGE + end_idx = start_idx + self.ITEMS_PER_PAGE + items = all_rows[start_idx:end_idx] + except Exception as e: + usage_logger.error(f"Error reading Retention policies CSV: {e}") + return items, total_count + + def _update_grid(self): + for w in self.grid_frame.winfo_children(): + info = w.grid_info() + if "row" in info and int(info["row"]) > 0: w.destroy() + + page_data, total_count = self._load_page() + total_pages = (total_count - 1) // self.ITEMS_PER_PAGE + 1 if total_count > 0 else 1 + + for offset, policy in enumerate(page_data, start=1): + r_idx = offset + bg_style = "transparent" if r_idx % 2 != 0 else COLOR_SURFACE_VARIANT + + name = policy.get("Name", "N/A") + comment = policy.get("Comment", "") + workload = policy.get("Workload", "N/A") + duration_val = str(policy.get("Duration", "N/A")) + trigger_val = policy.get("RetentionTrigger", "N/A") + dist_status = policy.get("DistributionStatus", "Success") + + duration_str = duration_val + if duration_val.lower() == "unlimited": + duration_str = "Keep Forever" + elif duration_val.isdigit(): + days = int(duration_val) + if days >= 365: + years = days / 365.0 + duration_str = f"{int(years)} Years ({days} days)" if years.is_integer() else f"{years:.1f} Years ({days} days)" + else: + duration_str = f"{days} days" + + if trigger_val and trigger_val != "N/A": + trigger_map = {"DateCreated": "created date", "DateModified": "last modified date", "DateLabeled": "labeled date"} + duration_str += f"\n(from {trigger_map.get(trigger_val, trigger_val)})" + + enabled_val = policy.get("Enabled", True) + is_enabled = enabled_val.lower() in ("true", "1", "yes") if isinstance(enabled_val, str) else bool(enabled_val) + status = "🟢 Enabled" if is_enabled else "🔴 Disabled" + + c0 = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + c0.grid(row=r_idx, column=0, sticky="nsew", padx=1, pady=1) + has_comment = bool(comment and comment != name) + lbl_name = ctk.CTkLabel(c0, text=name, font=FONT_BODY_BOLD, text_color=COLOR_TEXT_MAIN) + lbl_name.pack(padx=10, pady=(6, 2) if has_comment else 6, anchor="w") + if has_comment: + lbl_comment = ctk.CTkLabel(c0, text=comment, font=FONT_BODY_SMALL, text_color=COLOR_TEXT_SUB) + lbl_comment.pack(padx=10, pady=(0, 6), anchor="w") + c0.bind("", lambda e, l1=lbl_name, l2=lbl_comment: (l1.configure(wraplength=e.width - 20), l2.configure(wraplength=e.width - 20))) + else: + c0.bind("", lambda e, l=lbl_name: l.configure(wraplength=e.width - 20)) + + c1 = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + c1.grid(row=r_idx, column=1, sticky="nsew", padx=1, pady=1) + lbl_workload = ctk.CTkLabel(c1, text=workload, font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN) + lbl_workload.pack(padx=10, pady=6, anchor="w") + c1.bind("", lambda e, l=lbl_workload: l.configure(wraplength=e.width - 20)) + + c2 = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + c2.grid(row=r_idx, column=2, sticky="nsew", padx=1, pady=1) + lbl_duration = ctk.CTkLabel(c2, text=duration_str, font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN, justify="left") + lbl_duration.pack(padx=10, pady=6, anchor="w") + c2.bind("", lambda e, l=lbl_duration: l.configure(wraplength=e.width - 20)) + + c3 = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + c3.grid(row=r_idx, column=3, sticky="nsew", padx=1, pady=1) + ctk.CTkLabel(c3, text=dist_status, font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=6, anchor="w") + + c4 = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + c4.grid(row=r_idx, column=4, sticky="nsew", padx=1, pady=1) + ctk.CTkLabel(c4, text=status, font=FONT_BODY_BOLD, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=6, anchor="w") + + # Pagination controls row + control_frame = ctk.CTkFrame(self.grid_frame, fg_color="transparent") + control_frame.grid(row=self.ITEMS_PER_PAGE + 1, column=0, columnspan=5, pady=0, sticky="ew") + + center_container = ctk.CTkFrame(control_frame, fg_color="transparent") + center_container.pack(pady=(5, 10)) + + prev_state = "normal" if self.page > 0 else "disabled" + ctk.CTkButton( + center_container, text="◀ Prev", width=70, height=22, corner_radius=6, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, state=prev_state, + command=lambda: self._change_page(-1) + ).pack(side="left", padx=5) + + ctk.CTkLabel(center_container, text=f"Page {self.page + 1} of {total_pages}", font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_SUB).pack(side="left", padx=15) + + next_state = "normal" if self.page < total_pages - 1 else "disabled" + ctk.CTkButton( + center_container, text="Next ▶", width=70, height=22, corner_radius=6, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, state=next_state, + command=lambda: self._change_page(1) + ).pack(side="left", padx=5) + + def _change_page(self, delta): + self.page += delta + self._update_grid() + + def _export(self): + if not self.csv_path or not os.path.exists(self.csv_path): + messagebox.showinfo("No Data", "There is no retention policies data to export. Please run a scan first.", parent=self) + return + + ts = datetime.now().strftime("%Y%m%d_%H%M%S") + f = filedialog.asksaveasfilename( + initialfile=f"retention_policies_{ts}.csv", + defaultextension=".csv", + filetypes=[("CSV Files", "*.csv"), ("All Files", "*.*")], + parent=self + ) + if not f: return + try: + shutil.copyfile(self.csv_path, f) + messagebox.showinfo("Export Successful", f"Retention policies exported successfully to:\n{f}", parent=self) + except Exception as e: + messagebox.showerror("Export Failed", f"Failed to export CSV: {e}", parent=self) + + @property + def last_data(self): + if not self.csv_path or not os.path.exists(self.csv_path): return [] + try: + with open(self.csv_path, 'r', encoding='utf-8') as f: + reader = csv.DictReader(f) + return list(reader) + except Exception: + return [] + + def cancel(self): + self.is_cancelled = True + self.status = None diff --git a/telemetry/security/sensitive_info_types.py b/telemetry/security/sensitive_info_types.py new file mode 100644 index 00000000..38777323 --- /dev/null +++ b/telemetry/security/sensitive_info_types.py @@ -0,0 +1,303 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""UI component for Sensitive Information Types (SIT) telemetry.""" + +import os +import time +import logging +import threading +import csv +import shutil +from datetime import datetime +import customtkinter as ctk +from tkinter import filedialog, messagebox + +from core.graph.security.sensitive_info_types import run_sensitive_info_types_pipeline +from telemetry.styles import * + +usage_logger = logging.getLogger("M365TelemetryAsyncLogger.SITUI") + +class SensitiveInfoTypesSubFrame(ctk.CTkFrame): + """Sub-frame for Sensitive Information Types.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, semaphore=None, **kwargs): + super().__init__(master, fg_color="transparent", **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.semaphore = semaphore + self.status = None + self.page = 0 + self.ITEMS_PER_PAGE = 5 + self.csv_path = None + self.is_cancelled = False + + self.build_ui() + + def build_ui(self): + self.header_frame = ctk.CTkFrame(self, fg_color="transparent") + self.header_frame.pack(fill="x", pady=(0, 10)) + + ctk.CTkLabel(self.header_frame, text="Sensitive Information Types (SIT)", font=FONT_BODY_BOLD, text_color=COLOR_TEXT_MAIN).pack(side="left") + + self.btn_reload = ctk.CTkButton( + self.header_frame, state="disabled", text="↻ Reload", width=80, height=24, + font=ctk.CTkFont(family="Segoe UI", size=12), + fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color="#2563EB", hover_color="#DBEAFE", + command=self.trigger_fetch_individual + ) + self.btn_reload.pack(side="right", padx=(10, 0)) + + self.btn_export = ctk.CTkButton( + self.header_frame, text="Export SIT Data", width=180, height=32, corner_radius=16, + font=FONT_BODY_BOLD, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, + command=self._export, state="disabled" + ) + self.btn_export.pack(side="right") + + self.state_frame = ctk.CTkFrame(self, fg_color="transparent") + + self.grid_frame = ctk.CTkFrame(self, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + + self.grid_frame.grid_columnconfigure(0, weight=1) + self.grid_frame.grid_columnconfigure(1, weight=1) + self.grid_frame.grid_columnconfigure(2, weight=1) + self.grid_frame.grid_columnconfigure(3, weight=3) + + headers = ["SIT Name", "Type", "Confidence", "Description"] + for col_idx, head_text in enumerate(headers): + cell = ctk.CTkFrame(self.grid_frame, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + self.reset_view() + + def reset_view(self): + self.status = None + self.is_cancelled = False + self.csv_path = None + self.page = 0 + self.state_frame.pack_forget() + self.grid_frame.pack_forget() + self.btn_export.configure(state="disabled") + for w in self.state_frame.winfo_children(): w.destroy() + for w in self.grid_frame.winfo_children(): + info = w.grid_info() + if "row" in info and int(info["row"]) > 0: w.destroy() + + def _set_loading_state(self, msg): + for w in self.state_frame.winfo_children(): w.destroy() + self.state_frame.pack(fill="x", expand=True) + ctk.CTkLabel(self.state_frame, text=f"⏳ {msg}", text_color=COLOR_TEXT_SUB, font=FONT_BODY_MEDIUM).pack(pady=(15, 5)) + pb = ctk.CTkProgressBar(self.state_frame, mode="indeterminate", width=200, fg_color=COLOR_SURFACE_VARIANT, progress_color=COLOR_PRIMARY) + pb.pack(pady=(0, 15)) + pb.start() + + def trigger_fetch_individual(self): + tenant, clients, secrets = self.get_credentials() + if tenant: + self.trigger_fetch(tenant, clients[0], secrets[0]) + + def trigger_fetch(self, tenant, client_id, client_secret): + self.status = "loading" + self.is_cancelled = False + self.page = 0 + self.btn_reload.configure(state="disabled") + self.btn_export.configure(state="disabled") + self.grid_frame.pack_forget() + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + if os.path.basename(script_dir) == "security": + script_dir = os.path.dirname(script_dir) + reports_dir = os.path.join(script_dir, "reports", f"{tenant}_{client_id}") + self.csv_path = os.path.join(reports_dir, "sensitive_info_types.csv") + + self._set_loading_state("Scanning Sensitive Info Types...") + self.on_status_change() + + threading.Thread(target=self._execute_worker, args=(tenant, client_id, client_secret), daemon=True).start() + + def _execute_worker(self, tenant, client_id, client_secret): + if self.semaphore: self.semaphore.acquire() + try: + sit_data = run_sensitive_info_types_pipeline( + client_id=client_id, + client_secret=client_secret, + tenant_id=tenant + ) + + # Stream to CSV + if isinstance(sit_data, dict) and "value" in sit_data: + sit_data = sit_data["value"] + sit_list = sit_data if isinstance(sit_data, list) else [sit_data] + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + if os.path.basename(script_dir) == "security": + script_dir = os.path.dirname(script_dir) + reports_dir = os.path.join(script_dir, "reports", f"{tenant}_{client_id}") + os.makedirs(reports_dir, exist_ok=True) + self.csv_path = os.path.join(reports_dir, "sensitive_info_types.csv") + + with open(self.csv_path, 'w', encoding='utf-8', newline='') as f: + if sit_list: + writer = csv.DictWriter(f, fieldnames=sit_list[0].keys()) + writer.writeheader() + writer.writerows(sit_list) + + self.status = "success" + self.after(0, self._render_success) + except Exception as e: + usage_logger.error(f"SIT fetch error: {e}", exc_info=True) + self.status = "error" + self.after(0, self._render_error, str(e)) + finally: + if self.semaphore: self.semaphore.release() + self.after(0, self.on_status_change) + + def _render_success(self): + self.btn_reload.configure(state="normal") + self.btn_export.configure(state="normal") + self.state_frame.pack_forget() + self.grid_frame.pack(fill="x") + + self._update_grid() + + def _render_error(self, err_msg): + self.btn_reload.configure(state="normal") + self.grid_frame.pack_forget() + for w in self.state_frame.winfo_children(): w.destroy() + self.state_frame.pack(fill="x", expand=True) + display_msg = err_msg + if "401" in err_msg or "403" in err_msg or "permission" in err_msg.lower(): + display_msg = "SIT telemetry permission required.\nPlease grant required application permissions to your App Registration in Microsoft Entra ID." + ctk.CTkLabel(self.state_frame, text=f"✖ {display_msg}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center", wraplength=700).pack(pady=(15, 5)) + ctk.CTkButton(self.state_frame, text="Try Again", command=self.trigger_fetch_individual, width=100, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 15)) + + def _load_page(self): + if not self.csv_path or not os.path.exists(self.csv_path): return [], 0 + items = [] + total_count = 0 + try: + with open(self.csv_path, 'r', encoding='utf-8') as f: + reader = csv.DictReader(f) + all_rows = list(reader) + total_count = len(all_rows) + start_idx = self.page * self.ITEMS_PER_PAGE + end_idx = start_idx + self.ITEMS_PER_PAGE + items = all_rows[start_idx:end_idx] + except Exception as e: + usage_logger.error(f"Error reading SIT CSV: {e}") + return items, total_count + + def _update_grid(self): + for w in self.grid_frame.winfo_children(): + info = w.grid_info() + if "row" in info and int(info["row"]) > 0: w.destroy() + + page_data, total_count = self._load_page() + total_pages = (total_count - 1) // self.ITEMS_PER_PAGE + 1 if total_count > 0 else 1 + + for offset, sit in enumerate(page_data, start=1): + r_idx = offset + bg_style = COLOR_SURFACE if r_idx % 2 != 0 else COLOR_SURFACE_VARIANT + + name = sit.get("Name", "N/A") + sit_type = sit.get("Type", "N/A") + conf = str(sit.get("RecommendedConfidence", "N/A")) + desc = sit.get("Description", "N/A") + + c0 = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + c0.grid(row=r_idx, column=0, sticky="nsew", padx=1, pady=1) + lbl_name = ctk.CTkLabel(c0, text=name, font=FONT_BODY_BOLD, text_color=COLOR_TEXT_MAIN) + lbl_name.pack(padx=10, pady=6, anchor="w") + c0.bind("", lambda e, l=lbl_name: l.configure(wraplength=e.width - 20)) + + c1 = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + c1.grid(row=r_idx, column=1, sticky="nsew", padx=1, pady=1) + ctk.CTkLabel(c1, text=sit_type, font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=6, anchor="w") + + c2 = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + c2.grid(row=r_idx, column=2, sticky="nsew", padx=1, pady=1) + ctk.CTkLabel(c2, text=f"{conf}%" if conf.isdigit() else conf, font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=6, anchor="w") + + c3 = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + c3.grid(row=r_idx, column=3, sticky="nsew", padx=1, pady=1) + lbl_desc = ctk.CTkLabel(c3, text=desc, font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_SUB, justify="left") + lbl_desc.pack(padx=10, pady=6, anchor="w") + c3.bind("", lambda e, l=lbl_desc: l.configure(wraplength=e.width - 20)) + + # Pagination controls row + control_frame = ctk.CTkFrame(self.grid_frame, fg_color="transparent") + control_frame.grid(row=self.ITEMS_PER_PAGE + 1, column=0, columnspan=4, pady=0, sticky="ew") + + center_container = ctk.CTkFrame(control_frame, fg_color="transparent") + center_container.pack(pady=(5, 10)) + + prev_state = "normal" if self.page > 0 else "disabled" + ctk.CTkButton( + center_container, text="◀ Prev", width=70, height=22, corner_radius=6, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, state=prev_state, + command=lambda: self._change_page(-1) + ).pack(side="left", padx=5) + + ctk.CTkLabel(center_container, text=f"Page {self.page + 1} of {total_pages}", font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_SUB).pack(side="left", padx=15) + + next_state = "normal" if self.page < total_pages - 1 else "disabled" + ctk.CTkButton( + center_container, text="Next ▶", width=70, height=22, corner_radius=6, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, state=next_state, + command=lambda: self._change_page(1) + ).pack(side="left", padx=5) + + def _change_page(self, delta): + self.page += delta + self._update_grid() + + def _export(self): + if not self.csv_path or not os.path.exists(self.csv_path): + messagebox.showinfo("No Data", "There is no SIT data to export.", parent=self) + return + + ts = datetime.now().strftime("%Y%m%d_%H%M%S") + f = filedialog.asksaveasfilename( + initialfile=f"sensitive_info_types_{ts}.csv", + defaultextension=".csv", + filetypes=[("CSV Files", "*.csv")], + parent=self + ) + if not f: return + try: + shutil.copyfile(self.csv_path, f) + messagebox.showinfo("Export Successful", f"SIT exported to:\n{f}", parent=self) + except Exception as e: + messagebox.showerror("Export Failed", f"Error: {e}", parent=self) + + @property + def last_data(self): + if not self.csv_path or not os.path.exists(self.csv_path): return [] + try: + with open(self.csv_path, 'r', encoding='utf-8') as f: + reader = csv.DictReader(f) + return list(reader) + except Exception: + return [] + + def cancel(self): + self.is_cancelled = True + self.status = None diff --git a/telemetry/security/sensitivity_labels.py b/telemetry/security/sensitivity_labels.py new file mode 100644 index 00000000..0709d425 --- /dev/null +++ b/telemetry/security/sensitivity_labels.py @@ -0,0 +1,319 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""UI component for Sensitivity Labels telemetry.""" + +import os +import time +import logging +import threading +import csv +import shutil +from datetime import datetime +import customtkinter as ctk +from tkinter import filedialog, messagebox + +from core.graph.security.sensitivity_labels import run_sensitivity_labels_pipeline +from telemetry.styles import * + +usage_logger = logging.getLogger("M365TelemetryAsyncLogger.SensitivityLabelsUI") + +class SensitivityLabelsSubFrame(ctk.CTkFrame): + """Sub-frame for Sensitivity Labels telemetry.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, semaphore=None, **kwargs): + super().__init__(master, fg_color="transparent", **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.semaphore = semaphore + self.status = None + self.page = 0 + self.ITEMS_PER_PAGE = 5 + self.csv_path = None + self.is_cancelled = False + + self.build_ui() + + def build_ui(self): + self.header_frame = ctk.CTkFrame(self, fg_color="transparent") + self.header_frame.pack(fill="x", pady=(0, 10)) + + ctk.CTkLabel(self.header_frame, text="Sensitivity Labels", font=FONT_BODY_BOLD, text_color=COLOR_TEXT_MAIN).pack(side="left") + + self.lbl_link = ctk.CTkLabel( + self.header_frame, text="Open Purview Sensitivity Label Portal ↗", + font=FONT_BODY_BOLD, text_color=COLOR_PRIMARY, cursor="hand2" + ) + self.lbl_link.pack(side="left", anchor="w", padx=(15, 0)) + self.lbl_link.bind("", lambda e: __import__("webbrowser").open("https://purview.microsoft.com/informationprotection/informationprotectionlabels/sensitivitylabels")) + self.lbl_link.bind("", lambda e: self.lbl_link.configure(text_color=COLOR_PRIMARY_HOVER)) + self.lbl_link.bind("", lambda e: self.lbl_link.configure(text_color=COLOR_PRIMARY)) + + self.btn_reload = ctk.CTkButton( + self.header_frame, state="disabled", text="↻ Reload", width=80, height=24, + font=ctk.CTkFont(family="Segoe UI", size=12), + fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color="#2563EB", hover_color="#DBEAFE", + command=self.trigger_fetch_individual + ) + self.btn_reload.pack(side="right", padx=(10, 0)) + + self.btn_export = ctk.CTkButton( + self.header_frame, text="Export Sensitivity Labels", width=180, height=32, corner_radius=16, + font=FONT_BODY_BOLD, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, + command=self._export, state="disabled" + ) + self.btn_export.pack(side="right") + + self.state_frame = ctk.CTkFrame(self, fg_color="transparent") + + self.grid_frame = ctk.CTkFrame(self, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + + self.grid_frame.grid_columnconfigure(0, weight=2) + self.grid_frame.grid_columnconfigure(1, weight=3) + self.grid_frame.grid_columnconfigure(2, weight=1) + self.grid_frame.grid_columnconfigure(3, weight=1) + self.grid_frame.grid_columnconfigure(4, weight=1) + self.grid_frame.grid_columnconfigure(5, weight=2) + self.grid_frame.grid_columnconfigure(6, weight=1) + + headers = ["Sensitivity Label", "Description", "Protection", "Mode", "Priority", "Applicable Targets", "Status"] + for col_idx, head_text in enumerate(headers): + cell = ctk.CTkFrame(self.grid_frame, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + self.reset_view() + + def reset_view(self): + self.status = None + self.is_cancelled = False + self.csv_path = None + self.page = 0 + self.state_frame.pack_forget() + self.grid_frame.pack_forget() + self.btn_export.configure(state="disabled") + for w in self.state_frame.winfo_children(): w.destroy() + for w in self.grid_frame.winfo_children(): + info = w.grid_info() + if "row" in info and int(info["row"]) > 0: w.destroy() + + def _set_loading_state(self, msg): + for w in self.state_frame.winfo_children(): w.destroy() + self.state_frame.pack(fill="x", expand=True) + ctk.CTkLabel(self.state_frame, text=f"⏳ {msg}", text_color=COLOR_TEXT_SUB, font=FONT_BODY_MEDIUM).pack(pady=(15, 5)) + pb = ctk.CTkProgressBar(self.state_frame, mode="indeterminate", width=200, fg_color=COLOR_SURFACE_VARIANT, progress_color=COLOR_PRIMARY) + pb.pack(pady=(0, 15)) + pb.start() + + def trigger_fetch_individual(self): + tenant, clients, secrets = self.get_credentials() + if tenant: + self.trigger_fetch(tenant, clients[0], secrets[0]) + + def trigger_fetch(self, tenant, client_id, client_secret): + self.status = "loading" + self.is_cancelled = False + self.page = 0 + self.btn_reload.configure(state="disabled") + self.btn_export.configure(state="disabled") + self.grid_frame.pack_forget() + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + if os.path.basename(script_dir) == "security": + script_dir = os.path.dirname(script_dir) + reports_dir = os.path.join(script_dir, "reports", f"{tenant}_{client_id}") + self.csv_path = os.path.join(reports_dir, "sensitivity_labels.csv") + + self._set_loading_state("Scanning Sensitivity Labels...") + self.on_status_change() + + threading.Thread(target=self._execute_worker, args=(tenant, client_id, client_secret), daemon=True).start() + + def _execute_worker(self, tenant, client_id, client_secret): + if self.semaphore: self.semaphore.acquire() + try: + run_sensitivity_labels_pipeline( + client_id=client_id, + client_secret=client_secret, + tenant_id=tenant, + csv_path=self.csv_path, + is_cancelled_callback=lambda: self.is_cancelled + ) + self.status = "success" + self.after(0, self._render_success) + except Exception as e: + usage_logger.error(f"Sensitivity labels fetch error: {e}", exc_info=True) + self.status = "error" + self.after(0, self._render_error, str(e)) + finally: + if self.semaphore: self.semaphore.release() + self.after(0, self.on_status_change) + + def _render_success(self): + self.btn_reload.configure(state="normal") + self.btn_export.configure(state="normal") + self.state_frame.pack_forget() + self.grid_frame.pack(fill="x") + + self._update_grid() + + def _render_error(self, err_msg): + self.btn_reload.configure(state="normal") + self.grid_frame.pack_forget() + for w in self.state_frame.winfo_children(): w.destroy() + self.state_frame.pack(fill="x", expand=True) + display_msg = err_msg + if "401" in err_msg or "403" in err_msg or "unauthorized" in err_msg.lower() or "forbidden" in err_msg.lower(): + display_msg = "Information Protection permission required.\nPlease grant the 'SensitivityLabels.Read.All' application permission to your App Registration in Microsoft Entra ID." + ctk.CTkLabel(self.state_frame, text=f"✖ {display_msg}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center", wraplength=700).pack(pady=(15, 5)) + ctk.CTkButton(self.state_frame, text="Try Again", command=self.trigger_fetch_individual, width=100, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 15)) + + def _load_page(self): + if not self.csv_path or not os.path.exists(self.csv_path): return [], 0 + items = [] + total_count = 0 + try: + with open(self.csv_path, 'r', encoding='utf-8') as f: + reader = csv.DictReader(f) + all_rows = list(reader) + total_count = len(all_rows) + start_idx = self.page * self.ITEMS_PER_PAGE + end_idx = start_idx + self.ITEMS_PER_PAGE + items = all_rows[start_idx:end_idx] + except Exception as e: + usage_logger.error(f"Error reading Sensitivity Labels CSV: {e}") + return items, total_count + + def _update_grid(self): + for w in self.grid_frame.winfo_children(): + info = w.grid_info() + if "row" in info and int(info["row"]) > 0: w.destroy() + + page_data, total_count = self._load_page() + total_pages = (total_count - 1) // self.ITEMS_PER_PAGE + 1 if total_count > 0 else 1 + + for offset, row_item in enumerate(page_data, start=1): + r_idx = offset + bg_style = COLOR_SURFACE if r_idx % 2 == 0 else COLOR_SURFACE_VARIANT + + name = row_item.get("name", "N/A") + desc = row_item.get("description", "N/A") + protection = "🛡️ Yes" if str(row_item.get("hasProtection")) == "1" else "🔓 No" + mode = str(row_item.get("applicationMode")).capitalize() + priority = str(row_item.get("priority")) + applicable = ", ".join([x.capitalize() for x in str(row_item.get("applicableTo")).split(",") if x.strip()]) or "N/A" + status = "🟢 Enabled" if str(row_item.get("isEnabled")) == "1" else "🔴 Disabled" + is_sublabel = str(row_item.get("is_sublabel")) == "1" + + name_color = COLOR_TEXT_MAIN if not is_sublabel else COLOR_TEXT_SUB + name_font = FONT_BODY_BOLD if not is_sublabel else FONT_BODY_MEDIUM + + c0 = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + c0.grid(row=r_idx, column=0, sticky="nsew", padx=0, pady=(0, 1)) + lbl_name = ctk.CTkLabel(c0, text=name, font=name_font, text_color=name_color) + lbl_name.pack(padx=10, pady=6, anchor="w") + c0.bind("", lambda e, l=lbl_name: l.configure(wraplength=e.width - 20)) + + c1 = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + c1.grid(row=r_idx, column=1, sticky="nsew", padx=0, pady=(0, 1)) + lbl_desc = ctk.CTkLabel(c1, text=desc, font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN) + lbl_desc.pack(padx=10, pady=6, anchor="w") + c1.bind("", lambda e, l=lbl_desc: l.configure(wraplength=e.width - 20)) + + c2 = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + c2.grid(row=r_idx, column=2, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c2, text=protection, font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=6, anchor="w") + + c3 = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + c3.grid(row=r_idx, column=3, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c3, text=mode, font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=6, anchor="w") + + c4 = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + c4.grid(row=r_idx, column=4, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c4, text=priority, font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=6, anchor="w") + + c5 = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + c5.grid(row=r_idx, column=5, sticky="nsew", padx=0, pady=(0, 1)) + lbl_app = ctk.CTkLabel(c5, text=applicable, font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN) + lbl_app.pack(padx=10, pady=6, anchor="w") + c5.bind("", lambda e, l=lbl_app: l.configure(wraplength=e.width - 20)) + + c6 = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + c6.grid(row=r_idx, column=6, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c6, text=status, font=FONT_BODY_BOLD, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=6, anchor="w") + + # Pagination controls row + control_frame = ctk.CTkFrame(self.grid_frame, fg_color="transparent") + control_frame.grid(row=self.ITEMS_PER_PAGE + 1, column=0, columnspan=7, pady=0, sticky="ew") + + center_container = ctk.CTkFrame(control_frame, fg_color="transparent") + center_container.pack(pady=(5, 10)) + + prev_state = "normal" if self.page > 0 else "disabled" + ctk.CTkButton( + center_container, text="◀ Prev", width=70, height=22, corner_radius=6, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, state=prev_state, + command=lambda: self._change_page(-1) + ).pack(side="left", padx=5) + + ctk.CTkLabel(center_container, text=f"Page {self.page + 1} of {total_pages}", font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_SUB).pack(side="left", padx=15) + + next_state = "normal" if self.page < total_pages - 1 else "disabled" + ctk.CTkButton( + center_container, text="Next ▶", width=70, height=22, corner_radius=6, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, state=next_state, + command=lambda: self._change_page(1) + ).pack(side="left", padx=5) + + def _change_page(self, delta): + self.page += delta + self._update_grid() + + def _export(self): + if not self.csv_path or not os.path.exists(self.csv_path): + messagebox.showinfo("No Data", "There is no sensitivity labels data to export. Please run a scan first.", parent=self) + return + + ts = datetime.now().strftime("%Y%m%d_%H%M%S") + f = filedialog.asksaveasfilename( + initialfile=f"sensitivity_labels_{ts}.csv", + defaultextension=".csv", + filetypes=[("CSV Files", "*.csv"), ("All Files", "*.*")], + parent=self + ) + if not f: return + try: + shutil.copyfile(self.csv_path, f) + messagebox.showinfo("Export Successful", f"Sensitivity labels exported successfully to:\n{f}", parent=self) + except Exception as e: + messagebox.showerror("Export Failed", f"Failed to export CSV: {e}", parent=self) + + @property + def last_data(self): + if not self.csv_path or not os.path.exists(self.csv_path): return [] + try: + with open(self.csv_path, 'r', encoding='utf-8') as f: + reader = csv.DictReader(f) + return list(reader) + except Exception: + return [] + + def cancel(self): + self.is_cancelled = True + self.status = None diff --git a/telemetry/security/service_principals_sso.py b/telemetry/security/service_principals_sso.py new file mode 100644 index 00000000..2be392a0 --- /dev/null +++ b/telemetry/security/service_principals_sso.py @@ -0,0 +1,249 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""UI component for Service Principals SSO Configurations telemetry.""" + +import os +import time +import logging +import threading +import csv +import shutil +from datetime import datetime +import customtkinter as ctk +from tkinter import filedialog, messagebox + +from core.graph.security.service_principals_sso import run_service_principals_sso_pipeline +from telemetry.styles import * + +usage_logger = logging.getLogger("M365TelemetryAsyncLogger.SSOUI") + +class ServicePrincipalsSsoSubFrame(ctk.CTkFrame): + """Sub-frame for Service Principals SSO modes.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, semaphore=None, **kwargs): + super().__init__(master, fg_color="transparent", **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.semaphore = semaphore + self.status = None + self.csv_path = None + self.is_cancelled = False + + self.build_ui() + + def build_ui(self): + self.header_frame = ctk.CTkFrame(self, fg_color="transparent") + self.header_frame.pack(fill="x", pady=(0, 10)) + + ctk.CTkLabel(self.header_frame, text="Service Principals Single Sign-On (SSO) Modes", font=FONT_BODY_BOLD, text_color=COLOR_TEXT_MAIN).pack(side="left") + + self.lbl_link = ctk.CTkLabel( + self.header_frame, text="Open Enterprise Applications ↗", + font=FONT_BODY_BOLD, text_color=COLOR_PRIMARY, cursor="hand2" + ) + self.lbl_link.pack(side="left", anchor="w", padx=(15, 0)) + self.lbl_link.bind("", lambda e: __import__("webbrowser").open("https://entra.microsoft.com/#view/Microsoft_AAD_IAM/StartboardApplicationsMenuBlade/~/AppAppsPreview")) + self.lbl_link.bind("", lambda e: self.lbl_link.configure(text_color=COLOR_PRIMARY_HOVER)) + self.lbl_link.bind("", lambda e: self.lbl_link.configure(text_color=COLOR_PRIMARY)) + + self.btn_reload = ctk.CTkButton( + self.header_frame, state="disabled", text="↻ Reload", width=80, height=24, + font=ctk.CTkFont(family="Segoe UI", size=12), + fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color="#2563EB", hover_color="#DBEAFE", + command=self.trigger_fetch_individual + ) + self.btn_reload.pack(side="right", padx=(10, 0)) + + self.btn_export = ctk.CTkButton( + self.header_frame, text="Export SSO Data", width=150, height=32, corner_radius=16, + font=FONT_BODY_BOLD, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, + command=self._export, state="disabled" + ) + self.btn_export.pack(side="right") + + self.state_frame = ctk.CTkFrame(self, fg_color="transparent") + + self.grid_frame = ctk.CTkFrame(self, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + self.grid_frame.grid_columnconfigure(0, weight=1) + self.grid_frame.grid_columnconfigure(1, weight=1) + + headers = ["SSO Mode", "Application Count"] + for col_idx, head_text in enumerate(headers): + cell = ctk.CTkFrame(self.grid_frame, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=1, column=col_idx, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + self.reset_view() + + def reset_view(self): + self.status = None + self.is_cancelled = False + self.csv_path = None + self.state_frame.pack_forget() + self.grid_frame.pack_forget() + self.btn_export.configure(state="disabled") + for w in self.state_frame.winfo_children(): w.destroy() + for w in self.grid_frame.winfo_children(): + info = w.grid_info() + if "row" in info and int(info["row"]) > 1: w.destroy() + + def _set_loading_state(self, msg): + for w in self.state_frame.winfo_children(): w.destroy() + self.state_frame.pack(fill="x", expand=True) + ctk.CTkLabel(self.state_frame, text=f"⏳ {msg}", text_color=COLOR_TEXT_SUB, font=FONT_BODY_MEDIUM).pack(pady=(15, 5)) + pb = ctk.CTkProgressBar(self.state_frame, mode="indeterminate", width=200, fg_color=COLOR_SURFACE_VARIANT, progress_color=COLOR_PRIMARY) + pb.pack(pady=(0, 15)) + pb.start() + + def trigger_fetch_individual(self): + tenant, clients, secrets = self.get_credentials() + if tenant: + self.trigger_fetch(tenant, clients[0], secrets[0]) + + def trigger_fetch(self, tenant, client_id, client_secret): + self.status = "loading" + self.is_cancelled = False + self.btn_reload.configure(state="disabled") + self.btn_export.configure(state="disabled") + self.grid_frame.pack_forget() + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + if os.path.basename(script_dir) == "security": + script_dir = os.path.dirname(script_dir) + reports_dir = os.path.join(script_dir, "reports", f"{tenant}_{client_id}") + self.csv_path = os.path.join(reports_dir, "service_principals_sso.csv") + + # Ensure directory exists and headers are initialized + os.makedirs(reports_dir, exist_ok=True) + with open(self.csv_path, 'w', encoding='utf-8', newline='') as f: + writer = csv.writer(f) + writer.writerow(["appDisplayName", "preferredSingleSignOnMode"]) + + self._set_loading_state("Scanning Service Principals SSO modes...") + self.on_status_change() + + threading.Thread(target=self._execute_worker, args=(tenant, client_id, client_secret), daemon=True).start() + + def _execute_worker(self, tenant, client_id, client_secret): + if self.semaphore: self.semaphore.acquire() + try: + run_service_principals_sso_pipeline( + client_id=client_id, + client_secret=client_secret, + tenant_id=tenant, + csv_path=self.csv_path, + is_cancelled_callback=lambda: self.is_cancelled + ) + self.status = "success" + self.after(0, self._render_success) + except Exception as e: + usage_logger.error(f"Service Principals SSO fetch error: {e}", exc_info=True) + self.status = "error" + self.after(0, self._render_error, str(e)) + finally: + if self.semaphore: self.semaphore.release() + self.after(0, self.on_status_change) + + def _render_success(self): + self.btn_reload.configure(state="normal") + self.btn_export.configure(state="normal") + self.state_frame.pack_forget() + self.grid_frame.pack(fill="x") + + self._update_grid() + + def _render_error(self, err_msg): + self.btn_reload.configure(state="normal") + self.grid_frame.pack_forget() + for w in self.state_frame.winfo_children(): w.destroy() + self.state_frame.pack(fill="x", expand=True) + display_msg = err_msg + if "401" in err_msg or "403" in err_msg or "permission" in err_msg.lower() or "unauthorized" in err_msg.lower(): + display_msg = "Service Principals telemetry permission required.\nPlease grant the 'Application.Read.All' application permission to your App Registration in Microsoft Entra ID." + ctk.CTkLabel(self.state_frame, text=f"✖ {display_msg}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center", wraplength=700).pack(pady=(15, 5)) + ctk.CTkButton(self.state_frame, text="Try Again", command=self.trigger_fetch_individual, width=100, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 15)) + + def _update_grid(self): + for w in self.grid_frame.winfo_children(): + info = w.grid_info() + if "row" in info and int(info["row"]) > 1: w.destroy() + + saml = oidc = password = none_count = 0 + if self.csv_path and os.path.exists(self.csv_path): + try: + with open(self.csv_path, 'r', encoding='utf-8') as f: + reader = csv.DictReader(f) + for row in reader: + m = row.get("preferredSingleSignOnMode", "").lower() + if m == "saml": saml += 1 + elif m == "oidc": oidc += 1 + elif m == "password": password += 1 + else: none_count += 1 + except Exception as e: + usage_logger.error(f"Error reading SSO CSV: {e}") + + rows = [ + ("SAML", saml), + ("OIDC", oidc), + ("Password", password), + ("Null / Not Supported", none_count) + ] + + for idx, (mode_name, count) in enumerate(rows, start=2): + bg_style = COLOR_SURFACE if idx % 2 != 0 else COLOR_SURFACE_VARIANT + + c0 = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + c0.grid(row=idx, column=0, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c0, text=mode_name, font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=12, anchor="w") + + c1 = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + c1.grid(row=idx, column=1, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c1, text=str(count), font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=12, anchor="w") + + def _export(self): + if not self.csv_path or not os.path.exists(self.csv_path): + messagebox.showinfo("No Data", "There is no SSO data to export.", parent=self) + return + + ts = datetime.now().strftime("%Y%m%d_%H%M%S") + f = filedialog.asksaveasfilename( + initialfile=f"service_principals_sso_{ts}.csv", + defaultextension=".csv", + filetypes=[("CSV Files", "*.csv")], + parent=self + ) + if not f: return + try: + shutil.copyfile(self.csv_path, f) + messagebox.showinfo("Export Successful", f"SSO exported to:\n{f}", parent=self) + except Exception as e: + messagebox.showerror("Export Failed", f"Error: {e}", parent=self) + + @property + def last_data(self): + if not self.csv_path or not os.path.exists(self.csv_path): return [] + try: + with open(self.csv_path, 'r', encoding='utf-8') as f: + reader = csv.DictReader(f) + return list(reader) + except Exception: + return [] + + def cancel(self): + self.is_cancelled = True + self.status = None diff --git a/telemetry/sharepoint_onedrive_usage.py b/telemetry/sharepoint_onedrive_usage.py new file mode 100644 index 00000000..137157a1 --- /dev/null +++ b/telemetry/sharepoint_onedrive_usage.py @@ -0,0 +1,23 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Backward compatibility facade for SharePoint and OneDrive telemetry.""" + +# Re-export pipeline and helper from core backend +from core.graph.files.sharepoint import run_sharepoint_pipeline, parse_sharepoint_csv +from core.graph.files.onedrive import run_onedrive_pipeline, parse_onedrive_csv, format_bytes + +# Re-export UI subframes from telemetry package +from telemetry.files.sharepoint import SharePointUsageFrame +from telemetry.files.onedrive import OneDriveUsageFrame diff --git a/telemetry/styles.py b/telemetry/styles.py new file mode 100644 index 00000000..e430e2c4 --- /dev/null +++ b/telemetry/styles.py @@ -0,0 +1,39 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shared Material 3 style constants and typography tokens for the Telemetry module.""" + +COLOR_PRIMARY = "#1A73E8" +COLOR_PRIMARY_HOVER = "#1557B0" +COLOR_SURFACE = "#FFFFFF" +COLOR_SURFACE_VARIANT = "#F1F3F4" +COLOR_SURFACE_HOVER = "#EFF6FF" +COLOR_TONAL_BG = "#E8F0FE" +COLOR_TONAL_TEXT = "#1A73E8" +COLOR_TEXT_MAIN = "#202124" +COLOR_TEXT_SUB = "#5F6368" +COLOR_OUTLINE = "#80868B" +COLOR_OUTLINE_LIGHT = "#E8EAED" +COLOR_SUCCESS = "#137333" +COLOR_ERROR = "#C5221F" +COLOR_SECONDARY_HOVER = "#F1F3F4" + +FONT_HEADER_MEDIUM = ("Segoe UI", 22, "bold") +FONT_HEADER_SMALL = ("Segoe UI", 16, "bold") +FONT_SUBSECTION_HEADER = ("Segoe UI", 14, "bold") +FONT_BODY_BOLD = ("Segoe UI", 12, "bold") +FONT_BODY_MEDIUM = ("Segoe UI", 12, "normal") +FONT_BODY_SMALL = ("Segoe UI", 10, "normal") +FONT_BODY_SMALL_UNDERLINED = ("Segoe UI", 10, "underline") + diff --git a/telemetry/subscribed_skus.py b/telemetry/subscribed_skus.py new file mode 100644 index 00000000..fa564221 --- /dev/null +++ b/telemetry/subscribed_skus.py @@ -0,0 +1,426 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Modular Subscribed SKUs Inventory Summary telemetry scanners and visual interfaces.""" + +import os +import csv +import logging +import threading +import webbrowser +import pandas as pd +from datetime import datetime +from tkinter import filedialog, messagebox +from typing import Any, Dict, List, Optional +import customtkinter as ctk +import asyncio + +# Import unified core service layer +from core.graph.client import GraphClient +from core.graph.directory import DirectoryService +from core.graph.db import import_csv_to_sqlite, query_page_sync + +# Bind to the async logger initialized in m365_telemetry.py +usage_logger = logging.getLogger("M365TelemetryAsyncLogger") + +# ================================================================================= +# CONSTANTS & STYLES (Imported from shared styles) +# ================================================================================= +from telemetry.styles import * + +class SubscribedSKUsFrame(ctk.CTkFrame): + """Self-contained customtkinter component wrapping Subscribed SKUs Inventory Summary UI.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, retries_var=None, backoff_var=None, **kwargs): + self.semaphore = kwargs.pop("concurrency_semaphore", None) + super().__init__(master, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=12, **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.retries = retries_var + self.backoff = backoff_var + self.status = None # 'loading', 'success', 'error', None + self.last_licenses_items = [] + + # Pagination variables + self.ITEMS_PER_PAGE = 10 + self.current_page = 0 + self.csv_path = None + + self.build_ui() + + def build_ui(self): + """Creates card container for the tab.""" + self.pack(fill="x", expand=True, pady=10) + + self.inner_pad = ctk.CTkFrame(self, fg_color="transparent") + self.inner_pad.pack(fill="both", expand=True, padx=20, pady=20) + + self.lic_header = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.lic_header.pack(fill="x", pady=(0, 10)) + + self.header = ctk.CTkFrame(self.lic_header, fg_color="transparent") + self.header.pack(fill="x", pady=(0, 10)) + ctk.CTkLabel(self.header, text="Subscribed SKUs", font=FONT_HEADER_SMALL, text_color=COLOR_TEXT_MAIN).pack(side="left") + self.reload_btn = ctk.CTkButton( + self.header, + state="disabled", text="↻ Reload", + width=80, + height=24, + font=__import__("customtkinter").CTkFont(family="Segoe UI", size=12), + fg_color="transparent", + border_width=1, + text_color="#2563EB", + hover_color="#DBEAFE", + command=self._retry_fetch + ) + self.reload_btn.pack(side="right") + + self.lic_reference_link = ctk.CTkLabel( + self.lic_header, + text="Service Plan Reference ↗", + font=FONT_BODY_BOLD, + text_color=COLOR_PRIMARY, + cursor="hand2" + ) + self.lic_reference_link.pack(side="left", padx=(15, 0)) + self.lic_reference_link.bind("", lambda e: webbrowser.open("https://learn.microsoft.com/en-us/entra/identity/users/licensing-service-plan-reference")) + self.lic_reference_link.bind("", lambda e: self.lic_reference_link.configure(text_color=COLOR_PRIMARY_HOVER)) + self.lic_reference_link.bind("", lambda e: self.lic_reference_link.configure(text_color=COLOR_PRIMARY)) + + ctk.CTkLabel(self.lic_header, text="* To view specific services offered, export the spreadsheet.", font=FONT_BODY_SMALL, text_color=COLOR_TEXT_SUB).pack(side="left", padx=(10, 0)) + + + + self.state_frame = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.grid_frame = ctk.CTkFrame(self.inner_pad, fg_color=COLOR_OUTLINE_LIGHT, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + + self.reset_view() + + def reset_view(self): + """Resets and hides grids.""" + self.pack_forget() + self.state_frame.pack_forget() + self.grid_frame.pack_forget() + if hasattr(self, "pagination_frame") and self.pagination_frame.winfo_exists(): + self.pagination_frame.destroy() + + self.last_licenses_items = [] + self.current_page = 0 + self.csv_path = None + + for w in self.state_frame.winfo_children(): + w.destroy() + for w in self.grid_frame.winfo_children(): + w.destroy() + + def _set_state_loading(self, msg="Loading..."): + for w in self.state_frame.winfo_children(): + w.destroy() + ctk.CTkLabel(self.state_frame, text=f"⏳ {msg}", text_color=COLOR_TEXT_SUB, font=FONT_BODY_MEDIUM).pack(pady=(20, 5)) + self.progress = __import__("customtkinter").CTkProgressBar(self.state_frame, mode="indeterminate", width=250, fg_color="#F3F4F6", progress_color="#2563EB") + self.progress.pack(pady=(0, 20)) + self.progress.start() + self.state_frame.pack(fill="x", expand=True) + + def _set_state_error(self, error_msg): + for w in self.state_frame.winfo_children(): + w.destroy() + + display_msg = error_msg + if "401" in error_msg or "403" in error_msg or "unauthorized" in error_msg.lower() or "forbidden" in error_msg.lower(): + display_msg = "Directory/Organization read permission required.\nPlease grant the 'Organization.Read.All' or 'Directory.Read.All' permission to your App Registration in Entra ID." + + ctk.CTkLabel(self.state_frame, text=f"✖ {display_msg}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center").pack(pady=(20, 10)) + ctk.CTkButton(self.state_frame, text="Try Again", command=self._retry_fetch, width=120, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 20)) + self.state_frame.pack(fill="x", expand=True) + + def _retry_fetch(self): + if hasattr(self, 'reload_btn') and self.reload_btn.winfo_exists(): + self.reload_btn.configure(state="disabled") + tenant, clients, secrets = self.get_credentials() + if tenant: + self.trigger_fetch(tenant, clients, secrets) + + def trigger_fetch(self, tenant, clients, secrets): + """Triggers SKU fetch inside background thread.""" + usage_logger.info("SKU fetch trigger_fetch called. Spawning background worker thread...") + self.status = "loading" + self.on_status_change() + + self.current_page = 0 + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + reports_dir = os.path.join(script_dir, "reports", f"{tenant}_{clients[0]}") + self.csv_path = os.path.join(reports_dir, "subscribed_skus.csv") + + self.pack(fill="x", expand=True, pady=10) + self.grid_frame.pack_forget() + if hasattr(self, "pagination_frame") and self.pagination_frame.winfo_exists(): + self.pagination_frame.destroy() + + self._set_state_loading("Fetching subscribed SKUs...") + + retries_val = self.retries.get() if self.retries else 5 + backoff_val = self.backoff.get() if self.backoff else 2 + + threading.Thread( + target=self._execute_worker, + args=(tenant, clients, secrets, retries_val, backoff_val), + daemon=True + ).start() + + def _execute_worker(self, tenant: str, clients: List[str], secrets: List[str], retries_val: int, backoff_val: int): + usage_logger.info("Executing thread: _execute_sku_worker") + if self.semaphore: + self.semaphore.acquire() + try: + client_id = clients[0] + client_secret = secrets[0] + + self.log_msg(f"Authenticating app {client_id[:5]}...") + + client = GraphClient( + tenant_id=tenant, + client_ids=client_id, + client_secrets=client_secret, + concurrency=1, + retries=retries_val, + backoff=backoff_val + ) + + required_scopes = ["Organization.Read.All", "Directory.Read.All"] + client.authenticate(required_scopes=required_scopes) + + self.log_msg("Querying Graph API endpoint for SKUs...") + dir_service = DirectoryService(client) + sku_data = dir_service.get_subscribed_skus() + client.close() + + usage_logger.info("Successfully fetched SKU data. Writing to disk...") + + # Write to CSV on disk + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + reports_dir = os.path.join(script_dir, "reports", f"{tenant}_{client_id}") + os.makedirs(reports_dir, exist_ok=True) + csv_path = os.path.join(reports_dir, "subscribed_skus.csv") + + headers = ["SKU Part Number", "Units", "Consumed Units", "Included Service Plans", "Applies To"] + rows = [] + + items = sku_data.get("value", []) + for item in items: + sku_name = item.get("skuPartNumber", "UNKNOWN_SKU") + prepaid = item.get("prepaidUnits", {}) + enabled_units = prepaid.get("enabled", 0) + warn_units = prepaid.get("warning", 0) + susp_units = prepaid.get("suspended", 0) + + prepaid_str = f"Enabled: {enabled_units:,}" + if warn_units > 0: prepaid_str += f"\nWarn: {warn_units:,}" + if susp_units > 0: prepaid_str += f"\nSusp: {susp_units:,}" + consumed_str = f"{item.get('consumedUnits', 0):,}" + + plans = item.get("servicePlans", []) + + if not plans: + rows.append([sku_name, prepaid_str, consumed_str, "None designated.", "-"]) + else: + for idx, p in enumerate(plans): + p_name = p.get("servicePlanName", "UnnamedPlan") + p_scope = p.get("appliesTo", "Unknown") + if idx == 0: + rows.append([sku_name, prepaid_str, consumed_str, p_name, p_scope]) + else: + rows.append(["", "", "", p_name, p_scope]) + + with open(csv_path, 'w', encoding='utf-8', newline='') as f: + writer = csv.writer(f) + writer.writerow(headers) + writer.writerows(rows) + + usage_logger.info(f"Successfully wrote SKU data to {csv_path}") + + db_path = os.path.join(reports_dir, "telemetry_cache.db") + asyncio.run(import_csv_to_sqlite(csv_path, db_path, "subscribed_skus")) + + self.after(0, self._render_success, sku_data) + except Exception as e: + usage_logger.error("Exception caught in SubscribedSKUs worker.", exc_info=True) + self.after(0, self._render_error, str(e)) + finally: + if self.semaphore: + self.semaphore.release() + + def _render_success(self, sku_dict: Dict[str, Any]): + usage_logger.info("Executing UI render for SKU table.") + if hasattr(self, 'reload_btn') and self.reload_btn.winfo_exists(): + self.reload_btn.configure(state="normal") + + items = sku_dict.get("value", []) + items.sort(key=lambda x: len(x.get("servicePlans", [])), reverse=True) + self.last_licenses_items = items + + self._update_ui_paginated() + + def _load_page_from_csv(self, page): + if not self.csv_path or not os.path.exists(self.csv_path): + return [], 0 + + reports_dir = os.path.dirname(self.csv_path) + db_path = os.path.join(reports_dir, "telemetry_cache.db") + + if not os.path.exists(db_path): + return [], 0 + + import sqlite3 + conn = sqlite3.connect(db_path) + try: + conn.row_factory = sqlite3.Row + cursor = conn.cursor() + + # Count only unique SKUs + cursor.execute("SELECT COUNT(*) FROM subscribed_skus WHERE [SKU_Part_Number] IS NOT NULL AND [SKU_Part_Number] != ''") + row = cursor.fetchone() + total_count = row[0] if row else 0 + + # Get page of unique SKUs + offset = page * self.ITEMS_PER_PAGE + cursor.execute( + "SELECT [SKU_Part_Number], [Units], [Consumed_Units] FROM subscribed_skus WHERE [SKU_Part_Number] IS NOT NULL AND [SKU_Part_Number] != '' LIMIT ? OFFSET ?", + (self.ITEMS_PER_PAGE, offset) + ) + rows = cursor.fetchall() + + page_data = [] + for r in rows: + page_data.append(( + r["SKU_Part_Number"] or "", + r["Units"] or "", + r["Consumed_Units"] or "" + )) + return page_data, total_count + except Exception as e: + usage_logger.error(f"Error loading SKU page from SQLite: {e}") + return [], 0 + finally: + conn.close() + + def _update_ui_paginated(self, data=None): + self.state_frame.pack_forget() + for w in self.grid_frame.winfo_children(): + w.destroy() + + self.grid_frame.pack(fill="x", expand=True) + + # Get the page slice + page_data, total_count = self._load_page_from_csv(self.current_page) + + self.grid_frame.grid_columnconfigure(0, weight=2) + self.grid_frame.grid_columnconfigure(1, weight=1) + self.grid_frame.grid_columnconfigure(2, weight=1) + + headers = ["SKU Part Number", "Units", "Consumed Units"] + for col_idx, head_text in enumerate(headers): + cell = ctk.CTkFrame(self.grid_frame, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + if not page_data: + empty_cell = ctk.CTkFrame(self.grid_frame, fg_color="transparent") + empty_cell.grid(row=1, column=0, columnspan=3, sticky="nsew", pady=15) + ctk.CTkLabel(empty_cell, text="No subscribed product configurations found in scope.", text_color=COLOR_TEXT_SUB).pack() + else: + for item_idx, (sku_name, prepaid_str, consumed_str) in enumerate(page_data): + bg_style = COLOR_SURFACE if item_idx % 2 == 0 else COLOR_SURFACE_VARIANT + + c0 = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + c0.grid(row=item_idx + 1, column=0, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c0, text=sku_name, font=FONT_BODY_BOLD, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=8, anchor="nw") + + c1 = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + c1.grid(row=item_idx + 1, column=1, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c1, text=prepaid_str, text_color=COLOR_TEXT_MAIN, justify="left").pack(padx=10, pady=8, anchor="nw") + + c2 = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + c2.grid(row=item_idx + 1, column=2, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c2, text=consumed_str, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=8, anchor="nw") + + # Draw pagination controls if we have multiple pages + if total_count > 0: + self._draw_pagination_controls(total_count) + + self.status = "success" + self.on_status_change() + + def _draw_pagination_controls(self, total_count): + total_pages = (total_count + self.ITEMS_PER_PAGE - 1) // self.ITEMS_PER_PAGE + if total_pages <= 1: + return + + if hasattr(self, "pagination_frame") and self.pagination_frame.winfo_exists(): + self.pagination_frame.destroy() + + self.pagination_frame = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.pagination_frame.pack(fill="x", pady=(2, 0)) + + left_spacer = ctk.CTkFrame(self.pagination_frame, fg_color="transparent") + left_spacer.pack(side="left", fill="x", expand=True) + + center_container = ctk.CTkFrame(self.pagination_frame, fg_color="transparent") + center_container.pack(side="left") + + prev_state = "normal" if self.current_page > 0 else "disabled" + btn_prev = ctk.CTkButton( + center_container, text="◀ Prev", width=70, height=22, corner_radius=6, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, + state=prev_state, + command=lambda: self._change_page(-1) + ) + btn_prev.pack(side="left", padx=5) + + page_lbl = ctk.CTkLabel( + center_container, + text=f"Page {self.current_page + 1} of {total_pages} ({total_count} items)", + font=FONT_BODY_MEDIUM, + text_color=COLOR_TEXT_SUB + ) + page_lbl.pack(side="left", padx=15) + + next_state = "normal" if self.current_page < total_pages - 1 else "disabled" + btn_next = ctk.CTkButton( + center_container, text="Next ▶", width=70, height=22, corner_radius=6, + font=FONT_BODY_SMALL, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, + state=next_state, + command=lambda: self._change_page(1) + ) + btn_next.pack(side="left", padx=5) + + right_spacer = ctk.CTkFrame(self.pagination_frame, fg_color="transparent") + right_spacer.pack(side="right", fill="x", expand=True) + + def _change_page(self, delta): + self.current_page += delta + self._update_ui_paginated() + + def _render_error(self, err_msg): + usage_logger.warning(f"Rendering SKU table error state: {err_msg}") + if hasattr(self, 'reload_btn') and self.reload_btn.winfo_exists(): + self.reload_btn.configure(state="normal") + self._set_state_error(err_msg) + + self.status = "error" + self.on_status_change() + diff --git a/telemetry/transport_rules_ui.py b/telemetry/transport_rules_ui.py new file mode 100644 index 00000000..fa578226 --- /dev/null +++ b/telemetry/transport_rules_ui.py @@ -0,0 +1,21 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Backward compatibility facade for Exchange Online Transport Rules telemetry.""" + +# Re-export pipeline from core backend +from core.graph.exchange.transport_rules import run_transport_rules_pipeline + +# Re-export UI subframe from telemetry package +from telemetry.exchange.transport_rules import TransportRulesFrame diff --git a/tests/files/data_state_creator.py b/tests/files/data_state_creator.py index 0b2a8241..874bd111 100644 --- a/tests/files/data_state_creator.py +++ b/tests/files/data_state_creator.py @@ -422,10 +422,10 @@ def calculate_expected(ignore_failures=False): if f_metrics and f_metrics["subTreeCount"] >= 50: # Using limit 50 drive_metrics["largeResources"].append({ "type": "FOLDER", - "id": sub_item["id"], + "id": sub_item["name"], "subTreeCount": f_metrics["subTreeCount"], "Limit": 50, - "parent": drive_id + "drive": drive_id }) expected["tenantLevelLargeResources"].append(drive_metrics["largeResources"][-1]) @@ -467,57 +467,9 @@ def calculate_expected(ignore_failures=False): expected["siteMetrics"][root_site_id]["fileCountExceedingDepthLimit"] += drive_metric["fileCountExceedingDepthLimit"] expected["siteMetrics"][root_site_id]["totalSize"] += drive_metric["totalSize"] - - # Second pass: compute DL, Subsite, and Site Collection large resources - for site_id, site in data["sites"].items(): - curr_site = site - while "parentReference" in curr_site and "siteId" in curr_site["parentReference"]: - parent_id = curr_site["parentReference"]["siteId"] - curr_site = data["sites"][parent_id] - root_site_id = curr_site["id"] - - subsite_item_count = 0 - for drive_id in site["drives"]: - if drive_id in expected["driveMetrics"]: - drive_metric = expected["driveMetrics"][drive_id] - drive_item_count = drive_metric["folderCount"] + drive_metric["fileCount"] - subsite_item_count += drive_item_count - - # Check if DL is a Large Resource - if drive_item_count > 50: - expected["siteMetrics"][root_site_id]["largeResourceCount"] += 1 - expected["tenantLevelLargeResources"].append({ - "type": "DOCUMENT LIBRARY", - "id": drive_id, - "subTreeCount": drive_item_count, - "parent": site_id, - "Limit": 50 - }) - - # Check if Subsite is a Large Resource - if site["siteLevel"] > 0 and subsite_item_count > 50: - expected["siteMetrics"][root_site_id]["largeResourceCount"] += 1 - expected["tenantLevelLargeResources"].append({ - "type": "SUBSITE", - "id": site_id, - "subTreeCount": subsite_item_count, - "parent": root_site_id, - "Limit": 50 - }) for root_site_id, s_metrics in expected["siteMetrics"].items(): s_metrics["resourceCount"] = s_metrics["folderCount"] + s_metrics["fileCount"] + s_metrics["shortcutCount"] - - # Check if Site Collection is a Large Resource - total_site_count = s_metrics["folderCount"] + s_metrics["fileCount"] - if total_site_count > 50: - expected["tenantLevelLargeResources"].append({ - "type": "SITE COLLECTION", - "id": root_site_id, - "subTreeCount": total_site_count, - "parent": "N/A (Top level site)", - "Limit": 50 - }) for site_id, site in data["sites"].items(): is_personal = site.get("isPersonalSite", False) diff --git a/tests/files/load_tests.py b/tests/files/load_tests.py index a011abc5..901ab0ea 100644 --- a/tests/files/load_tests.py +++ b/tests/files/load_tests.py @@ -139,11 +139,7 @@ def test_load_simulation_all_sites(self): self.assertEqual(r_site.get("shortcutCount", 0), e_site.get("shortcutCount", 0)) self.assertEqual(r_site.get("folderCountExceedingDepthLimit", 0), e_site.get("folderCountExceedingDepthLimit", 0)) self.assertEqual(r_site.get("fileCountExceedingDepthLimit", 0), e_site.get("fileCountExceedingDepthLimit", 0)) - try: - self.assertEqual(r_site.get("largeResourceCount", 0), e_site.get("largeResourceCount", 0)) - except AssertionError as e: - print(f"\nDISCREPANCY for site {site_id}: result={r_site} | expected={e_site}\n") - raise e + self.assertEqual(r_site.get("largeResourceCount", 0), e_site.get("largeResourceCount", 0)) self.assertEqual(r_site.get("totalSize", 0), e_site.get("totalSize", 0)) self.assertEqual(r_site.get("resourceCount", 0), e_site.get("resourceCount", 0)) @@ -155,8 +151,7 @@ def test_load_simulation_all_sites(self): self.assertEqual(sum(s.get("shortcutCount", 0) for s in site_metrics_values), result.get("shortcutCount", 0)) self.assertEqual(sum(s.get("folderCountExceedingDepthLimit", 0) for s in site_metrics_values), result.get("folderCountExceedingDepthLimit", 0)) self.assertEqual(sum(s.get("fileCountExceedingDepthLimit", 0) for s in site_metrics_values), result.get("fileCountExceedingDepthLimit", 0)) - site_collection_large_res_count = sum(1 for res in result.get("tenantLevelLargeResources", []) if res.get("type") == "SITE COLLECTION") - self.assertEqual(sum(s.get("largeResourceCount", 0) for s in site_metrics_values) + site_collection_large_res_count, result.get("tenantLevelLargeResourceCount", 0)) + self.assertEqual(sum(s.get("largeResourceCount", 0) for s in site_metrics_values), result.get("tenantLevelLargeResourceCount", 0)) self.assertEqual(sum(s.get("dlCount", 0) for s in site_metrics_values), sum(result.get("driveCounts", {}).values())) def _get_expected_for_subset(self, email_ids: List[str]) -> Tuple[Dict[str, Any], List[str]]: @@ -210,7 +205,7 @@ def collect_subsites(site_id, level): "siteCount": len(root_site_ids), "subsiteCount": len(all_site_ids) - len(root_site_ids), "personalSiteCount": len(email_ids), - "teamSiteCount": len(all_site_ids) - len(root_site_ids), + "teamSiteCount": 0, "personalSiteDLCount": personal_site_dl_count, "teamSiteDLCount": team_site_dl_count, "listCount": sum(len(self.test_data.get("sites", {}).get(sid, {}).get("lists", [])) for sid in all_site_ids), @@ -250,19 +245,6 @@ def collect_subsites(site_id, level): r_bucket = next(b for b in expected["tenantLevelFileSizeDistribution"]["buckets"] if b["sizeRange"] == tuple(bucket["sizeRange"])) r_bucket["count"] += bucket["count"] - # Count other large resources (DLs, Subsites, Site Collections) - all_large_resources = self.test_data.get(expected_key, {}).get("tenantLevelLargeResources", []) - for lr in all_large_resources: - lr_type = lr.get("type") - lr_id = lr.get("id") - - if lr_type == "DOCUMENT LIBRARY" and lr_id in scanned_drives: - expected["tenantLevelLargeResourceCount"] += 1 - elif lr_type == "SUBSITE" and lr_id in all_site_ids and lr_id not in root_site_ids: - expected["tenantLevelLargeResourceCount"] += 1 - elif lr_type == "SITE COLLECTION" and lr_id in root_site_ids: - expected["tenantLevelLargeResourceCount"] += 1 - return expected, subset_drive_ids def test_load_simulation_from_csv(self): diff --git a/ui/chats_ui.py b/ui/chats_ui.py index 5369502e..24ff3391 100644 --- a/ui/chats_ui.py +++ b/ui/chats_ui.py @@ -15,6 +15,10 @@ import webbrowser import customtkinter as ctk + +# Performance optimizations for CustomTkinter across OS +ctk.set_window_scaling(1.0) +ctk.set_widget_scaling(1.0) from estimators.estimator import Estimator from estimators.factory import EstimatorFactory import pandas as pd diff --git a/ui/exchange_online_ui.py b/ui/exchange_online_ui.py index 1b8c7caa..2c432df3 100644 --- a/ui/exchange_online_ui.py +++ b/ui/exchange_online_ui.py @@ -1,5 +1,10 @@ from tkinter import filedialog, messagebox import customtkinter as ctk + +# Performance optimizations for CustomTkinter across OS +ctk.set_window_scaling(1.0) +ctk.set_widget_scaling(1.0) + import re from typing import Any, Callable, Dict, List, Optional, Tuple import queue @@ -2648,7 +2653,6 @@ def get_batch_eta(subset_df): "user_limit": ETA_EMAIL_USER_LIMIT, "batch_size": ETA_EMAIL_BATCH_SIZE, "batch_time": ETA_EMAIL_BATCH_TIME, - "multiplier": IPA_ETA_MULTIPLIER } ) eta_shared_mail_box = 0.0 diff --git a/ui/files_ui.py b/ui/files_ui.py index 586a3020..18b48332 100644 --- a/ui/files_ui.py +++ b/ui/files_ui.py @@ -4,6 +4,10 @@ from datetime import timedelta, datetime import os import customtkinter as ctk + +# Performance optimizations for CustomTkinter across OS +ctk.set_window_scaling(1.0) +ctk.set_widget_scaling(1.0) import time import psutil from tkinter import messagebox @@ -362,7 +366,7 @@ def _try_get_metrics_from_csv_report(self, config): "Shortcut Count", "Folder Count > Depth Limit 100", "File Count > Depth Limit 100", - "Entities with > 500k item count", + "Folder with > 500k item count", "Corpus Size", } id_col = "Site URL/Name" if "Site URL/Name" in df.columns else ("Site Id" if "Site Id" in df.columns else ("Entity" if "Entity" in df.columns else None)) @@ -408,7 +412,7 @@ def _parse_size_str(val): "shortcutCount": shortcut_cnt, "folderCountExceedingDepthLimit": int(pd.to_numeric(row.get("Folder Count > Depth Limit 100", 0), errors="coerce") or 0), "fileCountExceedingDepthLimit": int(pd.to_numeric(row.get("File Count > Depth Limit 100", 0), errors="coerce") or 0), - "largeResourceCount": int(pd.to_numeric(row.get("Entities with > 500k item count", 0), errors="coerce") or 0), + "largeResourceCount": int(pd.to_numeric(row.get("Folder with > 500k item count", 0), errors="coerce") or 0), "totalSize": _parse_size_str(row.get("Corpus Size", 0)), "resourceCount": res_cnt, } @@ -430,7 +434,7 @@ def _parse_size_str(val): "listCount": int(pd.to_numeric(df.get("List Count", pd.Series([0])), errors="coerce").fillna(0).sum()), "folderCountExceedingDepthLimit": int(pd.to_numeric(df.get("Folder Count > Depth Limit 100", pd.Series([0])), errors="coerce").fillna(0).sum()), "fileCountExceedingDepthLimit": int(pd.to_numeric(df.get("File Count > Depth Limit 100", pd.Series([0])), errors="coerce").fillna(0).sum()), - "tenantLevelLargeResourceCount": int(pd.to_numeric(df.get("Entities with > 500k item count", pd.Series([0])), errors="coerce").fillna(0).sum()), + "tenantLevelLargeResourceCount": int(pd.to_numeric(df.get("Folder with > 500k item count", pd.Series([0])), errors="coerce").fillna(0).sum()), "siteClassification": {site_id: "personal" for site_id in site_metrics.keys()}, "licenseMetrics": {}, "tenantLevelFileSizeDistribution": {}, @@ -553,7 +557,7 @@ def execute_migration_scan(self, config): "Shortcut Count": s_data.get("shortcutCount", 0), "Folder Count > Depth Limit 100": s_data.get("folderCountExceedingDepthLimit", 0), "File Count > Depth Limit 100": s_data.get("fileCountExceedingDepthLimit", 0), - "Entities with > 500k item count": s_data.get("largeResourceCount", 0), + "Folder with > 500k item count": s_data.get("largeResourceCount", 0), "Corpus Size": s_data.get("totalSize", 0), "Resource Count": s_data.get("resourceCount", 0) }) @@ -959,7 +963,7 @@ def show_results_content(self, data): self.create_stat_card(card_frame, "List Count", f"{data.get('listCount', 0):,}", "🗃️") self.create_stat_card(card_frame, "Folder count beyond depth limit 100", f"{data.get('folderCountExceedingDepthLimit', 0):,}", "📁") self.create_stat_card(card_frame, "File count beyond depth limit 100", f"{data.get('fileCountExceedingDepthLimit', 0):,}", "📄") - self.create_stat_card(card_frame, "Large Resource Count (Entities with >500k items)", f"{data.get('tenantLevelLargeResourceCount', 0):,}", "📄") + self.create_stat_card(card_frame, "Large Resource Count (Folders with >500k items)", f"{data.get('tenantLevelLargeResourceCount', 0):,}", "📄") if self.show_eta: # Timeline @@ -1177,7 +1181,7 @@ def _validate_csv(self): "Shortcut Count", "Folder Count > Depth Limit 100", "File Count > Depth Limit 100", - "Entities with > 500k item count", + "Folder with > 500k item count", "Corpus Size", } is_report_csv = "Entity" in df.columns and report_cols.issubset(df.columns) @@ -1263,7 +1267,7 @@ def export_current_report(self): ("Shortcut Count", data.get("shortcutCount", 0)), ("Folder count beyond depth limit 100", data.get("folderCountExceedingDepthLimit", 0)), ("File count beyond depth limit 100", data.get("fileCountExceedingDepthLimit", 0)), - ("Large Resource Count (Entities with >500k items)", data.get("tenantLevelLargeResourceCount", 0)) + ("Large Resource Count (Folders with >500k items)", data.get("tenantLevelLargeResourceCount", 0)) ] for label, val in summary_rows: @@ -1292,20 +1296,15 @@ def export_current_report(self): # Section 4: Large Resources if len(data.get("tenantLevelLargeResources", [])) > 0: - weights = { - "SITE COLLECTION": 1, - "SUBSITE": 2, - "DOCUMENT LIBRARY": 3, - "FOLDER": 4 - } - sorted_large_resources = sorted(data.get("tenantLevelLargeResources", []), key=lambda x: weights.get(x.get("Type", x.get("type", "")), 0), reverse=False) - writer.writerow(["Large Resources (Entities with >500k items)", ""]) - writer.writerow(["Type", "URL", "Item Count"]) - for res in sorted_large_resources: + writer.writerow(["Large Resources", ""]) + writer.writerow(["Type", "ID", "SubTreeCount", "Drive"]) + large_resources = data.get("tenantLevelLargeResources", []) + for res in large_resources: writer.writerow([ res.get("Type", res.get("type", "")), - self._get_display_name(res.get("Id", res.get("id", ""))), - res.get("subTreeCount", 0) + res.get("Id", res.get("id", "")), + res.get("subTreeCount", 0), + self._get_display_name(res.get("drive", "")) ]) writer.writerow([]) # Blank line separator @@ -1314,9 +1313,9 @@ def export_current_report(self): # Section 5: Site Details writer.writerow(["Site Details", ""]) if "siteIdToMail" not in data: - row = ["Site Collection", "Subsite Count", "DL Count", "List Count", "Folder Count", "File Count", "Shortcut Count", "Folder Count > Depth Limit 100", "File Count > Depth Limit 100", "Entities with > 500k item count", "Corpus Size"] + row = ["Site Collection", "Subsite Count", "DL Count", "List Count", "Folder Count", "File Count", "Shortcut Count", "Folder Count > Depth Limit 100", "File Count > Depth Limit 100", "Folder with > 500k item count", "Corpus Size"] else: - row = ["Site Collection", "Email Id", "Subsite Count", "DL Count", "List Count", "Folder Count", "File Count", "Shortcut Count", "Folder Count > Depth Limit 100", "File Count > Depth Limit 100", "Entities with > 500k item count", "Corpus Size"] + row = ["Site Collection", "Email Id", "Subsite Count", "DL Count", "List Count", "Folder Count", "File Count", "Shortcut Count", "Folder Count > Depth Limit 100", "File Count > Depth Limit 100", "Folder with > 500k item count", "Corpus Size"] if self.show_eta: row.append("Suggested Batch") diff --git a/util/constants.py b/util/constants.py index ae968d39..6281f95e 100644 --- a/util/constants.py +++ b/util/constants.py @@ -73,8 +73,6 @@ ETA_EMAIL_BATCH_SIZE = 1 ETA_EMAIL_BATCH_TIME = 6 -IPA_ETA_MULTIPLIER = 1.5 - FILES_GLOBAL_COUNT_LIMIT = 4 # 4 files/folders per second FILES_GLOBAL_CORPUS_SIZE_LIMIT = (400 * 1024 * 1024 * 1024) // 3600 # 400 GB per hour in bytes per second diff --git a/util/enums.py b/util/enums.py index c1665407..00eae1ab 100644 --- a/util/enums.py +++ b/util/enums.py @@ -10,8 +10,7 @@ class FailureType(Enum): UNKNOWN_ERROR = 7 class ResourceType(Enum): - SITE = "SITE COLLECTION" + SITE = "SITE" FOLDER = "FOLDER" - DL = "DOCUMENT LIBRARY" - FILE = "FILE" - SUBSITE = "SUBSITE" \ No newline at end of file + DL = "DL" + FILE = "FILE" \ No newline at end of file