From 8c98a2394928068b4d8d4e6468bfcd7bd988069b Mon Sep 17 00:00:00 2001 From: Matt Gotteiner Date: Tue, 14 Jul 2026 11:24:40 -0700 Subject: [PATCH] Add multi-source Foundry IQ sample Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- Foundry-IQ-Multi-Source/.gitignore | 2 + Foundry-IQ-Multi-Source/README.md | 163 ++++++ .../data/release-brief.json | 20 + Foundry-IQ-Multi-Source/pyproject.toml | 27 + Foundry-IQ-Multi-Source/sample.env | 19 + .../src/foundry_iq_multi_source/__init__.py | 14 + .../src/foundry_iq_multi_source/__main__.py | 3 + .../src/foundry_iq_multi_source/cli.py | 68 +++ .../src/foundry_iq_multi_source/config.py | 88 ++++ .../src/foundry_iq_multi_source/sample.py | 463 ++++++++++++++++++ .../src/foundry_iq_multi_source/trace.py | 152 ++++++ .../tests/test_cleanup_safety.py | 153 ++++++ .../tests/test_payloads.py | 80 +++ .../tests/test_sample_data.py | 18 + Foundry-IQ-Multi-Source/tests/test_trace.py | 63 +++ README.md | 1 + 16 files changed, 1334 insertions(+) create mode 100644 Foundry-IQ-Multi-Source/.gitignore create mode 100644 Foundry-IQ-Multi-Source/README.md create mode 100644 Foundry-IQ-Multi-Source/data/release-brief.json create mode 100644 Foundry-IQ-Multi-Source/pyproject.toml create mode 100644 Foundry-IQ-Multi-Source/sample.env create mode 100644 Foundry-IQ-Multi-Source/src/foundry_iq_multi_source/__init__.py create mode 100644 Foundry-IQ-Multi-Source/src/foundry_iq_multi_source/__main__.py create mode 100644 Foundry-IQ-Multi-Source/src/foundry_iq_multi_source/cli.py create mode 100644 Foundry-IQ-Multi-Source/src/foundry_iq_multi_source/config.py create mode 100644 Foundry-IQ-Multi-Source/src/foundry_iq_multi_source/sample.py create mode 100644 Foundry-IQ-Multi-Source/src/foundry_iq_multi_source/trace.py create mode 100644 Foundry-IQ-Multi-Source/tests/test_cleanup_safety.py create mode 100644 Foundry-IQ-Multi-Source/tests/test_payloads.py create mode 100644 Foundry-IQ-Multi-Source/tests/test_sample_data.py create mode 100644 Foundry-IQ-Multi-Source/tests/test_trace.py diff --git a/Foundry-IQ-Multi-Source/.gitignore b/Foundry-IQ-Multi-Source/.gitignore new file mode 100644 index 000000000..648142b28 --- /dev/null +++ b/Foundry-IQ-Multi-Source/.gitignore @@ -0,0 +1,2 @@ +.env +.outputs/ diff --git a/Foundry-IQ-Multi-Source/README.md b/Foundry-IQ-Multi-Source/README.md new file mode 100644 index 000000000..74b74a5bf --- /dev/null +++ b/Foundry-IQ-Multi-Source/README.md @@ -0,0 +1,163 @@ +# Foundry IQ with a Search index and external MCP source + +This sample creates one Foundry IQ knowledge base that references both a +`SearchIndexKnowledgeSource` and an `McpServerKnowledgeSource`. A single +complex query combines fictional public release facts from an Azure AI Search +index with current product guidance from the public Microsoft Learn MCP server. +The run fails unless the returned `activity` and `references` prove that both +sources contributed. + +## What the sample proves + +```text +Complex query + | + v +Foundry IQ knowledge base + |-- SearchIndexKnowledgeSource --> bundled Contoso release brief + `-- McpServerKnowledgeSource ----> https://learn.microsoft.com/api/mcp + microsoft_docs_search + | + v +Synthesized answer + activity + references +``` + +The bundled query asks for a Wave B date and response-time SLO that exist only +in `data/release-brief.json`, plus current MCP retrieval guidance available +through Microsoft Learn. This makes a one-source answer incomplete by design. + +## Prerequisites + +- Python 3.11 or later. +- Azure AI Search in a region and API deployment that supports MCP Server + knowledge sources (`2026-05-01-preview`). +- Semantic ranker enabled on the Search service. +- An Azure OpenAI chat deployment supported by Foundry IQ. +- Permission to create indexes, knowledge sources, and knowledge bases. + +The sample uses `DefaultAzureCredential` for Azure AI Search by default. Assign +your user the Search Service Contributor and Search Index Data Contributor +roles. For keyless model access, enable a managed identity on the Search +service and grant it Cognitive Services OpenAI User on the Azure OpenAI +resource. Optional key environment variables are supported for environments +where RBAC isn't configured. + +## Run from scratch + +```powershell +cd Foundry-IQ-Multi-Source +python -m venv .venv +.\.venv\Scripts\Activate.ps1 +python -m pip install -e ".[dev]" +Copy-Item sample.env .env +``` + +Edit `.env`, sign in, and run: + +```powershell +az login +foundry-iq-multi-source run +``` + +The `run` action creates the index, uploads the bundled public sample data, +creates both knowledge sources and the knowledge base, executes the +cross-source query, verifies source participation, writes the full response to +`.outputs/latest-result.json`, and deletes the created resources. + +Setup refuses to overwrite any existing resource with the configured names. +Each successful creation is recorded in the ignored +`.outputs/resources.json` ownership manifest. Cleanup deletes only resources +listed in that manifest and uses the creation ETag with a conditional delete. +A failed or interrupted run therefore can't delete unrelated, modified, or +replacement resources with similar names. + +To inspect resources before cleanup: + +```powershell +foundry-iq-multi-source run --keep-resources +foundry-iq-multi-source cleanup +``` + +The actions are also independently runnable: + +```powershell +foundry-iq-multi-source setup +foundry-iq-multi-source query +foundry-iq-multi-source cleanup +``` + +## How both sources are made reliable + +| Control | Setting | Why | +|---|---|---| +| Search request routing | `always_query_source=True` | The indexed release brief is always queried. | +| MCP tool inclusion | `inclusion_mode="always"` | Parsed MCP output isn't dropped during final reranking. | +| Initial reranker floor | `reranker_threshold=0.0` on both sources | Validation starts permissively so evidence isn't filtered before tuning. | +| Failure behavior | `fail_on_error=True` on both sources | A missing source is visible instead of producing a plausible partial answer. | +| Per-source document cap | `max_output_documents=50` | Uses the preview API's minimum accepted value (50–200). | +| Runtime budget | `max_runtime_in_seconds=120` | Remote MCP calls can take longer than index retrieval. | +| Trace visibility | `include_activity=True`, `include_references=True`, and `include_reference_source_data=True` | The response contains auditable routing and grounding evidence. | + +`always_query_source` is intentionally omitted from +`McpServerKnowledgeSourceParams`: MCP Server knowledge sources don't support +that request-time control. The cross-source query and retrieval instructions +make the tool relevant, while tool-level `inclusion_mode="always"` keeps its +parsed output in the candidate set. + +After the flow is stable, change MCP inclusion to `"reranked"` and increase +`reranker_threshold` gradually while checking the same trace verifier. Don't +optimize those settings from answer text alone. + +## Expected evidence + +The CLI prints a compact result similar to: + +```json +{ + "search": { + "knowledge_source_name": "foundry-iq-search-source", + "activity_count": 1, + "reference_count": 2 + }, + "mcp": { + "knowledge_source_name": "foundry-iq-learn-mcp-source", + "activity_count": 1, + "reference_count": 3 + }, + "both_participated": true +} +``` + +The verifier doesn't trust names in prose. Each reference must either name the +knowledge source or link through `activitySource` to an activity record for +that source. + +## Troubleshooting + +- **`mcpServer` is rejected during knowledge-source creation:** The Search + service deployment or region doesn't expose the preview feature. +- **Only the Search source appears:** Keep the query's explicit request for + current Microsoft Learn guidance, retain MCP `inclusion_mode="always"`, and + inspect MCP activity errors. +- **MCP activity exists but no MCP reference survives:** Keep the initial + `reranker_threshold=0.0`; confirm `include_references` and + `include_reference_source_data` are enabled. +- **The request times out:** Increase `max_runtime_in_seconds`; don't lower it + below the latency budget required by the remote tool. +- **Model authorization fails:** Grant the Search service managed identity + Cognitive Services OpenAI User, or set `AZURE_OPENAI_API_KEY` locally. +- **A configured resource name already exists:** Choose different names in + `.env`. The sample never updates or deletes a resource it didn't create. + +## External service boundary + +The Microsoft Learn MCP endpoint is public and unauthenticated, but content +still leaves the Search service boundary for the external tool call. Review +the MCP server's terms, data-handling behavior, network reachability, and +compliance fit before replacing the public sample with enterprise data. + +References: + +- [Create an MCP Server knowledge source](https://learn.microsoft.com/azure/search/agentic-knowledge-source-how-to-mcp-server) +- [Create a knowledge base](https://learn.microsoft.com/azure/search/agentic-retrieval-how-to-create-knowledge-base) +- [Retrieve from a knowledge base](https://learn.microsoft.com/azure/search/agentic-retrieval-how-to-retrieve) diff --git a/Foundry-IQ-Multi-Source/data/release-brief.json b/Foundry-IQ-Multi-Source/data/release-brief.json new file mode 100644 index 000000000..291f41574 --- /dev/null +++ b/Foundry-IQ-Multi-Source/data/release-brief.json @@ -0,0 +1,20 @@ +[ + { + "id": "trail-release-wave-b", + "title": "Contoso Trail Guide public release brief", + "content": "Wave B of the Contoso Trail Guide public demo is scheduled for August 18, 2026. The end-to-end response-time service objective is 75 seconds. The go-live reviewer must confirm that the final recommendation combines the release brief with current platform guidance rather than answering from either source alone.", + "source_url": "data/release-brief.json#trail-release-wave-b" + }, + { + "id": "trail-release-evidence", + "title": "Contoso Trail Guide evidence checklist", + "content": "The release gate requires machine-readable evidence that both the indexed release brief and the external documentation source contributed to the answer. A prose claim that both sources ran is not sufficient.", + "source_url": "data/release-brief.json#trail-release-evidence" + }, + { + "id": "trail-release-data-policy", + "title": "Contoso Trail Guide sample data policy", + "content": "This Contoso demonstration uses only fictional release facts and public Microsoft documentation. It must not store credentials, tenant content, or customer data in the index or repository.", + "source_url": "data/release-brief.json#trail-release-data-policy" + } +] diff --git a/Foundry-IQ-Multi-Source/pyproject.toml b/Foundry-IQ-Multi-Source/pyproject.toml new file mode 100644 index 000000000..f04605dc9 --- /dev/null +++ b/Foundry-IQ-Multi-Source/pyproject.toml @@ -0,0 +1,27 @@ +[build-system] +requires = ["setuptools>=77"] +build-backend = "setuptools.build_meta" + +[project] +name = "foundry-iq-multi-source" +version = "0.1.0" +description = "Foundry IQ sample combining a Search index and an external MCP server." +requires-python = ">=3.11" +dependencies = [ + "azure-identity>=1.23.0", + "azure-search-documents==12.1.0b1", + "python-dotenv>=1.1.0", +] + +[project.optional-dependencies] +dev = ["pytest>=8.4.0"] + +[project.scripts] +foundry-iq-multi-source = "foundry_iq_multi_source.cli:main" + +[tool.pytest.ini_options] +pythonpath = ["src"] +testpaths = ["tests"] + +[tool.setuptools.packages.find] +where = ["src"] diff --git a/Foundry-IQ-Multi-Source/sample.env b/Foundry-IQ-Multi-Source/sample.env new file mode 100644 index 000000000..063fc804b --- /dev/null +++ b/Foundry-IQ-Multi-Source/sample.env @@ -0,0 +1,19 @@ +# Required Azure resources. Copy this file to .env and replace placeholders. +SEARCH_ENDPOINT=https://your-search-service.search.windows.net +AZURE_OPENAI_ENDPOINT=https://your-openai-resource.openai.azure.com +AZURE_OPENAI_DEPLOYMENT=gpt-5-mini +AZURE_OPENAI_MODEL=gpt-5-mini + +# Optional deterministic resource names. +INDEX_NAME=foundry-iq-multi-source-index +SEARCH_KNOWLEDGE_SOURCE_NAME=foundry-iq-search-source +MCP_KNOWLEDGE_SOURCE_NAME=foundry-iq-learn-mcp-source +KNOWLEDGE_BASE_NAME=foundry-iq-multi-source-kb + +# The public Microsoft Learn MCP server requires no secret. +MCP_SERVER_URL=https://learn.microsoft.com/api/mcp +MCP_TOOL_NAME=microsoft_docs_search + +# Optional key-based fallback. Prefer Azure RBAC and managed identity. +# SEARCH_API_KEY= +# AZURE_OPENAI_API_KEY= diff --git a/Foundry-IQ-Multi-Source/src/foundry_iq_multi_source/__init__.py b/Foundry-IQ-Multi-Source/src/foundry_iq_multi_source/__init__.py new file mode 100644 index 000000000..70060c77d --- /dev/null +++ b/Foundry-IQ-Multi-Source/src/foundry_iq_multi_source/__init__.py @@ -0,0 +1,14 @@ +"""Foundry IQ multi-source sample.""" + +from .sample import build_knowledge_base, build_mcp_knowledge_source +from .sample import build_retrieval_request, build_search_knowledge_source +from .trace import DualSourceEvidence, verify_dual_source_evidence + +__all__ = [ + "DualSourceEvidence", + "build_knowledge_base", + "build_mcp_knowledge_source", + "build_retrieval_request", + "build_search_knowledge_source", + "verify_dual_source_evidence", +] diff --git a/Foundry-IQ-Multi-Source/src/foundry_iq_multi_source/__main__.py b/Foundry-IQ-Multi-Source/src/foundry_iq_multi_source/__main__.py new file mode 100644 index 000000000..4e28416e1 --- /dev/null +++ b/Foundry-IQ-Multi-Source/src/foundry_iq_multi_source/__main__.py @@ -0,0 +1,3 @@ +from .cli import main + +main() diff --git a/Foundry-IQ-Multi-Source/src/foundry_iq_multi_source/cli.py b/Foundry-IQ-Multi-Source/src/foundry_iq_multi_source/cli.py new file mode 100644 index 000000000..531e02ff9 --- /dev/null +++ b/Foundry-IQ-Multi-Source/src/foundry_iq_multi_source/cli.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +import argparse +import json +import sys + +from .config import DEFAULT_QUERY, Settings +from .sample import MultiSourceSample + + +def main() -> None: + parser = argparse.ArgumentParser( + description=( + "Run a Foundry IQ query across a Search index and an external MCP " + "server." + ) + ) + parser.add_argument( + "action", + choices=["run", "setup", "query", "cleanup"], + nargs="?", + default="run", + ) + parser.add_argument("--query", default=DEFAULT_QUERY) + parser.add_argument( + "--keep-resources", + action="store_true", + help="Keep resources after the run action for inspection.", + ) + args = parser.parse_args() + + settings = Settings.from_env() + sample = MultiSourceSample(settings, settings.search_credential()) + operation_failed = False + try: + if args.action in {"run", "setup"}: + sample.setup() + print("Created the index, both knowledge sources, and knowledge base.") + if args.action in {"run", "query"}: + answer, evidence = sample.query(args.query) + print("\nSynthesized answer\n------------------") + print(answer) + print("\nDual-source evidence\n--------------------") + print(json.dumps(evidence.as_dict(), indent=2)) + print(f"\nFull response: {settings.output_path}") + if args.action == "cleanup": + sample.cleanup() + print("Deleted all sample resources.") + except Exception: + operation_failed = True + raise + finally: + if args.action == "run" and not args.keep_resources: + try: + sample.cleanup() + print("Deleted all sample resources.") + except Exception as cleanup_error: + if operation_failed: + print( + f"Cleanup also failed: {cleanup_error}", + file=sys.stderr, + ) + else: + raise + finally: + sample.close() + else: + sample.close() diff --git a/Foundry-IQ-Multi-Source/src/foundry_iq_multi_source/config.py b/Foundry-IQ-Multi-Source/src/foundry_iq_multi_source/config.py new file mode 100644 index 000000000..289a57ec9 --- /dev/null +++ b/Foundry-IQ-Multi-Source/src/foundry_iq_multi_source/config.py @@ -0,0 +1,88 @@ +from __future__ import annotations + +import os +from dataclasses import dataclass +from pathlib import Path + +from azure.core.credentials import AzureKeyCredential, TokenCredential +from azure.identity import DefaultAzureCredential +from dotenv import load_dotenv + + +DEFAULT_QUERY = """ +Use the indexed Contoso Trail Guide release brief to state the Wave B date and +response-time SLO. Then use the current Microsoft Learn MCP Server knowledge +source guide to identify the exact `maxRuntimeInSeconds` request property that +should exceed that SLO and explain how the `activity` and `references` response +sections prove that both sources participated. Return one recommendation that +cites facts from both source types. +""".strip() + + +@dataclass(frozen=True) +class Settings: + search_endpoint: str + azure_openai_endpoint: str + azure_openai_deployment: str + azure_openai_model: str + index_name: str = "foundry-iq-multi-source-index" + search_knowledge_source_name: str = "foundry-iq-search-source" + mcp_knowledge_source_name: str = "foundry-iq-learn-mcp-source" + knowledge_base_name: str = "foundry-iq-multi-source-kb" + mcp_server_url: str = "https://learn.microsoft.com/api/mcp" + mcp_tool_name: str = "microsoft_docs_search" + search_api_key: str | None = None + azure_openai_api_key: str | None = None + + @classmethod + def from_env(cls) -> Settings: + load_dotenv() + return cls( + search_endpoint=_required("SEARCH_ENDPOINT"), + azure_openai_endpoint=_required("AZURE_OPENAI_ENDPOINT").rstrip("/"), + azure_openai_deployment=_required("AZURE_OPENAI_DEPLOYMENT"), + azure_openai_model=_required("AZURE_OPENAI_MODEL"), + index_name=os.getenv("INDEX_NAME", cls.index_name), + search_knowledge_source_name=os.getenv( + "SEARCH_KNOWLEDGE_SOURCE_NAME", + cls.search_knowledge_source_name, + ), + mcp_knowledge_source_name=os.getenv( + "MCP_KNOWLEDGE_SOURCE_NAME", + cls.mcp_knowledge_source_name, + ), + knowledge_base_name=os.getenv( + "KNOWLEDGE_BASE_NAME", + cls.knowledge_base_name, + ), + mcp_server_url=os.getenv("MCP_SERVER_URL", cls.mcp_server_url), + mcp_tool_name=os.getenv("MCP_TOOL_NAME", cls.mcp_tool_name), + search_api_key=os.getenv("SEARCH_API_KEY"), + azure_openai_api_key=os.getenv("AZURE_OPENAI_API_KEY"), + ) + + @property + def data_path(self) -> Path: + return Path(__file__).resolve().parents[2] / "data" / "release-brief.json" + + @property + def output_path(self) -> Path: + return Path(__file__).resolve().parents[2] / ".outputs" / "latest-result.json" + + @property + def manifest_path(self) -> Path: + return Path(__file__).resolve().parents[2] / ".outputs" / "resources.json" + + def search_credential(self) -> AzureKeyCredential | TokenCredential: + if self.search_api_key: + return AzureKeyCredential(self.search_api_key) + return DefaultAzureCredential() + + +def _required(name: str) -> str: + value = os.getenv(name) + if not value: + raise RuntimeError( + f"Missing {name}. Copy sample.env to .env and set the required values." + ) + return value diff --git a/Foundry-IQ-Multi-Source/src/foundry_iq_multi_source/sample.py b/Foundry-IQ-Multi-Source/src/foundry_iq_multi_source/sample.py new file mode 100644 index 000000000..bf201d11e --- /dev/null +++ b/Foundry-IQ-Multi-Source/src/foundry_iq_multi_source/sample.py @@ -0,0 +1,463 @@ +from __future__ import annotations + +import json +import time +from pathlib import Path +from typing import Any + +from azure.core import MatchConditions +from azure.core.credentials import AzureKeyCredential, TokenCredential +from azure.core.exceptions import HttpResponseError, ResourceNotFoundError +from azure.search.documents import SearchClient +from azure.search.documents.indexes import SearchIndexClient +from azure.search.documents.indexes.models import ( + AzureOpenAIVectorizerParameters, + KnowledgeBase, + KnowledgeBaseAzureOpenAIModel, + KnowledgeSourceReference, + McpServerAutoOutputParsing, + McpServerKnowledgeSource, + McpServerKnowledgeSourceParameters, + McpServerTool, + SearchableField, + SearchFieldDataType, + SearchIndex, + SearchIndexFieldReference, + SearchIndexKnowledgeSource, + SearchIndexKnowledgeSourceParameters, + SemanticConfiguration, + SemanticField, + SemanticPrioritizedFields, + SemanticSearch, + SimpleField, +) +from azure.search.documents.knowledgebases import KnowledgeBaseRetrievalClient +from azure.search.documents.knowledgebases.models import ( + KnowledgeBaseMessage, + KnowledgeBaseMessageTextContent, + KnowledgeBaseRetrievalRequest, + KnowledgeRetrievalLowReasoningEffort, + KnowledgeRetrievalOutputMode, + McpServerKnowledgeSourceParams, + SearchIndexKnowledgeSourceParams, +) + +from .config import DEFAULT_QUERY, Settings +from .trace import DualSourceEvidence, verify_dual_source_evidence + + +SEMANTIC_CONFIGURATION_NAME = "multi-source-semantic-config" + + +def build_index(index_name: str) -> SearchIndex: + return SearchIndex( + name=index_name, + fields=[ + SimpleField( + name="id", + type=SearchFieldDataType.String, + key=True, + filterable=True, + ), + SearchableField(name="title", type=SearchFieldDataType.String), + SearchableField(name="content", type=SearchFieldDataType.String), + SimpleField( + name="source_url", + type=SearchFieldDataType.String, + retrievable=True, + ), + ], + semantic_search=SemanticSearch( + default_configuration_name=SEMANTIC_CONFIGURATION_NAME, + configurations=[ + SemanticConfiguration( + name=SEMANTIC_CONFIGURATION_NAME, + prioritized_fields=SemanticPrioritizedFields( + title_field=SemanticField(field_name="title"), + content_fields=[SemanticField(field_name="content")], + ), + ) + ], + ), + ) + + +def build_search_knowledge_source(settings: Settings) -> SearchIndexKnowledgeSource: + return SearchIndexKnowledgeSource( + name=settings.search_knowledge_source_name, + description="Indexed fictional Contoso Trail Guide release facts.", + search_index_parameters=SearchIndexKnowledgeSourceParameters( + search_index_name=settings.index_name, + semantic_configuration_name=SEMANTIC_CONFIGURATION_NAME, + source_data_fields=[ + SearchIndexFieldReference(name="id"), + SearchIndexFieldReference(name="title"), + SearchIndexFieldReference(name="content"), + SearchIndexFieldReference(name="source_url"), + ], + ), + ) + + +def build_mcp_knowledge_source(settings: Settings) -> McpServerKnowledgeSource: + return McpServerKnowledgeSource( + name=settings.mcp_knowledge_source_name, + description="Live public Microsoft Learn documentation through MCP.", + mcp_server_parameters=McpServerKnowledgeSourceParameters( + server_url=settings.mcp_server_url, + tools=[ + McpServerTool( + name=settings.mcp_tool_name, + output_parsing=McpServerAutoOutputParsing(), + inclusion_mode="always", + max_output_tokens=3000, + ) + ], + ), + ) + + +def build_knowledge_base(settings: Settings) -> KnowledgeBase: + model_parameters: dict[str, Any] = { + "resource_url": settings.azure_openai_endpoint, + "deployment_name": settings.azure_openai_deployment, + "model_name": settings.azure_openai_model, + } + if settings.azure_openai_api_key: + model_parameters["api_key"] = settings.azure_openai_api_key + + return KnowledgeBase( + name=settings.knowledge_base_name, + description=( + "Combines indexed Contoso Trail release facts with live Microsoft " + "Learn guidance." + ), + models=[ + KnowledgeBaseAzureOpenAIModel( + azure_open_ai_parameters=AzureOpenAIVectorizerParameters( + **model_parameters + ) + ) + ], + knowledge_sources=[ + KnowledgeSourceReference( + name=settings.search_knowledge_source_name + ), + KnowledgeSourceReference( + name=settings.mcp_knowledge_source_name + ), + ], + retrieval_instructions=( + "For every request, retrieve the named Contoso release facts from " + "the Search index and current product guidance from the Microsoft " + "Learn MCP tool. For runtime questions, retrieve the MCP Server " + "knowledge source guide that documents maxRuntimeInSeconds, " + "activity, and references. Do not infer one source's facts from " + "the other." + ), + answer_instructions=( + "Answer concisely. Cite the indexed release brief for Contoso facts " + "and Microsoft Learn MCP references for product guidance." + ), + retrieval_reasoning_effort=KnowledgeRetrievalLowReasoningEffort(), + output_mode=KnowledgeRetrievalOutputMode.ANSWER_SYNTHESIS, + ) + + +def build_retrieval_request( + settings: Settings, + query: str = DEFAULT_QUERY, +) -> KnowledgeBaseRetrievalRequest: + return KnowledgeBaseRetrievalRequest( + messages=[ + KnowledgeBaseMessage( + role="user", + content=[KnowledgeBaseMessageTextContent(text=query)], + ) + ], + knowledge_source_params=[ + SearchIndexKnowledgeSourceParams( + knowledge_source_name=settings.search_knowledge_source_name, + include_references=True, + include_reference_source_data=True, + always_query_source=True, + fail_on_error=True, + reranker_threshold=0.0, + max_output_documents=50, + ), + McpServerKnowledgeSourceParams( + knowledge_source_name=settings.mcp_knowledge_source_name, + include_references=True, + include_reference_source_data=True, + fail_on_error=True, + reranker_threshold=0.0, + max_output_documents=50, + ), + ], + include_activity=True, + max_runtime_in_seconds=120, + retrieval_reasoning_effort=KnowledgeRetrievalLowReasoningEffort(), + output_mode=KnowledgeRetrievalOutputMode.ANSWER_SYNTHESIS, + ) + + +class MultiSourceSample: + def __init__( + self, + settings: Settings, + credential: AzureKeyCredential | TokenCredential, + ) -> None: + self.settings = settings + self.credential = credential + self._created_resources: list[dict[str, str]] = [] + self.index_client = SearchIndexClient( + endpoint=settings.search_endpoint, + credential=credential, + ) + + def setup(self) -> None: + self._ensure_names_available() + created_index = self.index_client.create_index( + build_index(self.settings.index_name) + ) + self._record_created( + "index", + self.settings.index_name, + created_index.e_tag, + ) + documents = _load_documents(self.settings.data_path) + with SearchClient( + endpoint=self.settings.search_endpoint, + index_name=self.settings.index_name, + credential=self.credential, + ) as search_client: + results = search_client.upload_documents(documents) + failed = [result for result in results if not result.succeeded] + if failed: + raise RuntimeError(f"Failed to index documents: {failed}") + _wait_for_documents(search_client, len(documents)) + + created_search_source = self.index_client.create_knowledge_source( + build_search_knowledge_source(self.settings) + ) + self._record_created( + "knowledge_source", + self.settings.search_knowledge_source_name, + created_search_source.e_tag, + ) + created_mcp_source = self.index_client.create_knowledge_source( + build_mcp_knowledge_source(self.settings) + ) + self._record_created( + "knowledge_source", + self.settings.mcp_knowledge_source_name, + created_mcp_source.e_tag, + ) + created_knowledge_base = self.index_client.create_knowledge_base( + build_knowledge_base(self.settings) + ) + self._record_created( + "knowledge_base", + self.settings.knowledge_base_name, + created_knowledge_base.e_tag, + ) + + def query(self, query: str = DEFAULT_QUERY) -> tuple[str, DualSourceEvidence]: + with KnowledgeBaseRetrievalClient( + endpoint=self.settings.search_endpoint, + knowledge_base_name=self.settings.knowledge_base_name, + credential=self.credential, + ) as client: + result = client.retrieve( + retrieval_request=build_retrieval_request(self.settings, query) + ) + + payload = result.as_dict() + evidence = verify_dual_source_evidence( + payload, + search_knowledge_source_name=( + self.settings.search_knowledge_source_name + ), + mcp_knowledge_source_name=self.settings.mcp_knowledge_source_name, + ) + answer = _extract_answer(payload) + if not answer: + raise RuntimeError("The knowledge base returned no synthesized answer.") + + output = { + "answer": answer, + "evidence": evidence.as_dict(), + "response": payload, + } + self.settings.output_path.parent.mkdir(parents=True, exist_ok=True) + self.settings.output_path.write_text( + json.dumps(output, indent=2), + encoding="utf-8", + ) + return answer, evidence + + def cleanup(self) -> None: + failures: list[str] = [] + resources = self._created_resources or self._load_manifest() + operations_by_kind = { + "knowledge_base": ( + self.index_client.get_knowledge_base, + self.index_client.delete_knowledge_base, + ), + "knowledge_source": ( + self.index_client.get_knowledge_source, + self.index_client.delete_knowledge_source, + ), + "index": ( + self.index_client.get_index, + self.index_client.delete_index, + ), + } + for resource in reversed(resources): + kind = resource["kind"] + name = resource["name"] + operations = operations_by_kind.get(kind) + if operations is None: + failures.append(f"unknown resource kind '{kind}' for '{name}'") + continue + get_resource, delete = operations + try: + current = get_resource(name) + if current.e_tag != resource.get("e_tag"): + failures.append( + f"{kind} '{name}' changed ownership or was modified; " + "cleanup refused to delete it" + ) + continue + delete( + current, + match_condition=MatchConditions.IfNotModified, + ) + except ResourceNotFoundError: + continue + except HttpResponseError as exc: + failures.append(f"{kind} '{name}': {exc.message}") + if failures: + raise RuntimeError("Cleanup failed:\n- " + "\n- ".join(failures)) + self._created_resources.clear() + self.settings.manifest_path.unlink(missing_ok=True) + + def _ensure_names_available(self) -> None: + if self.settings.manifest_path.exists(): + raise RuntimeError( + f"Resource ownership manifest already exists at " + f"{self.settings.manifest_path}. Run cleanup before setup." + ) + + checks = [ + ("index", self.settings.index_name, self.index_client.get_index), + ( + "Search knowledge source", + self.settings.search_knowledge_source_name, + self.index_client.get_knowledge_source, + ), + ( + "MCP knowledge source", + self.settings.mcp_knowledge_source_name, + self.index_client.get_knowledge_source, + ), + ( + "knowledge base", + self.settings.knowledge_base_name, + self.index_client.get_knowledge_base, + ), + ] + collisions: list[str] = [] + for kind, name, get_resource in checks: + try: + get_resource(name) + except ResourceNotFoundError: + continue + collisions.append(f"{kind} '{name}'") + if collisions: + raise RuntimeError( + "Setup stopped because these names already exist and aren't " + "owned by this run:\n- " + "\n- ".join(collisions) + ) + + def _record_created(self, kind: str, name: str, e_tag: str | None) -> None: + if not e_tag: + raise RuntimeError( + f"Created {kind} '{name}' without an ETag; refusing unsafe " + "name-only ownership tracking." + ) + self._created_resources.append( + {"kind": kind, "name": name, "e_tag": e_tag} + ) + self.settings.manifest_path.parent.mkdir(parents=True, exist_ok=True) + self.settings.manifest_path.write_text( + json.dumps( + { + "search_endpoint": self.settings.search_endpoint, + "resources": self._created_resources, + }, + indent=2, + ), + encoding="utf-8", + ) + + def _load_manifest(self) -> list[dict[str, str]]: + path = self.settings.manifest_path + if not path.exists(): + return [] + manifest = json.loads(path.read_text(encoding="utf-8")) + if manifest.get("search_endpoint") != self.settings.search_endpoint: + raise RuntimeError( + f"Ownership manifest endpoint doesn't match SEARCH_ENDPOINT: " + f"{path}" + ) + resources = manifest.get("resources") + if not isinstance(resources, list): + raise RuntimeError(f"Invalid resource ownership manifest: {path}") + if any( + not isinstance(resource, dict) + or not all(resource.get(key) for key in ("kind", "name", "e_tag")) + for resource in resources + ): + raise RuntimeError(f"Invalid resource ownership manifest: {path}") + return resources + + def close(self) -> None: + self.index_client.close() + close = getattr(self.credential, "close", None) + if close: + close() + + +def _load_documents(path: Path) -> list[dict[str, Any]]: + payload = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(payload, list) or not payload: + raise RuntimeError(f"Expected a non-empty JSON array in {path}") + return payload + + +def _wait_for_documents( + search_client: SearchClient, + expected_count: int, + timeout_seconds: int = 60, +) -> None: + deadline = time.monotonic() + timeout_seconds + while time.monotonic() < deadline: + if search_client.get_document_count() >= expected_count: + return + time.sleep(2) + raise TimeoutError( + f"Index did not report {expected_count} documents within " + f"{timeout_seconds} seconds." + ) + + +def _extract_answer(payload: dict[str, Any]) -> str: + parts: list[str] = [] + for message in payload.get("response", []): + if not isinstance(message, dict): + continue + for content in message.get("content", []): + if isinstance(content, dict) and content.get("text"): + parts.append(str(content["text"])) + return "\n\n".join(parts) diff --git a/Foundry-IQ-Multi-Source/src/foundry_iq_multi_source/trace.py b/Foundry-IQ-Multi-Source/src/foundry_iq_multi_source/trace.py new file mode 100644 index 000000000..b73cd8124 --- /dev/null +++ b/Foundry-IQ-Multi-Source/src/foundry_iq_multi_source/trace.py @@ -0,0 +1,152 @@ +from __future__ import annotations + +from dataclasses import asdict, dataclass +from typing import Any + + +@dataclass(frozen=True) +class SourceEvidence: + knowledge_source_name: str + activity_count: int + reference_count: int + activity_types: tuple[str, ...] + reference_types: tuple[str, ...] + + @property + def participated(self) -> bool: + return self.activity_count > 0 and self.reference_count > 0 + + +@dataclass(frozen=True) +class DualSourceEvidence: + search: SourceEvidence + mcp: SourceEvidence + + @property + def both_participated(self) -> bool: + return self.search.participated and self.mcp.participated + + def as_dict(self) -> dict[str, Any]: + return { + **asdict(self), + "both_participated": self.both_participated, + } + + +def verify_dual_source_evidence( + response: dict[str, Any], + *, + search_knowledge_source_name: str, + mcp_knowledge_source_name: str, +) -> DualSourceEvidence: + """Require both activity and references evidence for both knowledge sources.""" + activities = _list(response, "activity", "activity_contents") + references = _list(response, "references", "references_contents") + activity_by_id: dict[str, dict[str, Any]] = {} + activity_sources: dict[str, list[dict[str, Any]]] = { + search_knowledge_source_name: [], + mcp_knowledge_source_name: [], + } + + for activity in activities: + activity_id = _value(activity, "id") + if activity_id is not None: + activity_by_id[str(activity_id)] = activity + source_name = _value( + activity, + "knowledgeSourceName", + "knowledge_source_name", + ) + if source_name in activity_sources: + activity_sources[source_name].append(activity) + + reference_sources: dict[str, list[dict[str, Any]]] = { + search_knowledge_source_name: [], + mcp_knowledge_source_name: [], + } + for reference in references: + source_name = _value( + reference, + "knowledgeSourceName", + "knowledge_source_name", + ) + if source_name not in reference_sources: + activity_source = _value( + reference, + "activitySource", + "activity_source", + ) + linked_activity = activity_by_id.get(str(activity_source)) + if linked_activity: + source_name = _value( + linked_activity, + "knowledgeSourceName", + "knowledge_source_name", + ) + if source_name in reference_sources: + reference_sources[source_name].append(reference) + + evidence = DualSourceEvidence( + search=_source_evidence( + search_knowledge_source_name, + activity_sources, + reference_sources, + ), + mcp=_source_evidence( + mcp_knowledge_source_name, + activity_sources, + reference_sources, + ), + ) + if not evidence.both_participated: + raise RuntimeError( + "Cross-source verification failed. Expected at least one activity " + "record and one linked reference from each knowledge source. " + f"Observed: {evidence.as_dict()}" + ) + return evidence + + +def _source_evidence( + name: str, + activity_sources: dict[str, list[dict[str, Any]]], + reference_sources: dict[str, list[dict[str, Any]]], +) -> SourceEvidence: + activities = activity_sources[name] + references = reference_sources[name] + return SourceEvidence( + knowledge_source_name=name, + activity_count=len(activities), + reference_count=len(references), + activity_types=tuple( + sorted( + { + str(_value(item, "type", "kind") or "unknown") + for item in activities + } + ) + ), + reference_types=tuple( + sorted( + { + str(_value(item, "type", "kind") or "unknown") + for item in references + } + ) + ), + ) + + +def _list(response: dict[str, Any], *names: str) -> list[dict[str, Any]]: + for name in names: + value = response.get(name) + if isinstance(value, list): + return [item for item in value if isinstance(item, dict)] + return [] + + +def _value(item: dict[str, Any], *names: str) -> Any: + for name in names: + if name in item: + return item[name] + return None diff --git a/Foundry-IQ-Multi-Source/tests/test_cleanup_safety.py b/Foundry-IQ-Multi-Source/tests/test_cleanup_safety.py new file mode 100644 index 000000000..7d44d3779 --- /dev/null +++ b/Foundry-IQ-Multi-Source/tests/test_cleanup_safety.py @@ -0,0 +1,153 @@ +import json +from types import SimpleNamespace + +import pytest +from azure.core import MatchConditions +from foundry_iq_multi_source.config import Settings +from foundry_iq_multi_source.sample import MultiSourceSample + + +class FakeIndexClient: + def __init__(self) -> None: + self.deleted: list[tuple[str, str, MatchConditions]] = [] + self.resources = { + ("index", "sample-owned-index"): SimpleNamespace( + name="sample-owned-index", + e_tag='"owned-etag"', + ) + } + + def get_index(self, name: str): + return self.resources[("index", name)] + + def get_knowledge_source(self, name: str): + return self.resources[("knowledge_source", name)] + + def get_knowledge_base(self, name: str): + return self.resources[("knowledge_base", name)] + + def delete_index(self, resource, *, match_condition) -> None: + self.deleted.append(("index", resource.name, match_condition)) + + def delete_knowledge_source(self, resource, *, match_condition) -> None: + self.deleted.append( + ("knowledge_source", resource.name, match_condition) + ) + + def delete_knowledge_base(self, resource, *, match_condition) -> None: + self.deleted.append( + ("knowledge_base", resource.name, match_condition) + ) + + +def settings() -> Settings: + return Settings( + search_endpoint="https://example.search.windows.net", + azure_openai_endpoint="https://example.openai.azure.com", + azure_openai_deployment="gpt-5-mini", + azure_openai_model="gpt-5-mini", + ) + + +def sample_with_fake_client( + config: Settings, + client: FakeIndexClient, +) -> MultiSourceSample: + sample = object.__new__(MultiSourceSample) + sample.settings = config + sample.credential = object() + sample._created_resources = [] + sample.index_client = client + return sample + + +def test_cleanup_without_manifest_deletes_nothing( + monkeypatch, + tmp_path, +) -> None: + manifest_path = tmp_path / "resources.json" + monkeypatch.setattr( + Settings, + "manifest_path", + property(lambda self: manifest_path), + ) + client = FakeIndexClient() + + sample_with_fake_client(settings(), client).cleanup() + + assert client.deleted == [] + + +def test_cleanup_deletes_only_manifest_owned_resources( + monkeypatch, + tmp_path, +) -> None: + manifest_path = tmp_path / "resources.json" + monkeypatch.setattr( + Settings, + "manifest_path", + property(lambda self: manifest_path), + ) + config = settings() + manifest_path.write_text( + json.dumps( + { + "search_endpoint": config.search_endpoint, + "resources": [ + { + "kind": "index", + "name": "sample-owned-index", + "e_tag": "\"owned-etag\"", + } + ], + } + ), + encoding="utf-8", + ) + client = FakeIndexClient() + + sample_with_fake_client(config, client).cleanup() + + assert client.deleted == [ + ( + "index", + "sample-owned-index", + MatchConditions.IfNotModified, + ) + ] + assert not manifest_path.exists() + + +def test_cleanup_refuses_replacement_with_different_etag( + monkeypatch, + tmp_path, +) -> None: + manifest_path = tmp_path / "resources.json" + monkeypatch.setattr( + Settings, + "manifest_path", + property(lambda self: manifest_path), + ) + config = settings() + manifest_path.write_text( + json.dumps( + { + "search_endpoint": config.search_endpoint, + "resources": [ + { + "kind": "index", + "name": "sample-owned-index", + "e_tag": "\"original-etag\"", + } + ], + } + ), + encoding="utf-8", + ) + client = FakeIndexClient() + + with pytest.raises(RuntimeError, match="refused to delete"): + sample_with_fake_client(config, client).cleanup() + + assert client.deleted == [] + assert manifest_path.exists() diff --git a/Foundry-IQ-Multi-Source/tests/test_payloads.py b/Foundry-IQ-Multi-Source/tests/test_payloads.py new file mode 100644 index 000000000..0842ed788 --- /dev/null +++ b/Foundry-IQ-Multi-Source/tests/test_payloads.py @@ -0,0 +1,80 @@ +from azure.search.documents.knowledgebases.models import ( + McpServerKnowledgeSourceParams, + SearchIndexKnowledgeSourceParams, +) + +from foundry_iq_multi_source.config import Settings +from foundry_iq_multi_source.sample import ( + build_knowledge_base, + build_mcp_knowledge_source, + build_retrieval_request, + build_search_knowledge_source, +) + + +def settings() -> Settings: + return Settings( + search_endpoint="https://example.search.windows.net", + azure_openai_endpoint="https://example.openai.azure.com", + azure_openai_deployment="gpt-5-mini", + azure_openai_model="gpt-5-mini", + ) + + +def test_knowledge_base_references_both_sources() -> None: + config = settings() + payload = build_knowledge_base(config).as_dict() + + assert payload["knowledgeSources"] == [ + {"name": config.search_knowledge_source_name}, + {"name": config.mcp_knowledge_source_name}, + ] + assert payload["outputMode"] == "answerSynthesis" + assert "apiKey" not in payload["models"][0]["azureOpenAIParameters"] + + +def test_search_index_knowledge_source_uses_semantic_config() -> None: + payload = build_search_knowledge_source(settings()).as_dict() + + assert payload["kind"] == "searchIndex" + assert payload["searchIndexParameters"]["semanticConfigurationName"] + + +def test_mcp_knowledge_source_forces_tool_output_inclusion() -> None: + payload = build_mcp_knowledge_source(settings()).as_dict() + tool = payload["mcpServerParameters"]["tools"][0] + + assert payload["kind"] == "mcpServer" + assert payload["mcpServerParameters"]["serverURL"].startswith("https://") + assert tool["name"] == "microsoft_docs_search" + assert tool["inclusionMode"] == "always" + assert tool["outputParsing"] == {"kind": "auto"} + + +def test_retrieval_request_forces_only_the_search_source() -> None: + request = build_retrieval_request(settings()).as_dict() + source_params = request["knowledgeSourceParams"] + search = next( + item for item in source_params if item["kind"] == "searchIndex" + ) + mcp = next(item for item in source_params if item["kind"] == "mcpServer") + + assert request["includeActivity"] is True + assert request["maxRuntimeInSeconds"] == 120 + assert "maxRuntimeInSeconds" in request["messages"][0]["content"][0]["text"] + assert search["alwaysQuerySource"] is True + assert search["rerankerThreshold"] == 0.0 + assert search["maxOutputDocuments"] == 50 + assert "alwaysQuerySource" not in mcp + assert mcp["rerankerThreshold"] == 0.0 + assert mcp["maxOutputDocuments"] == 50 + assert all(item["includeReferences"] for item in source_params) + assert all(item["includeReferenceSourceData"] for item in source_params) + assert isinstance( + build_retrieval_request(settings()).knowledge_source_params[0], + SearchIndexKnowledgeSourceParams, + ) + assert isinstance( + build_retrieval_request(settings()).knowledge_source_params[1], + McpServerKnowledgeSourceParams, + ) diff --git a/Foundry-IQ-Multi-Source/tests/test_sample_data.py b/Foundry-IQ-Multi-Source/tests/test_sample_data.py new file mode 100644 index 000000000..79bd7424c --- /dev/null +++ b/Foundry-IQ-Multi-Source/tests/test_sample_data.py @@ -0,0 +1,18 @@ +import json +from pathlib import Path + + +SAMPLE_ROOT = Path(__file__).resolve().parents[1] + + +def test_sample_data_is_public_and_fictional() -> None: + documents = json.loads( + (SAMPLE_ROOT / "data" / "release-brief.json").read_text( + encoding="utf-8" + ) + ) + + assert len(documents) >= 2 + assert all(document["title"].startswith("Contoso") for document in documents) + ignored = (SAMPLE_ROOT / ".gitignore").read_text(encoding="utf-8").splitlines() + assert ".env" in ignored diff --git a/Foundry-IQ-Multi-Source/tests/test_trace.py b/Foundry-IQ-Multi-Source/tests/test_trace.py new file mode 100644 index 000000000..dcd6d0fdd --- /dev/null +++ b/Foundry-IQ-Multi-Source/tests/test_trace.py @@ -0,0 +1,63 @@ +import pytest + +from foundry_iq_multi_source.trace import verify_dual_source_evidence + + +SEARCH_SOURCE = "foundry-iq-search-source" +MCP_SOURCE = "foundry-iq-learn-mcp-source" + + +def response_with_both_sources() -> dict: + return { + "activity": [ + { + "id": 1, + "type": "searchIndex", + "knowledgeSourceName": SEARCH_SOURCE, + }, + { + "id": 2, + "type": "mcpServer", + "knowledgeSourceName": MCP_SOURCE, + "mcpServerArguments": {"toolName": "microsoft_docs_search"}, + }, + ], + "references": [ + { + "id": "0", + "type": "searchIndex", + "activitySource": 1, + "sourceData": {"id": "trail-release-wave-b"}, + }, + { + "id": "1", + "type": "mcpServer", + "activitySource": 2, + "sourceData": {"title": "MCP Server knowledge source"}, + }, + ], + } + + +def test_verifier_confirms_activity_and_references_for_both_sources() -> None: + evidence = verify_dual_source_evidence( + response_with_both_sources(), + search_knowledge_source_name=SEARCH_SOURCE, + mcp_knowledge_source_name=MCP_SOURCE, + ) + + assert evidence.both_participated + assert evidence.search.reference_count == 1 + assert evidence.mcp.reference_count == 1 + + +def test_verifier_rejects_activity_without_linked_reference() -> None: + response = response_with_both_sources() + response["references"] = response["references"][:1] + + with pytest.raises(RuntimeError, match="Cross-source verification failed"): + verify_dual_source_evidence( + response, + search_knowledge_source_name=SEARCH_SOURCE, + mcp_knowledge_source_name=MCP_SOURCE, + ) diff --git a/README.md b/README.md index e2ce61797..13ffd2302 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ If your configuration uses a search service managed identity for indexer connect | Sample | Description | |--------|-------------| | [Quickstart-Agentic-Retrieval](Quickstart-Agentic-Retrieval/quickstart-agentic-retrieval.ipynb) | Create a knowledge base in Azure AI Search to integrate LLM reasoning into query planning. | +| [Foundry-IQ-Multi-Source](Foundry-IQ-Multi-Source/README.md) | Query one Foundry IQ knowledge base across an Azure AI Search index and an external MCP server, with trace verification that both sources contributed. | | [Quickstart-Document-Permissions-Pull-API](Quickstart-Document-Permissions-Pull-API/document-permissions-pull-api.ipynb) | Using an indexer "pull API" approach, flow access control lists from a data source to search results and apply permission filters that restrict access to authorized content. Indexer support is limited to Azure Data Lake Storage (ADLS) Gen2 permission metadata. | | [Quickstart-Document-Permissions-Push-API](Quickstart-Document-Permissions-Push-API/document-permissions-push-api.ipynb) | Using the push APIs for indexing a JSON payload, flow embedded permission metadata to indexed documents and search results that are filtered based on user access to authorized content. | | [Quickstart-Keyword-Search](Quickstart-Keyword-Search/README.md) | Introduces the fundamental tasks of working with a classic search index: create, load, and query. The index is modeled on a subset of the hotels dataset, which is widely used in Azure AI Search samples but reduced in this sample for readability and comprehension. |