From 62cf5ebabf7bf83f1b42fe7c8708ce5611ca8a3b Mon Sep 17 00:00:00 2001 From: Khushmeet Chandi Date: Fri, 17 Jul 2026 15:56:13 -0700 Subject: [PATCH 1/4] [azure-ai-evaluation] Fix README code samples, typos, and stale links Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 77e272d0-4d0f-42cf-b18b-50a5ab5b60e5 --- sdk/evaluation/azure-ai-evaluation/README.md | 50 ++++++++++---------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/README.md b/sdk/evaluation/azure-ai-evaluation/README.md index 7d8c42521ced..267d87acf095 100644 --- a/sdk/evaluation/azure-ai-evaluation/README.md +++ b/sdk/evaluation/azure-ai-evaluation/README.md @@ -58,7 +58,7 @@ from azure.ai.evaluation import evaluate, RelevanceEvaluator, ViolenceEvaluator, # NLP bleu score evaluator bleu_score_evaluator = BleuScoreEvaluator() -result = bleu_score( +result = bleu_score_evaluator( response="Tokyo is the capital of Japan.", ground_truth="The capital of Japan is Tokyo." ) @@ -66,7 +66,7 @@ result = bleu_score( # AI assisted quality evaluator model_config = { "azure_endpoint": os.environ.get("AZURE_OPENAI_ENDPOINT"), - "api_key": os.environ.get("AZURE_OPENAI_API_KEY"), + "api_key": os.environ.get("AZURE_OPENAI_KEY"), "azure_deployment": os.environ.get("AZURE_OPENAI_DEPLOYMENT"), } @@ -116,13 +116,13 @@ class BlocklistEvaluator: self._blocklist = blocklist def __call__(self, *, response: str, **kwargs): - score = any([word in answer for word in self._blocklist]) + score = any([word in response for word in self._blocklist]) return {"score": score} blocklist_evaluator = BlocklistEvaluator(blocklist=["bad, worst, terrible"]) result = response_length("The capital of Japan is Tokyo.") -result = blocklist_evaluator(answer="The capital of Japan is Tokyo.") +result = blocklist_evaluator(response="The capital of Japan is Tokyo.") ``` @@ -144,12 +144,12 @@ result = evaluate( evaluator_config={ "relevance": { "column_mapping": { - "query": "${data.queries}" - "ground_truth": "${data.ground_truth}" + "query": "${data.queries}", + "ground_truth": "${data.ground_truth}", "response": "${outputs.response}" } } - } + }, # Optionally provide your AI Foundry project information to track your evaluation results in your Azure AI Foundry project azure_ai_project = azure_ai_project, # Optionally provide an output path to dump a json of metric summary, row level data and metric and AI Foundry URL @@ -171,8 +171,8 @@ result = evaluate( evaluator_config={ "default": { "column_mapping": { - "query": "${data.queries}" - "context": "${outputs.context}" + "query": "${data.queries}", + "context": "${outputs.context}", "response": "${outputs.response}" } } @@ -186,7 +186,7 @@ For more details refer to [Evaluate on a target][evaluate_target] ### Simulator -Simulators allow users to generate synthentic data using their application. Simulator expects the user to have a callback method that invokes their AI application. The intergration between your AI application and the simulator happens at the callback method. Here's how a sample callback would look like: +Simulators allow users to generate synthetic data using their application. Simulator expects the user to have a callback method that invokes their AI application. The integration between your AI application and the simulator happens at the callback method. Here's how a sample callback would look like: ```python @@ -216,9 +216,9 @@ The simulator initialization and invocation looks like this: ```python from azure.ai.evaluation.simulator import Simulator model_config = { - "azure_endpoint": os.environ.get("AZURE_ENDPOINT"), - "azure_deployment": os.environ.get("AZURE_DEPLOYMENT_NAME"), - "api_version": os.environ.get("AZURE_API_VERSION"), + "azure_endpoint": os.environ.get("AZURE_OPENAI_ENDPOINT"), + "api_key": os.environ.get("AZURE_OPENAI_KEY"), + "azure_deployment": os.environ.get("AZURE_OPENAI_DEPLOYMENT"), } custom_simulator = Simulator(model_config=model_config) outputs = asyncio.run(custom_simulator( @@ -323,7 +323,7 @@ This project has adopted the [Microsoft Open Source Code of Conduct][code_of_con [evaluation_pypi]: https://pypi.org/project/azure-ai-evaluation/ [evaluation_ref_docs]: https://learn.microsoft.com/python/api/azure-ai-evaluation/azure.ai.evaluation?view=azure-python-preview [evaluation_samples]: https://github.com/Azure-Samples/azureai-samples/tree/main/scenarios -[product_documentation]: https://learn.microsoft.com/azure/ai-studio/how-to/develop/evaluate-sdk +[product_documentation]: https://learn.microsoft.com/azure/foundry-classic/how-to/develop/evaluate-sdk [python_logging]: https://docs.python.org/3/library/logging.html [sdk_logging_docs]: https://docs.microsoft.com/azure/developer/python/azure-sdk-logging [azure_core_readme]: https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/core/azure-core/README.md @@ -335,24 +335,24 @@ This project has adopted the [Microsoft Open Source Code of Conduct][code_of_con [code_of_conduct]: https://opensource.microsoft.com/codeofconduct/ [coc_faq]: https://opensource.microsoft.com/codeofconduct/faq/ [coc_contact]: mailto:opencode@microsoft.com -[evaluate_target]: https://learn.microsoft.com/azure/ai-studio/how-to/develop/evaluate-sdk#evaluate-on-a-target -[evaluate_dataset]: https://learn.microsoft.com/azure/ai-studio/how-to/develop/evaluate-sdk#evaluate-on-test-dataset-using-evaluate +[evaluate_target]: https://learn.microsoft.com/azure/foundry-classic/how-to/develop/evaluate-sdk#local-evaluation-on-a-target +[evaluate_dataset]: https://learn.microsoft.com/azure/foundry-classic/how-to/develop/evaluate-sdk#local-evaluation-on-test-datasets-using-evaluate [evaluators]: https://learn.microsoft.com/python/api/azure-ai-evaluation/azure.ai.evaluation?view=azure-python-preview [evaluate_api]: https://learn.microsoft.com/python/api/azure-ai-evaluation/azure.ai.evaluation?view=azure-python-preview#azure-ai-evaluation-evaluate [evaluate_app]: https://github.com/Azure-Samples/azureai-samples/tree/main/scenarios/evaluate/Supported_Evaluation_Targets/Evaluate_App_Endpoint [evaluation_tsg]: https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/evaluation/azure-ai-evaluation/TROUBLESHOOTING.md -[ai_studio]: https://learn.microsoft.com/azure/ai-studio/what-is-ai-studio -[ai_project]: https://learn.microsoft.com/azure/ai-studio/how-to/create-projects?tabs=ai-studio +[ai_studio]: https://learn.microsoft.com/azure/foundry/what-is-foundry +[ai_project]: https://learn.microsoft.com/azure/foundry/how-to/create-projects [azure_openai]: https://learn.microsoft.com/azure/ai-services/openai/ [evaluate_models]: https://github.com/Azure-Samples/azureai-samples/tree/main/scenarios/evaluate/Supported_Evaluation_Targets/Evaluate_Base_Model_Endpoint [custom_evaluators]: https://github.com/Azure-Samples/azureai-samples/tree/main/scenarios/evaluate/Supported_Evaluation_Metrics/Custom_Evaluators [evaluate_samples]: https://github.com/Azure-Samples/azureai-samples/tree/main/scenarios/evaluate -[evaluation_metrics]: https://learn.microsoft.com/azure/ai-studio/concepts/evaluation-metrics-built-in -[performance_and_quality_evaluators]: https://learn.microsoft.com/azure/ai-studio/how-to/develop/evaluate-sdk#performance-and-quality-evaluators -[risk_and_safety_evaluators]: https://learn.microsoft.com/azure/ai-studio/how-to/develop/evaluate-sdk#risk-and-safety-evaluators -[composite_evaluators]: https://learn.microsoft.com/azure/ai-studio/how-to/develop/evaluate-sdk#composite-evaluators -[adversarial_simulation_docs]: https://learn.microsoft.com/azure/ai-studio/how-to/develop/simulator-interaction-data#generate-adversarial-simulations-for-safety-evaluation -[adversarial_simulation_scenarios]: https://learn.microsoft.com/azure/ai-studio/how-to/develop/simulator-interaction-data#supported-adversarial-simulation-scenarios +[evaluation_metrics]: https://learn.microsoft.com/azure/foundry/concepts/built-in-evaluators +[performance_and_quality_evaluators]: https://learn.microsoft.com/azure/foundry-classic/how-to/develop/evaluate-sdk#built-in-evaluators +[risk_and_safety_evaluators]: https://learn.microsoft.com/azure/foundry-classic/concepts/evaluation-evaluators/risk-safety-evaluators +[composite_evaluators]: https://learn.microsoft.com/azure/foundry-classic/how-to/develop/evaluate-sdk#composite-evaluators +[adversarial_simulation_docs]: https://learn.microsoft.com/azure/foundry-classic/how-to/develop/simulator-interaction-data#generate-adversarial-simulations-for-safety-evaluation +[adversarial_simulation_scenarios]: https://learn.microsoft.com/azure/foundry-classic/how-to/develop/simulator-interaction-data#supported-adversarial-simulation-scenarios [adversarial_simulation]: https://github.com/Azure-Samples/azureai-samples/tree/main/scenarios/evaluate/Simulators/Simulate_Adversarial_Data [simulate_with_conversation_starter]: https://github.com/Azure-Samples/azureai-samples/tree/main/scenarios/evaluate/Simulators/Simulate_Context-Relevant_Data/Simulate_From_Conversation_Starter -[adversarial_jailbreak]: https://learn.microsoft.com/azure/ai-studio/how-to/develop/simulator-interaction-data#simulating-jailbreak-attacks +[adversarial_jailbreak]: https://learn.microsoft.com/azure/foundry-classic/how-to/develop/simulator-interaction-data#simulate-jailbreak-attacks From eef5187f42dafe0fc6e28158932d831e10e9efd0 Mon Sep 17 00:00:00 2001 From: Khushmeet Chandi Date: Fri, 17 Jul 2026 15:56:14 -0700 Subject: [PATCH 2/4] [azure-ai-evaluation] Fix ContentSafetyEvaluator syntax in migration guide Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 77e272d0-4d0f-42cf-b18b-50a5ab5b60e5 --- sdk/evaluation/azure-ai-evaluation/migration_guide.md | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/evaluation/azure-ai-evaluation/migration_guide.md b/sdk/evaluation/azure-ai-evaluation/migration_guide.md index 90e9ca2877ae..0fd2949a2221 100644 --- a/sdk/evaluation/azure-ai-evaluation/migration_guide.md +++ b/sdk/evaluation/azure-ai-evaluation/migration_guide.md @@ -88,6 +88,7 @@ content_safety_eval = ContentSafetyEvaluator(azure_cred, project_scope) content_safety_score = content_safety_eval( query="What is the capital of Japan?", response="The capital of Japan is Tokyo." +) pprint(content_safety_score) # Using Chat conversation From e91a9cb0510b81a759528283e1d4668a0d52fe5e Mon Sep 17 00:00:00 2001 From: Khushmeet Chandi Date: Fri, 17 Jul 2026 15:56:16 -0700 Subject: [PATCH 3/4] [azure-ai-evaluation] Update AI Studio hub -> AI Foundry hub Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 77e272d0-4d0f-42cf-b18b-50a5ab5b60e5 --- sdk/evaluation/azure-ai-evaluation/TROUBLESHOOTING.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/TROUBLESHOOTING.md b/sdk/evaluation/azure-ai-evaluation/TROUBLESHOOTING.md index 715a7152b474..8acc6613927c 100644 --- a/sdk/evaluation/azure-ai-evaluation/TROUBLESHOOTING.md +++ b/sdk/evaluation/azure-ai-evaluation/TROUBLESHOOTING.md @@ -28,11 +28,11 @@ This guide walks you through how to investigate failures, common errors in the ` - Before running `evaluate()`, to ensure that you can enable logging and tracing to your Azure AI project, make sure you are first logged in by running `az login`. -- Ensure that you assign the proper permissions to the storage account linked to your Azure AI Studio hub. This can be done with the following command. More information can be found [here](https://aka.ms/credentialleshub). +- Ensure that you assign the proper permissions to the storage account linked to your Azure AI Foundry hub. This can be done with the following command. More information can be found [here](https://aka.ms/credentialleshub). ```Shell - # : Subscription ID of the Azure AI Studio hub's linked storage account (available in Azure AI hub resource view in Azure Portal). - # : Resource group of the Azure AI Studio hub's linked storage account. + # : Subscription ID of the Azure AI Foundry hub's linked storage account (available in Azure AI hub resource view in Azure Portal). + # : Resource group of the Azure AI Foundry hub's linked storage account. # : User object ID for role assignment (retrieve with "az ad user show" command). az role assignment create --role "Storage Blob Data Contributor" --scope /subscriptions//resourceGroups/ --assignee-principal-type User --assignee-object-id "" From 5cf1fd251a44e57f37491895b9e250f92e31b3e7 Mon Sep 17 00:00:00 2001 From: Khushmeet Chandi Date: Fri, 17 Jul 2026 16:04:56 -0700 Subject: [PATCH 4/4] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- sdk/evaluation/azure-ai-evaluation/TROUBLESHOOTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/evaluation/azure-ai-evaluation/TROUBLESHOOTING.md b/sdk/evaluation/azure-ai-evaluation/TROUBLESHOOTING.md index 8acc6613927c..5574a0cf5d09 100644 --- a/sdk/evaluation/azure-ai-evaluation/TROUBLESHOOTING.md +++ b/sdk/evaluation/azure-ai-evaluation/TROUBLESHOOTING.md @@ -32,7 +32,7 @@ This guide walks you through how to investigate failures, common errors in the ` ```Shell # : Subscription ID of the Azure AI Foundry hub's linked storage account (available in Azure AI hub resource view in Azure Portal). - # : Resource group of the Azure AI Foundry hub's linked storage account. + # : Resource group of the Microsoft Foundry hub's linked storage account. # : User object ID for role assignment (retrieve with "az ad user show" command). az role assignment create --role "Storage Blob Data Contributor" --scope /subscriptions//resourceGroups/ --assignee-principal-type User --assignee-object-id ""