From 46653d07e3853281cfc0549f8ab00b586b72ed80 Mon Sep 17 00:00:00 2001 From: Jan Jones Date: Tue, 11 Aug 2026 11:17:48 +0200 Subject: [PATCH 01/14] Restore during source index build --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 90c4222beda..b40a5f4032a 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -132,7 +132,7 @@ variables: - name: enableSourceIndex value: false - name: sourceIndexBuildCommand - value: $(Build.SourcesDirectory)/build.cmd -ci -NativeToolsOnMachine + value: $(Build.SourcesDirectory)/build.cmd -restore -ci -NativeToolsOnMachine resources: repositories: From 5c72f529cc2992c7fc1279bde5577bf23ddeffd4 Mon Sep 17 00:00:00 2001 From: Jan Jones Date: Tue, 11 Aug 2026 11:27:47 +0200 Subject: [PATCH 02/14] Upload complog for source index --- .../steps/source-index-stage1-publish.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/eng/common/core-templates/steps/source-index-stage1-publish.yml b/eng/common/core-templates/steps/source-index-stage1-publish.yml index 6e7666b4dcf..0b24a3c7874 100644 --- a/eng/common/core-templates/steps/source-index-stage1-publish.yml +++ b/eng/common/core-templates/steps/source-index-stage1-publish.yml @@ -1,7 +1,8 @@ parameters: - sourceIndexUploadPackageVersion: 2.0.0-20250818.1 - sourceIndexProcessBinlogPackageVersion: 1.0.1-20250818.1 + sourceIndexUploadPackageVersion: '2.0.0-20250818.1' + sourceIndexComplogPackageVersion: '0.9.50' sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json + sourceIndexPublicPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/index.json binlogPath: artifacts/log/Debug/Build.binlog steps: @@ -14,14 +15,14 @@ steps: workingDirectory: $(Agent.TempDirectory) - script: | - $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version ${{parameters.sourceIndexProcessBinlogPackageVersion}} --source ${{parameters.SourceIndexPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools - $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version ${{parameters.sourceIndexUploadPackageVersion}} --source ${{parameters.SourceIndexPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools - displayName: "Source Index: Download netsourceindex Tools" + $(Agent.TempDirectory)/dotnet/dotnet tool install complog --version ${{parameters.sourceIndexComplogPackageVersion}} --source ${{parameters.sourceIndexPublicPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version ${{parameters.sourceIndexUploadPackageVersion}} --source ${{parameters.sourceIndexPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools + displayName: "Source Index: Download Tools" # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. workingDirectory: $(Agent.TempDirectory) -- script: $(Agent.TempDirectory)/.source-index/tools/BinLogToSln -i ${{parameters.BinlogPath}} -r $(System.DefaultWorkingDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output - displayName: "Source Index: Process Binlog into indexable sln" +- script: $(Agent.TempDirectory)/.source-index/tools/complog create ${{parameters.BinlogPath}} -o .source-index/stage1output + displayName: "Source Index: Process Binlog into Complog" - ${{ if and(ne(parameters.runAsPublic, 'true'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - task: AzureCLI@2 From a021f3678d23fcb99f45653847e8fb40dc1e537c Mon Sep 17 00:00:00 2001 From: Jan Jones Date: Tue, 11 Aug 2026 11:29:22 +0200 Subject: [PATCH 03/14] Allow force-enabling source index --- azure-pipelines.yml | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b40a5f4032a..75c2e537d4f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -125,12 +125,16 @@ variables: value: true # Enable source index only for main branch builds - - ${{ if and(ne(variables['runAsPublic'], 'true'), notin(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.SourceBranch'], 'refs/heads/main')) }}: - - name: enableSourceIndex - value: true - - ${{ else }}: - - name: enableSourceIndex - value: false + - name: sourceIndexCondition + value: | + or( + eq(variables['forceEnableSourceIndex'], 'true'), + and( + ne(variables['runAsPublic'], 'true'), + notin(variables['Build.Reason'], 'PullRequest'), + eq(variables['Build.SourceBranch'], 'refs/heads/main') + ) + ) - name: sourceIndexBuildCommand value: $(Build.SourcesDirectory)/build.cmd -restore -ci -NativeToolsOnMachine @@ -175,8 +179,9 @@ extends: manifests: true enableMicrobuild: true enableTelemetry: true - enableSourceIndex: ${{ variables['enableSourceIndex'] }} + enableSourceIndex: true sourceIndexParams: + condition: ${{ variables['sourceIndexCondition'] }} sourceIndexBuildCommand: ${{ variables['sourceIndexBuildCommand'] }} runAsPublic: ${{ variables['runAsPublic'] }} # Publish test logs From 1054f1a6225f109fa6cfb5784c8409e652a10946 Mon Sep 17 00:00:00 2001 From: Jan Jones Date: Tue, 11 Aug 2026 11:48:48 +0200 Subject: [PATCH 04/14] Avoid failing official builds if source indexing fails --- eng/common/core-templates/job/source-index-stage1.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/common/core-templates/job/source-index-stage1.yml b/eng/common/core-templates/job/source-index-stage1.yml index 76baf5c2725..46d5d4703ea 100644 --- a/eng/common/core-templates/job/source-index-stage1.yml +++ b/eng/common/core-templates/job/source-index-stage1.yml @@ -12,6 +12,7 @@ jobs: - job: SourceIndexStage1 dependsOn: ${{ parameters.dependsOn }} condition: ${{ parameters.condition }} + continueOnError: true variables: - name: BinlogPath value: ${{ parameters.binlogPath }} From 548fdb522bc3ec369188bcec8a4ed6d64713280f Mon Sep 17 00:00:00 2001 From: Jan Jones Date: Tue, 11 Aug 2026 12:02:19 +0200 Subject: [PATCH 05/14] Fix complog output --- eng/common/core-templates/steps/source-index-stage1-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/core-templates/steps/source-index-stage1-publish.yml b/eng/common/core-templates/steps/source-index-stage1-publish.yml index 0b24a3c7874..3e65ff6ee5d 100644 --- a/eng/common/core-templates/steps/source-index-stage1-publish.yml +++ b/eng/common/core-templates/steps/source-index-stage1-publish.yml @@ -21,7 +21,7 @@ steps: # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. workingDirectory: $(Agent.TempDirectory) -- script: $(Agent.TempDirectory)/.source-index/tools/complog create ${{parameters.BinlogPath}} -o .source-index/stage1output +- script: $(Agent.TempDirectory)/.source-index/tools/complog create ${{parameters.BinlogPath}} -o .source-index/stage1output/build.complog displayName: "Source Index: Process Binlog into Complog" - ${{ if and(ne(parameters.runAsPublic, 'true'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: From af33baa5631e919133e621e0e2b3240bd62f4968 Mon Sep 17 00:00:00 2001 From: Jan Jones Date: Tue, 11 Aug 2026 12:22:09 +0200 Subject: [PATCH 06/14] Create output dir --- eng/common/core-templates/steps/source-index-stage1-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/core-templates/steps/source-index-stage1-publish.yml b/eng/common/core-templates/steps/source-index-stage1-publish.yml index 3e65ff6ee5d..94d56811d71 100644 --- a/eng/common/core-templates/steps/source-index-stage1-publish.yml +++ b/eng/common/core-templates/steps/source-index-stage1-publish.yml @@ -21,7 +21,7 @@ steps: # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. workingDirectory: $(Agent.TempDirectory) -- script: $(Agent.TempDirectory)/.source-index/tools/complog create ${{parameters.BinlogPath}} -o .source-index/stage1output/build.complog +- script: mkdir ".source-index/stage1output" && $(Agent.TempDirectory)/.source-index/tools/complog create ${{parameters.BinlogPath}} -o .source-index/stage1output/build.complog displayName: "Source Index: Process Binlog into Complog" - ${{ if and(ne(parameters.runAsPublic, 'true'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: From 4118695ac9e7e8b7fa787f83d4d77a1ae8f94e34 Mon Sep 17 00:00:00 2001 From: Jan Jones Date: Tue, 11 Aug 2026 13:52:17 +0200 Subject: [PATCH 07/14] Write hash --- .../core-templates/steps/source-index-stage1-publish.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/eng/common/core-templates/steps/source-index-stage1-publish.yml b/eng/common/core-templates/steps/source-index-stage1-publish.yml index 94d56811d71..b3241e045de 100644 --- a/eng/common/core-templates/steps/source-index-stage1-publish.yml +++ b/eng/common/core-templates/steps/source-index-stage1-publish.yml @@ -21,7 +21,10 @@ steps: # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. workingDirectory: $(Agent.TempDirectory) -- script: mkdir ".source-index/stage1output" && $(Agent.TempDirectory)/.source-index/tools/complog create ${{parameters.BinlogPath}} -o .source-index/stage1output/build.complog +- script: | + mkdir ".source-index/stage1output" + git rev-parse HEAD > .source-index/stage1output/hash + $(Agent.TempDirectory)/.source-index/tools/complog create ${{parameters.BinlogPath}} -o .source-index/stage1output/build.complog displayName: "Source Index: Process Binlog into Complog" - ${{ if and(ne(parameters.runAsPublic, 'true'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: From c137d5e094803115d832fc5e451ed7dfa1ba2dce Mon Sep 17 00:00:00 2001 From: Jan Jones Date: Tue, 11 Aug 2026 14:03:09 +0200 Subject: [PATCH 08/14] Fix source index build command --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 75c2e537d4f..8533172eadb 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -136,7 +136,7 @@ variables: ) ) - name: sourceIndexBuildCommand - value: $(Build.SourcesDirectory)/build.cmd -restore -ci -NativeToolsOnMachine + value: $(Build.SourcesDirectory)/build.cmd -restore -build -ci -NativeToolsOnMachine resources: repositories: From d5d336d9835f02743b29167818b47d233ff4192d Mon Sep 17 00:00:00 2001 From: Jan Jones Date: Tue, 11 Aug 2026 14:36:31 +0200 Subject: [PATCH 09/14] Add NodeJS --- azure-pipelines.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 8533172eadb..16ef0c8faab 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -183,6 +183,12 @@ extends: sourceIndexParams: condition: ${{ variables['sourceIndexCondition'] }} sourceIndexBuildCommand: ${{ variables['sourceIndexBuildCommand'] }} + preSteps: + - task: NodeTool@0 + displayName: Add NodeJS/npm + inputs: + versionSpec: "20.x" + checkLatest: true runAsPublic: ${{ variables['runAsPublic'] }} # Publish test logs enablePublishTestResults: true From 925cb706022ae77de976b05038d8bdc15411aa9e Mon Sep 17 00:00:00 2001 From: Jan Jones Date: Tue, 11 Aug 2026 15:17:05 +0200 Subject: [PATCH 10/14] Upload complog after normal build --- azure-pipelines.yml | 28 +++++++------------ .../job/source-index-stage1.yml | 1 + .../steps/source-index-stage1-publish.yml | 11 ++++++++ eng/pipelines/templates/BuildAndTest.yml | 11 ++++++++ 4 files changed, 33 insertions(+), 18 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 16ef0c8faab..187bdab6baf 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -127,16 +127,17 @@ variables: # Enable source index only for main branch builds - name: sourceIndexCondition value: | - or( - eq(variables['forceEnableSourceIndex'], 'true'), - and( - ne(variables['runAsPublic'], 'true'), - notin(variables['Build.Reason'], 'PullRequest'), - eq(variables['Build.SourceBranch'], 'refs/heads/main') + and( + succeeded(), + or( + eq(variables['forceEnableSourceIndex'], 'true'), + and( + ne(variables['runAsPublic'], 'true'), + notin(variables['Build.Reason'], 'PullRequest'), + eq(variables['Build.SourceBranch'], 'refs/heads/main') + ) ) ) - - name: sourceIndexBuildCommand - value: $(Build.SourcesDirectory)/build.cmd -restore -build -ci -NativeToolsOnMachine resources: repositories: @@ -179,16 +180,6 @@ extends: manifests: true enableMicrobuild: true enableTelemetry: true - enableSourceIndex: true - sourceIndexParams: - condition: ${{ variables['sourceIndexCondition'] }} - sourceIndexBuildCommand: ${{ variables['sourceIndexBuildCommand'] }} - preSteps: - - task: NodeTool@0 - displayName: Add NodeJS/npm - inputs: - versionSpec: "20.x" - checkLatest: true runAsPublic: ${{ variables['runAsPublic'] }} # Publish test logs enablePublishTestResults: true @@ -245,6 +236,7 @@ extends: skipTests: ${{ not(parameters.runTests) }} skipQualityGates: ${{ eq(variables['SkipQualityGates'], 'true') }} isWindows: true + sourceIndexCondition: ${{ variables['sourceIndexCondition'] }} # ---------------------------------------------------------------- # This job build and run tests on Ubuntu diff --git a/eng/common/core-templates/job/source-index-stage1.yml b/eng/common/core-templates/job/source-index-stage1.yml index 46d5d4703ea..1dd4b6f8bb5 100644 --- a/eng/common/core-templates/job/source-index-stage1.yml +++ b/eng/common/core-templates/job/source-index-stage1.yml @@ -43,3 +43,4 @@ jobs: - template: /eng/common/core-templates/steps/source-index-stage1-publish.yml parameters: binLogPath: ${{ parameters.binLogPath }} + runAsPublic: ${{ parameters.runAsPublic }} diff --git a/eng/common/core-templates/steps/source-index-stage1-publish.yml b/eng/common/core-templates/steps/source-index-stage1-publish.yml index b3241e045de..185757543c8 100644 --- a/eng/common/core-templates/steps/source-index-stage1-publish.yml +++ b/eng/common/core-templates/steps/source-index-stage1-publish.yml @@ -1,13 +1,18 @@ parameters: + runAsPublic: false sourceIndexUploadPackageVersion: '2.0.0-20250818.1' sourceIndexComplogPackageVersion: '0.9.50' sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json sourceIndexPublicPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/index.json binlogPath: artifacts/log/Debug/Build.binlog + condition: succeeded() + continueOnError: false steps: - task: UseDotNet@2 displayName: "Source Index: Use .NET 9 SDK" + condition: ${{ parameters.condition }} + continueOnError: ${{ parameters.continueOnError }} inputs: packageType: sdk version: 9.0.x @@ -18,6 +23,8 @@ steps: $(Agent.TempDirectory)/dotnet/dotnet tool install complog --version ${{parameters.sourceIndexComplogPackageVersion}} --source ${{parameters.sourceIndexPublicPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version ${{parameters.sourceIndexUploadPackageVersion}} --source ${{parameters.sourceIndexPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools displayName: "Source Index: Download Tools" + condition: ${{ parameters.condition }} + continueOnError: ${{ parameters.continueOnError }} # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. workingDirectory: $(Agent.TempDirectory) @@ -26,10 +33,14 @@ steps: git rev-parse HEAD > .source-index/stage1output/hash $(Agent.TempDirectory)/.source-index/tools/complog create ${{parameters.BinlogPath}} -o .source-index/stage1output/build.complog displayName: "Source Index: Process Binlog into Complog" + condition: ${{ parameters.condition }} + continueOnError: ${{ parameters.continueOnError }} - ${{ if and(ne(parameters.runAsPublic, 'true'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - task: AzureCLI@2 displayName: "Source Index: Upload Source Index stage1 artifacts to Azure" + condition: ${{ parameters.condition }} + continueOnError: ${{ parameters.continueOnError }} inputs: azureSubscription: 'SourceDotNet Stage1 Publish' addSpnToEnvironment: true diff --git a/eng/pipelines/templates/BuildAndTest.yml b/eng/pipelines/templates/BuildAndTest.yml index ce540adfa24..24a2a6f0f9e 100644 --- a/eng/pipelines/templates/BuildAndTest.yml +++ b/eng/pipelines/templates/BuildAndTest.yml @@ -21,6 +21,9 @@ parameters: - name: runAsPublic type: boolean default: false + - name: sourceIndexCondition + type: string + default: 'false' steps: - task: NodeTool@0 @@ -53,6 +56,14 @@ steps: $(_OfficialBuildIdArgs) displayName: Build + - ${{ if eq(parameters.isWindows, 'true') }}: + - template: /eng/common/core-templates/steps/source-index-stage1-publish.yml + parameters: + binlogPath: ${{ parameters.repoLogPath }}/build.binlog + condition: ${{ parameters.sourceIndexCondition }} + continueOnError: true + runAsPublic: ${{ parameters.runAsPublic }} + - ${{ if ne(parameters.skipTests, 'true') }}: - script: $(Build.SourcesDirectory)/.dotnet/dotnet dotnet-coverage collect --settings $(Build.SourcesDirectory)/eng/CodeCoverage.config From 7196b931f8d76f4f22c6772e9c49a770c477c798 Mon Sep 17 00:00:00 2001 From: Jan Jones Date: Tue, 11 Aug 2026 16:07:44 +0200 Subject: [PATCH 11/14] Share build command --- azure-pipelines.yml | 16 ++++++- .../job/source-index-stage1.yml | 6 +-- .../steps/source-index-stage1-publish.yml | 11 ----- eng/pipelines/templates/BuildAndTest.yml | 46 +++---------------- eng/pipelines/templates/RestoreAndBuild.yml | 41 +++++++++++++++++ 5 files changed, 64 insertions(+), 56 deletions(-) create mode 100644 eng/pipelines/templates/RestoreAndBuild.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 187bdab6baf..bfee4ea8964 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -78,6 +78,8 @@ variables: value: $(Build.Arcade.ArtifactsPath)TestResults/$(_BuildConfig)/ - name: Build.Arcade.VSIXOutputPath value: $(Build.Arcade.ArtifactsPath)VSIX + - name: _WindowsBuildScript + value: $(Build.SourcesDirectory)/build.cmd -ci -NativeToolsOnMachine # Enable extraction of published outputs for analysis - name: GDN_EXTRACT_TOOLS @@ -180,6 +182,17 @@ extends: manifests: true enableMicrobuild: true enableTelemetry: true + enableSourceIndex: true + sourceIndexParams: + condition: ${{ variables['sourceIndexCondition'] }} + sourceIndexBuildCommand: '' + binlogPath: artifacts/log/Debug/build.binlog + preSteps: + - template: /eng/pipelines/templates/RestoreAndBuild.yml + parameters: + buildScript: $(_WindowsBuildScript) + buildConfig: Debug + repoLogPath: $(Build.SourcesDirectory)/artifacts/log/Debug runAsPublic: ${{ variables['runAsPublic'] }} # Publish test logs enablePublishTestResults: true @@ -203,7 +216,7 @@ extends: os: windows variables: - - _buildScript: $(Build.SourcesDirectory)/build.cmd -ci -NativeToolsOnMachine + - _buildScript: $(_WindowsBuildScript) templateContext: outputs: @@ -236,7 +249,6 @@ extends: skipTests: ${{ not(parameters.runTests) }} skipQualityGates: ${{ eq(variables['SkipQualityGates'], 'true') }} isWindows: true - sourceIndexCondition: ${{ variables['sourceIndexCondition'] }} # ---------------------------------------------------------------- # This job build and run tests on Ubuntu diff --git a/eng/common/core-templates/job/source-index-stage1.yml b/eng/common/core-templates/job/source-index-stage1.yml index 1dd4b6f8bb5..6a941c4e53d 100644 --- a/eng/common/core-templates/job/source-index-stage1.yml +++ b/eng/common/core-templates/job/source-index-stage1.yml @@ -37,10 +37,10 @@ jobs: - ${{ each preStep in parameters.preSteps }}: - ${{ preStep }} - - script: ${{ parameters.sourceIndexBuildCommand }} - displayName: Build Repository + - ${{ if ne(parameters.sourceIndexBuildCommand, '') }}: + - script: ${{ parameters.sourceIndexBuildCommand }} + displayName: Build Repository - template: /eng/common/core-templates/steps/source-index-stage1-publish.yml parameters: binLogPath: ${{ parameters.binLogPath }} - runAsPublic: ${{ parameters.runAsPublic }} diff --git a/eng/common/core-templates/steps/source-index-stage1-publish.yml b/eng/common/core-templates/steps/source-index-stage1-publish.yml index 185757543c8..b3241e045de 100644 --- a/eng/common/core-templates/steps/source-index-stage1-publish.yml +++ b/eng/common/core-templates/steps/source-index-stage1-publish.yml @@ -1,18 +1,13 @@ parameters: - runAsPublic: false sourceIndexUploadPackageVersion: '2.0.0-20250818.1' sourceIndexComplogPackageVersion: '0.9.50' sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json sourceIndexPublicPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/index.json binlogPath: artifacts/log/Debug/Build.binlog - condition: succeeded() - continueOnError: false steps: - task: UseDotNet@2 displayName: "Source Index: Use .NET 9 SDK" - condition: ${{ parameters.condition }} - continueOnError: ${{ parameters.continueOnError }} inputs: packageType: sdk version: 9.0.x @@ -23,8 +18,6 @@ steps: $(Agent.TempDirectory)/dotnet/dotnet tool install complog --version ${{parameters.sourceIndexComplogPackageVersion}} --source ${{parameters.sourceIndexPublicPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version ${{parameters.sourceIndexUploadPackageVersion}} --source ${{parameters.sourceIndexPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools displayName: "Source Index: Download Tools" - condition: ${{ parameters.condition }} - continueOnError: ${{ parameters.continueOnError }} # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. workingDirectory: $(Agent.TempDirectory) @@ -33,14 +26,10 @@ steps: git rev-parse HEAD > .source-index/stage1output/hash $(Agent.TempDirectory)/.source-index/tools/complog create ${{parameters.BinlogPath}} -o .source-index/stage1output/build.complog displayName: "Source Index: Process Binlog into Complog" - condition: ${{ parameters.condition }} - continueOnError: ${{ parameters.continueOnError }} - ${{ if and(ne(parameters.runAsPublic, 'true'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - task: AzureCLI@2 displayName: "Source Index: Upload Source Index stage1 artifacts to Azure" - condition: ${{ parameters.condition }} - continueOnError: ${{ parameters.continueOnError }} inputs: azureSubscription: 'SourceDotNet Stage1 Publish' addSpnToEnvironment: true diff --git a/eng/pipelines/templates/BuildAndTest.yml b/eng/pipelines/templates/BuildAndTest.yml index 24a2a6f0f9e..eef6c522510 100644 --- a/eng/pipelines/templates/BuildAndTest.yml +++ b/eng/pipelines/templates/BuildAndTest.yml @@ -21,48 +21,14 @@ parameters: - name: runAsPublic type: boolean default: false - - name: sourceIndexCondition - type: string - default: 'false' steps: - - task: NodeTool@0 - displayName: Add NodeJS/npm - inputs: - versionSpec: "20.x" - checkLatest: true - - - script: ${{ parameters.buildScript }} - -restore - -warnAsError ${{ parameters.warnAsError }} - /bl:${{ parameters.repoLogPath }}/restore.binlog - displayName: Restore - - - pwsh: | - $(Build.SourcesDirectory)/scripts/Slngen.ps1 -All -NoLaunch - displayName: Create solution - - - script: ${{ parameters.buildScript }} - -restore - -warnAsError ${{ parameters.warnAsError }} - /bl:${{ parameters.repoLogPath }}/restore2.binlog - displayName: Restore solution - - - script: ${{ parameters.buildScript }} - -build - -configuration ${{ parameters.buildConfig }} - -warnAsError ${{ parameters.warnAsError }} - /bl:${{ parameters.repoLogPath }}/build.binlog - $(_OfficialBuildIdArgs) - displayName: Build - - - ${{ if eq(parameters.isWindows, 'true') }}: - - template: /eng/common/core-templates/steps/source-index-stage1-publish.yml - parameters: - binlogPath: ${{ parameters.repoLogPath }}/build.binlog - condition: ${{ parameters.sourceIndexCondition }} - continueOnError: true - runAsPublic: ${{ parameters.runAsPublic }} + - template: /eng/pipelines/templates/RestoreAndBuild.yml + parameters: + buildScript: ${{ parameters.buildScript }} + buildConfig: ${{ parameters.buildConfig }} + repoLogPath: ${{ parameters.repoLogPath }} + warnAsError: ${{ parameters.warnAsError }} - ${{ if ne(parameters.skipTests, 'true') }}: - script: $(Build.SourcesDirectory)/.dotnet/dotnet dotnet-coverage collect diff --git a/eng/pipelines/templates/RestoreAndBuild.yml b/eng/pipelines/templates/RestoreAndBuild.yml new file mode 100644 index 00000000000..9b6bdf5cfa9 --- /dev/null +++ b/eng/pipelines/templates/RestoreAndBuild.yml @@ -0,0 +1,41 @@ +parameters: + - name: buildScript + type: string + - name: buildConfig + type: string + - name: repoLogPath + type: string + - name: warnAsError + type: number + default: 1 + +steps: + - task: NodeTool@0 + displayName: Add NodeJS/npm + inputs: + versionSpec: "20.x" + checkLatest: true + + - script: ${{ parameters.buildScript }} + -restore + -warnAsError ${{ parameters.warnAsError }} + /bl:${{ parameters.repoLogPath }}/restore.binlog + displayName: Restore + + - pwsh: | + $(Build.SourcesDirectory)/scripts/Slngen.ps1 -All -NoLaunch + displayName: Create solution + + - script: ${{ parameters.buildScript }} + -restore + -warnAsError ${{ parameters.warnAsError }} + /bl:${{ parameters.repoLogPath }}/restore2.binlog + displayName: Restore solution + + - script: ${{ parameters.buildScript }} + -build + -configuration ${{ parameters.buildConfig }} + -warnAsError ${{ parameters.warnAsError }} + /bl:${{ parameters.repoLogPath }}/build.binlog + $(_OfficialBuildIdArgs) + displayName: Build From c6fcf19d7d2bea00ca6469a77c9dc71230d3db5c Mon Sep 17 00:00:00 2001 From: Jan Jones Date: Tue, 11 Aug 2026 16:09:11 +0200 Subject: [PATCH 12/14] Propagate runAsPublic parameter --- eng/common/core-templates/job/source-index-stage1.yml | 1 + eng/common/core-templates/steps/source-index-stage1-publish.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/eng/common/core-templates/job/source-index-stage1.yml b/eng/common/core-templates/job/source-index-stage1.yml index 6a941c4e53d..37e1fba8360 100644 --- a/eng/common/core-templates/job/source-index-stage1.yml +++ b/eng/common/core-templates/job/source-index-stage1.yml @@ -44,3 +44,4 @@ jobs: - template: /eng/common/core-templates/steps/source-index-stage1-publish.yml parameters: binLogPath: ${{ parameters.binLogPath }} + runAsPublic: ${{ parameters.runAsPublic }} diff --git a/eng/common/core-templates/steps/source-index-stage1-publish.yml b/eng/common/core-templates/steps/source-index-stage1-publish.yml index b3241e045de..d5517d8cd5f 100644 --- a/eng/common/core-templates/steps/source-index-stage1-publish.yml +++ b/eng/common/core-templates/steps/source-index-stage1-publish.yml @@ -1,4 +1,5 @@ parameters: + runAsPublic: false sourceIndexUploadPackageVersion: '2.0.0-20250818.1' sourceIndexComplogPackageVersion: '0.9.50' sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json From e61b34520e1d0ac0297e2a0e8b4cc73482997a65 Mon Sep 17 00:00:00 2001 From: Jan Jones Date: Tue, 11 Aug 2026 16:39:32 +0200 Subject: [PATCH 13/14] Revert eng/common changes --- .../job/source-index-stage1.yml | 7 ++----- .../steps/source-index-stage1-publish.yml | 19 +++++++------------ 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/eng/common/core-templates/job/source-index-stage1.yml b/eng/common/core-templates/job/source-index-stage1.yml index 37e1fba8360..76baf5c2725 100644 --- a/eng/common/core-templates/job/source-index-stage1.yml +++ b/eng/common/core-templates/job/source-index-stage1.yml @@ -12,7 +12,6 @@ jobs: - job: SourceIndexStage1 dependsOn: ${{ parameters.dependsOn }} condition: ${{ parameters.condition }} - continueOnError: true variables: - name: BinlogPath value: ${{ parameters.binlogPath }} @@ -37,11 +36,9 @@ jobs: - ${{ each preStep in parameters.preSteps }}: - ${{ preStep }} - - ${{ if ne(parameters.sourceIndexBuildCommand, '') }}: - - script: ${{ parameters.sourceIndexBuildCommand }} - displayName: Build Repository + - script: ${{ parameters.sourceIndexBuildCommand }} + displayName: Build Repository - template: /eng/common/core-templates/steps/source-index-stage1-publish.yml parameters: binLogPath: ${{ parameters.binLogPath }} - runAsPublic: ${{ parameters.runAsPublic }} diff --git a/eng/common/core-templates/steps/source-index-stage1-publish.yml b/eng/common/core-templates/steps/source-index-stage1-publish.yml index d5517d8cd5f..6e7666b4dcf 100644 --- a/eng/common/core-templates/steps/source-index-stage1-publish.yml +++ b/eng/common/core-templates/steps/source-index-stage1-publish.yml @@ -1,9 +1,7 @@ parameters: - runAsPublic: false - sourceIndexUploadPackageVersion: '2.0.0-20250818.1' - sourceIndexComplogPackageVersion: '0.9.50' + sourceIndexUploadPackageVersion: 2.0.0-20250818.1 + sourceIndexProcessBinlogPackageVersion: 1.0.1-20250818.1 sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json - sourceIndexPublicPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/index.json binlogPath: artifacts/log/Debug/Build.binlog steps: @@ -16,17 +14,14 @@ steps: workingDirectory: $(Agent.TempDirectory) - script: | - $(Agent.TempDirectory)/dotnet/dotnet tool install complog --version ${{parameters.sourceIndexComplogPackageVersion}} --source ${{parameters.sourceIndexPublicPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools - $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version ${{parameters.sourceIndexUploadPackageVersion}} --source ${{parameters.sourceIndexPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools - displayName: "Source Index: Download Tools" + $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version ${{parameters.sourceIndexProcessBinlogPackageVersion}} --source ${{parameters.SourceIndexPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version ${{parameters.sourceIndexUploadPackageVersion}} --source ${{parameters.SourceIndexPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools + displayName: "Source Index: Download netsourceindex Tools" # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. workingDirectory: $(Agent.TempDirectory) -- script: | - mkdir ".source-index/stage1output" - git rev-parse HEAD > .source-index/stage1output/hash - $(Agent.TempDirectory)/.source-index/tools/complog create ${{parameters.BinlogPath}} -o .source-index/stage1output/build.complog - displayName: "Source Index: Process Binlog into Complog" +- script: $(Agent.TempDirectory)/.source-index/tools/BinLogToSln -i ${{parameters.BinlogPath}} -r $(System.DefaultWorkingDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output + displayName: "Source Index: Process Binlog into indexable sln" - ${{ if and(ne(parameters.runAsPublic, 'true'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - task: AzureCLI@2 From 39efa1e7d1554bf32a8f202b72b2fefc42a1be41 Mon Sep 17 00:00:00 2001 From: Jan Jones Date: Tue, 11 Aug 2026 16:44:11 +0200 Subject: [PATCH 14/14] Remove redundant parameter override --- azure-pipelines.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index bfee4ea8964..40c339d7ffa 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -186,7 +186,6 @@ extends: sourceIndexParams: condition: ${{ variables['sourceIndexCondition'] }} sourceIndexBuildCommand: '' - binlogPath: artifacts/log/Debug/build.binlog preSteps: - template: /eng/pipelines/templates/RestoreAndBuild.yml parameters: