diff --git a/.github/workflows/sync-sim-tests.yml b/.github/workflows/sync-sim-tests.yml new file mode 100644 index 000000000..ad8091e36 --- /dev/null +++ b/.github/workflows/sync-sim-tests.yml @@ -0,0 +1,219 @@ +name: Basis Sync Sim Tests + +on: + push: + branches: [main, developer] + paths: + - 'Basis/Packages/com.basis.framework/**' + - '.github/workflows/sync-sim-tests.yml' + pull_request: + branches: [main, developer] + paths: + - 'Basis/Packages/com.basis.framework/**' + - '.github/workflows/sync-sim-tests.yml' + workflow_dispatch: + inputs: + unityVersion: + description: 'Unity version to test (default: 6000.5.1f1)' + required: false + default: '6000.5.1f1' + testFilter: + description: 'NUnit filter (e.g. BasisSyncSimTests.Sim_Converges*)' + required: false + default: 'BasisSyncSimTests' + +jobs: + check-secret: + name: Check if secrets available + timeout-minutes: 5 + runs-on: ubuntu-latest + outputs: + secret-is-set: ${{ steps.secret-is-set.outputs.defined }} + steps: + - name: Check if secret is set, then set variable + id: secret-is-set + env: + TMP_SECRET1: ${{ secrets.UNITY_LICENSE }} + TMP_SECRET2: ${{ secrets.UNITY_EMAIL }} + TMP_SECRET3: ${{ secrets.UNITY_PASSWORD }} + if: "${{ env.TMP_SECRET1 != '' && env.TMP_SECRET2 != '' && env.TMP_SECRET3 != '' }}" + run: echo "defined=true" >> $GITHUB_OUTPUT + + test: + name: Unity ${{ github.event.inputs.unityVersion || '6000.5.1f1' }} EditMode Tests + timeout-minutes: 60 + runs-on: ubuntu-latest + permissions: + actions: write # to allow us to manage cache + checks: write # to publish GameCI test checks + contents: read + env: + projectPath: Basis + unityVersion: ${{ github.event.inputs.unityVersion || '6000.5.1f1' }} + testFilter: ${{ github.event.inputs.testFilter || 'BasisSyncSimTests' }} + testArtifactsPath: artifacts/sync-sim-tests + needs: [check-secret] + if: needs.check-secret.outputs.secret-is-set == 'true' + steps: + # We're running out of disk space in some cases. However, the actual test run is happening in a docker container. + # Therefore, we don't actually need most of the tools that github provides. This can save quite a bit of space. + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + tool-cache: true + android: true + dotnet: true + haskell: true + large-packages: true + docker-images: false + swap-storage: false + - name: "Checkout repository" + timeout-minutes: 10 + uses: actions/checkout@v4 + with: + fetch-depth: 0 + lfs: true + submodules: recursive + - name: "Restore Library cache" + id: restore-cache + timeout-minutes: 10 + uses: actions/cache/restore@v3 + with: + path: ${{ env.projectPath }}/Library + key: Library-${{ env.projectPath }}-sync-sim-tests-${{ hashFiles(env.projectPath) }} + restore-keys: Library-${{ env.projectPath }}-sync-sim-tests- + - name: "Run BasisSyncSimTests (EditMode)" + timeout-minutes: 60 + uses: game-ci/unity-test-runner@v4 + env: + UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }} + UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} + UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} + with: + projectPath: ${{ env.projectPath }} + unityVersion: ${{ env.unityVersion }} + testMode: editmode + artifactsPath: ${{ env.testArtifactsPath }} + githubToken: ${{ secrets.GITHUB_TOKEN }} + checkName: Basis Sync Sim EditMode Tests + customParameters: -nographics -testFilter ${{ env.testFilter }} + - name: "Save Library Cache" + uses: actions/cache/save@v3 + if: always() && github.ref_name == 'developer' + with: + path: ${{ env.projectPath }}/Library + key: ${{ steps.restore-cache.outputs.cache-primary-key }} + - name: "Only retain latest cache" + if: always() && github.ref_name == 'developer' + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + OLD_CACHE_IDS=$(gh cache list --sort created_at --key Library-${{ env.projectPath }}-sync-sim-tests- --json id --jq '.[1:] | map(.id) | @sh') + for cache_id in $OLD_CACHE_IDS; do + echo "Deleting cache id: $cache_id" + gh cache delete $cache_id + done + - name: "Upload test artifacts" + if: always() + uses: actions/upload-artifact@v4 + with: + name: sync-sim-test-artifacts-${{ env.unityVersion }} + path: ${{ env.testArtifactsPath }} + retention-days: 30 + + matrix-runner: + name: Full Matrix Runner (CSV Export) + timeout-minutes: 120 + runs-on: ubuntu-latest + permissions: + actions: write # to allow us to manage cache + contents: write # to upload release assets on tag dispatches + env: + projectPath: Basis + unityVersion: ${{ github.event.inputs.unityVersion || '6000.5.1f1' }} + matrixCsvPath: artifacts/sync-sim-matrix.csv + matrixOptionsPath: artifacts/sync-sim-matrix-options.json + matrixOptions: '{"Seeds":3,"QuantizeVariants":true,"InterpolateOffVariants":true,"ExtrapolateVariant":true,"TeleportThresholdVariant":true,"CompositeConfigs":true,"MultiObjectScenes":true,"BatchedTransport":true,"RealInterpJob":true,"LateJoinVariant":true,"Dt":0.013888889,"DurationSeconds":6,"SettleSeconds":1.8,"BaseSeed":1523023}' + needs: [check-secret] + if: github.event_name == 'workflow_dispatch' && needs.check-secret.outputs.secret-is-set == 'true' + steps: + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + tool-cache: true + android: true + dotnet: true + haskell: true + large-packages: true + docker-images: false + swap-storage: false + - name: "Checkout repository" + timeout-minutes: 10 + uses: actions/checkout@v4 + with: + fetch-depth: 0 + lfs: true + submodules: recursive + - name: "Restore Library cache" + id: restore-cache + timeout-minutes: 10 + uses: actions/cache/restore@v3 + with: + path: ${{ env.projectPath }}/Library + key: Library-${{ env.projectPath }}-sync-sim-matrix-${{ hashFiles(env.projectPath) }} + restore-keys: Library-${{ env.projectPath }}-sync-sim-matrix- + - name: "Write matrix options" + shell: bash + run: | + mkdir -p "$(dirname "${{ env.matrixOptionsPath }}")" + cat > "${{ env.matrixOptionsPath }}" <<'JSON' + ${{ env.matrixOptions }} + JSON + - name: "Run Full Matrix (Headless Editor Script)" + timeout-minutes: 120 + uses: BasisVR/unity-builder@main + env: + UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }} + UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} + UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} + with: + buildName: sync-sim-matrix + buildsPath: artifacts/matrix-build + buildMethod: Basis.Scripts.Networking.Sync.Testing.BasisSyncSimMatrix.RunAndExportCSV + customParameters: -matrixOptionsPath /github/workspace/${{ env.matrixOptionsPath }} -outputPath /github/workspace/${{ env.matrixCsvPath }} + manualExit: true + projectPath: ${{ env.projectPath }} + targetPlatform: StandaloneLinux64 + unityVersion: ${{ env.unityVersion }} + versioning: None + linux64RemoveExecutableExtension: false + - name: "Save Library Cache" + uses: actions/cache/save@v3 + if: always() && github.ref_name == 'developer' + with: + path: ${{ env.projectPath }}/Library + key: ${{ steps.restore-cache.outputs.cache-primary-key }} + - name: "Only retain latest cache" + if: always() && github.ref_name == 'developer' + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + OLD_CACHE_IDS=$(gh cache list --sort created_at --key Library-${{ env.projectPath }}-sync-sim-matrix- --json id --jq '.[1:] | map(.id) | @sh') + for cache_id in $OLD_CACHE_IDS; do + echo "Deleting cache id: $cache_id" + gh cache delete $cache_id + done + - name: "Upload Matrix CSV" + if: always() + uses: actions/upload-artifact@v4 + with: + name: sync-sim-matrix-csv + path: ${{ env.matrixCsvPath }} + retention-days: 30 + - name: "Upload Matrix CSV as Release Asset (on tag)" + if: github.event_name == 'workflow_dispatch' && startsWith(github.ref, 'refs/tags/') + uses: softprops/action-gh-release@v2 + with: + files: ${{ env.matrixCsvPath }} diff --git a/Basis/Packages/com.basis.framework/Editor/SyncTesting/BasisSyncSimMatrix.cs b/Basis/Packages/com.basis.framework/Editor/SyncTesting/BasisSyncSimMatrix.cs index b42ade3d1..afc3e0adc 100644 --- a/Basis/Packages/com.basis.framework/Editor/SyncTesting/BasisSyncSimMatrix.cs +++ b/Basis/Packages/com.basis.framework/Editor/SyncTesting/BasisSyncSimMatrix.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Text; +using UnityEngine; namespace Basis.Scripts.Networking.Sync.Testing { @@ -22,6 +23,7 @@ public sealed class FieldConfig public bool PositionFirst; } + [Serializable] public sealed class MatrixOptions { public bool QuantizeVariants = true; @@ -423,6 +425,96 @@ public static string ToCsv(List results) return sb.ToString(); } + /// + /// Command-line entry point for CI: runs the full matrix and writes CSV to the given path. + /// Usage: Unity -batchmode -executeMethod Basis.Scripts.Networking.Sync.Testing.BasisSyncSimMatrix.RunAndExportCSV -matrixOptions '{"Seeds":3}' -outputPath /path/to/output.csv + /// Unity -batchmode -executeMethod Basis.Scripts.Networking.Sync.Testing.BasisSyncSimMatrix.RunAndExportCSV -matrixOptionsPath /path/to/options.json -outputPath /path/to/output.csv + /// + public static void RunAndExportCSV() + { + string matrixOptionsJson = null; + string matrixOptionsPath = null; + string outputPath = null; + + var args = System.Environment.GetCommandLineArgs(); + for (int i = 0; i < args.Length; i++) + { + if (args[i] == "-matrixOptions" && i + 1 < args.Length) + matrixOptionsJson = args[i + 1]; + else if (args[i] == "-matrixOptionsPath" && i + 1 < args.Length) + matrixOptionsPath = args[i + 1]; + else if (args[i] == "-outputPath" && i + 1 < args.Length) + outputPath = args[i + 1]; + } + + if (!string.IsNullOrEmpty(matrixOptionsPath)) + { + try + { + matrixOptionsJson = System.IO.File.ReadAllText(matrixOptionsPath); + } + catch (System.Exception e) + { + UnityEngine.Debug.LogError($"Failed to read -matrixOptionsPath '{matrixOptionsPath}': {e.Message}"); + UnityEditor.EditorApplication.Exit(1); + return; + } + } + + MatrixOptions options = MatrixOptions.Full(); + if (!string.IsNullOrEmpty(matrixOptionsJson)) + { + try + { + JsonUtility.FromJsonOverwrite(matrixOptionsJson, options); + } + catch (System.Exception e) + { + UnityEngine.Debug.LogError($"Failed to parse -matrixOptions JSON: {e.Message}"); + UnityEditor.EditorApplication.Exit(1); + return; + } + } + + if (string.IsNullOrEmpty(outputPath)) + { + UnityEngine.Debug.LogError("-outputPath is required"); + UnityEditor.EditorApplication.Exit(1); + return; + } + + UnityEngine.Debug.Log($"Running sync sim matrix: {BasisSyncSimMatrix.CountScenarios(options)} scenarios"); + + var results = RunAll(options, (done, total, r) => + { + if (r != null) + UnityEngine.Debug.Log($"[{done}/{total}] {r.ScenarioName} => {(r.Warn ? "WARN" : (r.Pass ? "PASS" : "FAIL"))} {r.FailReason}"); + }, null); + + string csv = ToCsv(results); + string outputDirectory = System.IO.Path.GetDirectoryName(outputPath); + if (!string.IsNullOrEmpty(outputDirectory)) + System.IO.Directory.CreateDirectory(outputDirectory); + System.IO.File.WriteAllText(outputPath, csv); + + int failed = 0; + for (int i = 0; i < results.Count; i++) + { + if (!results[i].Pass) + failed++; + } + + UnityEngine.Debug.Log($"CSV written to {outputPath} ({results.Count} rows, {failed} failures)"); + if (failed > 0) + { + UnityEngine.Debug.LogError($"Sync sim matrix completed with {failed} failing rows. See CSV for details."); + UnityEditor.EditorApplication.Exit(1); + return; + } + + UnityEditor.EditorApplication.Exit(0); + } + static int Hash(string s) { if (string.IsNullOrEmpty(s)) return 0;