Ci/sync sim tests

.github/workflows/sync-sim-tests.yml

@@ -0,0 +1,219 @@
+name: Basis Sync Sim Tests
+
+on:
+  push:
+    branches: [main, developer]
+    paths:
+      - 'Basis/Packages/com.basis.framework/**'
+      - '.github/workflows/sync-sim-tests.yml'
+  pull_request:
+    branches: [main, developer]
+    paths:
+      - 'Basis/Packages/com.basis.framework/**'
+      - '.github/workflows/sync-sim-tests.yml'
+  workflow_dispatch:
+    inputs:
+      unityVersion:
+        description: 'Unity version to test (default: 6000.5.1f1)'
+        required: false
+        default: '6000.5.1f1'
+      testFilter:
+        description: 'NUnit filter (e.g. BasisSyncSimTests.Sim_Converges*)'
+        required: false
+        default: 'BasisSyncSimTests'
+
+jobs:
+  check-secret:
+    name: Check if secrets available
+    timeout-minutes: 5
+    runs-on: ubuntu-latest
+    outputs:
+      secret-is-set: ${{ steps.secret-is-set.outputs.defined }}
+    steps:
+      - name: Check if secret is set, then set variable
+        id: secret-is-set
+        env:
+          TMP_SECRET1: ${{ secrets.UNITY_LICENSE }}
+          TMP_SECRET2: ${{ secrets.UNITY_EMAIL }}
+          TMP_SECRET3: ${{ secrets.UNITY_PASSWORD }}
+        if: "${{ env.TMP_SECRET1 != '' && env.TMP_SECRET2 != '' && env.TMP_SECRET3 != '' }}"
+        run: echo "defined=true" >> $GITHUB_OUTPUT
+
+  test:
+    name: Unity ${{ github.event.inputs.unityVersion || '6000.5.1f1' }} EditMode Tests
+    timeout-minutes: 60
+    runs-on: ubuntu-latest
+    permissions:
+      actions: write # to allow us to manage cache
+      checks: write # to publish GameCI test checks
+      contents: read
+    env:
+      projectPath: Basis
+      unityVersion: ${{ github.event.inputs.unityVersion || '6000.5.1f1' }}
+      testFilter: ${{ github.event.inputs.testFilter || 'BasisSyncSimTests' }}
+      testArtifactsPath: artifacts/sync-sim-tests
+    needs: [check-secret]
+    if: needs.check-secret.outputs.secret-is-set == 'true'
+    steps:
+      # We're running out of disk space in some cases. However, the actual test run is happening in a docker container.
+      # Therefore, we don't actually need most of the tools that github provides. This can save quite a bit of space.
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          tool-cache: true
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: true
+          docker-images: false
+          swap-storage: false
+      - name: "Checkout repository"
+        timeout-minutes: 10
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          lfs: true
+          submodules: recursive
+      - name: "Restore Library cache"
+        id: restore-cache
+        timeout-minutes: 10
+        uses: actions/cache/restore@v3
+        with:
+          path: ${{ env.projectPath }}/Library
+          key: Library-${{ env.projectPath }}-sync-sim-tests-${{ hashFiles(env.projectPath) }}
+          restore-keys: Library-${{ env.projectPath }}-sync-sim-tests-
+      - name: "Run BasisSyncSimTests (EditMode)"
+        timeout-minutes: 60
+        uses: game-ci/unity-test-runner@v4
+        env:
+          UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }}
+          UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
+          UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
+        with:
+          projectPath: ${{ env.projectPath }}
+          unityVersion: ${{ env.unityVersion }}
+          testMode: editmode
+          artifactsPath: ${{ env.testArtifactsPath }}
+          githubToken: ${{ secrets.GITHUB_TOKEN }}
+          checkName: Basis Sync Sim EditMode Tests
+          customParameters: -nographics -testFilter ${{ env.testFilter }}
+      - name: "Save Library Cache"
+        uses: actions/cache/save@v3
+        if: always() && github.ref_name == 'developer'
+        with:
+          path: ${{ env.projectPath }}/Library
+          key: ${{ steps.restore-cache.outputs.cache-primary-key }}
+      - name: "Only retain latest cache"
+        if: always() && github.ref_name == 'developer'
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          OLD_CACHE_IDS=$(gh cache list --sort created_at --key Library-${{ env.projectPath }}-sync-sim-tests- --json id --jq '.[1:] | map(.id) | @sh')
+          for cache_id in $OLD_CACHE_IDS; do
+            echo "Deleting cache id: $cache_id"
+            gh cache delete $cache_id
+          done
+      - name: "Upload test artifacts"
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: sync-sim-test-artifacts-${{ env.unityVersion }}
+          path: ${{ env.testArtifactsPath }}
+          retention-days: 30
+
+  matrix-runner:
+    name: Full Matrix Runner (CSV Export)
+    timeout-minutes: 120
+    runs-on: ubuntu-latest
+    permissions:
+      actions: write # to allow us to manage cache
+      contents: write # to upload release assets on tag dispatches
+    env:
+      projectPath: Basis
+      unityVersion: ${{ github.event.inputs.unityVersion || '6000.5.1f1' }}
+      matrixCsvPath: artifacts/sync-sim-matrix.csv
+      matrixOptionsPath: artifacts/sync-sim-matrix-options.json
+      matrixOptions: '{"Seeds":3,"QuantizeVariants":true,"InterpolateOffVariants":true,"ExtrapolateVariant":true,"TeleportThresholdVariant":true,"CompositeConfigs":true,"MultiObjectScenes":true,"BatchedTransport":true,"RealInterpJob":true,"LateJoinVariant":true,"Dt":0.013888889,"DurationSeconds":6,"SettleSeconds":1.8,"BaseSeed":1523023}'
+    needs: [check-secret]
+    if: github.event_name == 'workflow_dispatch' && needs.check-secret.outputs.secret-is-set == 'true'
+    steps:
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          tool-cache: true
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: true
+          docker-images: false
+          swap-storage: false
+      - name: "Checkout repository"
+        timeout-minutes: 10
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          lfs: true
+          submodules: recursive
+      - name: "Restore Library cache"
+        id: restore-cache
+        timeout-minutes: 10
+        uses: actions/cache/restore@v3
+        with:
+          path: ${{ env.projectPath }}/Library
+          key: Library-${{ env.projectPath }}-sync-sim-matrix-${{ hashFiles(env.projectPath) }}
+          restore-keys: Library-${{ env.projectPath }}-sync-sim-matrix-
+      - name: "Write matrix options"
+        shell: bash
+        run: |
+          mkdir -p "$(dirname "${{ env.matrixOptionsPath }}")"
+          cat > "${{ env.matrixOptionsPath }}" <<'JSON'
+          ${{ env.matrixOptions }}
+          JSON
+      - name: "Run Full Matrix (Headless Editor Script)"
+        timeout-minutes: 120
+        uses: BasisVR/unity-builder@main
+        env:
+          UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }}
+          UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
+          UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
+        with:
+          buildName: sync-sim-matrix
+          buildsPath: artifacts/matrix-build
+          buildMethod: Basis.Scripts.Networking.Sync.Testing.BasisSyncSimMatrix.RunAndExportCSV
+          customParameters: -matrixOptionsPath /github/workspace/${{ env.matrixOptionsPath }} -outputPath /github/workspace/${{ env.matrixCsvPath }}
+          manualExit: true
+          projectPath: ${{ env.projectPath }}
+          targetPlatform: StandaloneLinux64
+          unityVersion: ${{ env.unityVersion }}
+          versioning: None
+          linux64RemoveExecutableExtension: false
+      - name: "Save Library Cache"
+        uses: actions/cache/save@v3
+        if: always() && github.ref_name == 'developer'
+        with:
+          path: ${{ env.projectPath }}/Library
+          key: ${{ steps.restore-cache.outputs.cache-primary-key }}
+      - name: "Only retain latest cache"
+        if: always() && github.ref_name == 'developer'
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          OLD_CACHE_IDS=$(gh cache list --sort created_at --key Library-${{ env.projectPath }}-sync-sim-matrix- --json id --jq '.[1:] | map(.id) | @sh')
+          for cache_id in $OLD_CACHE_IDS; do
+            echo "Deleting cache id: $cache_id"
+            gh cache delete $cache_id
+          done
+      - name: "Upload Matrix CSV"
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: sync-sim-matrix-csv
+          path: ${{ env.matrixCsvPath }}
+          retention-days: 30
+      - name: "Upload Matrix CSV as Release Asset (on tag)"
+        if: github.event_name == 'workflow_dispatch' && startsWith(github.ref, 'refs/tags/')
+        uses: softprops/action-gh-release@v2
+        with:
+          files: ${{ env.matrixCsvPath }}

Basis/Packages/com.basis.framework/Editor/SyncTesting/BasisSyncSimMatrix.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Text;
+using UnityEngine;
 
 namespace Basis.Scripts.Networking.Sync.Testing
 {
@@ -22,6 +23,7 @@ public sealed class FieldConfig
             public bool PositionFirst;
         }
 
+        [Serializable]
         public sealed class MatrixOptions
         {
             public bool QuantizeVariants = true;
@@ -423,6 +425,96 @@ public static string ToCsv(List<BasisSyncSimResult> results)
             return sb.ToString();
         }
 
+        /// <summary>
+        /// Command-line entry point for CI: runs the full matrix and writes CSV to the given path.
+        /// Usage: Unity -batchmode -executeMethod Basis.Scripts.Networking.Sync.Testing.BasisSyncSimMatrix.RunAndExportCSV -matrixOptions '{"Seeds":3}' -outputPath /path/to/output.csv
+        ///        Unity -batchmode -executeMethod Basis.Scripts.Networking.Sync.Testing.BasisSyncSimMatrix.RunAndExportCSV -matrixOptionsPath /path/to/options.json -outputPath /path/to/output.csv
+        /// </summary>
+        public static void RunAndExportCSV()
+        {
+            string matrixOptionsJson = null;
+            string matrixOptionsPath = null;
+            string outputPath = null;
+
+            var args = System.Environment.GetCommandLineArgs();
+            for (int i = 0; i < args.Length; i++)
+            {
+                if (args[i] == "-matrixOptions" && i + 1 < args.Length)
+                    matrixOptionsJson = args[i + 1];
+                else if (args[i] == "-matrixOptionsPath" && i + 1 < args.Length)
+                    matrixOptionsPath = args[i + 1];
+                else if (args[i] == "-outputPath" && i + 1 < args.Length)
+                    outputPath = args[i + 1];
+            }
+
+            if (!string.IsNullOrEmpty(matrixOptionsPath))
+            {
+                try
+                {
+                    matrixOptionsJson = System.IO.File.ReadAllText(matrixOptionsPath);
+                }
+                catch (System.Exception e)
+                {
+                    UnityEngine.Debug.LogError($"Failed to read -matrixOptionsPath '{matrixOptionsPath}': {e.Message}");
+                    UnityEditor.EditorApplication.Exit(1);
+                    return;
+                }
+            }
+
+            MatrixOptions options = MatrixOptions.Full();
+            if (!string.IsNullOrEmpty(matrixOptionsJson))
+            {
+                try
+                {
+                    JsonUtility.FromJsonOverwrite(matrixOptionsJson, options);
+                }
+                catch (System.Exception e)
+                {
+                    UnityEngine.Debug.LogError($"Failed to parse -matrixOptions JSON: {e.Message}");
+                    UnityEditor.EditorApplication.Exit(1);
+                    return;
+                }
+            }
+
+            if (string.IsNullOrEmpty(outputPath))
+            {
+                UnityEngine.Debug.LogError("-outputPath is required");
+                UnityEditor.EditorApplication.Exit(1);
+                return;
+            }
+
+            UnityEngine.Debug.Log($"Running sync sim matrix: {BasisSyncSimMatrix.CountScenarios(options)} scenarios");
+
+            var results = RunAll(options, (done, total, r) =>
+            {
+                if (r != null)
+                    UnityEngine.Debug.Log($"[{done}/{total}] {r.ScenarioName} => {(r.Warn ? "WARN" : (r.Pass ? "PASS" : "FAIL"))} {r.FailReason}");
+            }, null);
+
+            string csv = ToCsv(results);
+            string outputDirectory = System.IO.Path.GetDirectoryName(outputPath);
+            if (!string.IsNullOrEmpty(outputDirectory))
+                System.IO.Directory.CreateDirectory(outputDirectory);
+            System.IO.File.WriteAllText(outputPath, csv);
+
+            int failed = 0;
+            for (int i = 0; i < results.Count; i++)
+            {
+                if (!results[i].Pass)
+                    failed++;
+            }
+
+            UnityEngine.Debug.Log($"CSV written to {outputPath} ({results.Count} rows, {failed} failures)");
+            if (failed > 0)
+            {
+                UnityEngine.Debug.LogError($"Sync sim matrix completed with {failed} failing rows. See CSV for details.");
+                UnityEditor.EditorApplication.Exit(1);
+                return;
+            }
+
+            UnityEditor.EditorApplication.Exit(0);
+        }
+
         static int Hash(string s)
         {
             if (string.IsNullOrEmpty(s)) return 0;