Structured build diagnostics

src/Machine/src/Serval.Machine.Shared/Configuration/BuildJobOptions.cs

@@ -7,4 +7,5 @@ public class BuildJobOptions
     public IList<ClearMLBuildQueue> ClearML { get; set; } = new List<ClearMLBuildQueue>();
     public bool PreserveBuildFiles { get; set; } = false;
     public int MaxWarnings { get; set; } = 1000;
+    public int MaxDiagnostics { get; set; } = 1000;
 }

src/Machine/src/Serval.Machine.Shared/Models/BuildDiagnostic.cs

@@ -0,0 +1,17 @@
+namespace Serval.Machine.Shared.Models;
+
+public record BuildDiagnostic
+{
+    public required string Code { get; init; }
+    public required string Category { get; init; }
+    public required string Message { get; init; }
+    public required BuildDiagnosticSeverity Severity { get; init; }
+    public required Dictionary<string, object> Data { get; init; }
+}
+
+public enum BuildDiagnosticSeverity
+{
+    Info,
+    Warn,
+    Error,
+}

src/Machine/src/Serval.Machine.Shared/Models/BuildExecutionData.cs

@@ -9,6 +9,7 @@ public record BuildExecutionData
     public IReadOnlyDictionary<string, Dictionary<string, int>>? TrainVerseCount { get; init; }
     public IReadOnlyDictionary<string, Dictionary<string, int>>? InferenceVerseCount { get; init; }
     public IReadOnlyList<string>? Warnings { get; init; }
+    public IReadOnlyList<BuildDiagnostic>? Diagnostics { get; init; }
     public string? EngineSourceLanguageTag { get; init; }
     public string? EngineTargetLanguageTag { get; init; }
     public string? ResolvedSourceLanguage { get; init; }

src/Machine/src/Serval.Machine.Shared/Services/EchoPreprocessBuildJob.cs

@@ -30,18 +30,30 @@ protected override async Task UpdateBuildExecutionData(
         PreprocessStats stats,
         string sourceLanguageTag,
         string targetLanguageTag,
+        bool isNonPersistedTranslationEngine,
         IReadOnlyList<ParallelCorpusContract> parallelCorpora,
         CancellationToken cancellationToken
     )
     {
-        IReadOnlyList<string> warnings = GetWarnings(
+        IReadOnlyList<BuildDiagnostic> diagnostics = GetDiagnostics(
             stats.TrainCount,
             stats.InferenceCount,
             sourceLanguageTag,
             targetLanguageTag,
+            sourceLanguageHasNativeSupport: true,
+            targetLanguageHasNativeSupport: true,
+            isNonPersistedTranslationEngine,
             parallelCorpora
         );
 
+        IReadOnlyList<string> warnings = diagnostics.Select(d => d.Message).ToList();
+
+        int maxDiagnostics = BuildJobOptions.MaxDiagnostics;
+        if (diagnostics.Count > maxDiagnostics)
+        {
+            diagnostics = diagnostics.OrderByDescending(d => d.Severity).Take(maxDiagnostics).ToList();
+        }
+
         int maxWarnings = BuildJobOptions.MaxWarnings;
         if (warnings.Count > maxWarnings)
         {
@@ -74,6 +86,7 @@ CancellationToken cancellationToken
             TrainVerseCount = stats.TrainVerseCount,
             InferenceVerseCount = stats.InferenceVerseCount,
             Warnings = warnings,
+            Diagnostics = diagnostics,
             EngineSourceLanguageTag = sourceLanguageTag,
             EngineTargetLanguageTag = targetLanguageTag,
             ResolvedSourceLanguage = sourceLanguageTag,

src/Machine/src/Serval.Machine.Shared/Services/EchoWordAlignmentPreprocessBuildJob.cs

@@ -90,18 +90,38 @@ protected override async Task UpdateBuildExecutionData(
         PreprocessStats stats,
         string sourceLanguageTag,
         string targetLanguageTag,
+        bool isNonPersistedTranslationEngine,
         IReadOnlyList<ParallelCorpusContract> parallelCorpora,
         CancellationToken cancellationToken
     )
     {
-        IReadOnlyList<string> warnings = GetWarnings(
+        IReadOnlyList<BuildDiagnostic> diagnostics = GetDiagnostics(
             stats.TrainCount,
             stats.InferenceCount,
             sourceLanguageTag,
             targetLanguageTag,
+            sourceLanguageHasNativeSupport: true,
+            targetLanguageHasNativeSupport: true,
+            isNonPersistedTranslationEngine,
             parallelCorpora
         );
 
+        IReadOnlyList<string> warnings = diagnostics.Select(d => d.Message).ToList();
+
+        int maxDiagnostics = BuildJobOptions.MaxDiagnostics;
+        if (diagnostics.Count > maxDiagnostics)
+        {
+            diagnostics = diagnostics.OrderByDescending(d => d.Severity).Take(maxDiagnostics).ToList();
+        }
+
+        int maxWarnings = BuildJobOptions.MaxWarnings;
+        if (warnings.Count > maxWarnings)
+        {
+            string tooManyWarningsWarning =
+                $"There were {warnings.Count} warnings. Only the first {maxWarnings} are shown.";
+            warnings = [tooManyWarningsWarning, .. warnings.Take(maxWarnings)];
+        }
+
         // Log summary of build data
         var buildPreprocessSummary = new JsonObject
         {
@@ -124,6 +144,7 @@ CancellationToken cancellationToken
             IsInferenceFilteredByChapter = stats.IsInferenceFilteredByChapter,
             IsTrainFilteredByChapter = stats.IsTrainFilteredByChapter,
             Warnings = warnings,
+            Diagnostics = diagnostics,
             EngineSourceLanguageTag = sourceLanguageTag,
             EngineTargetLanguageTag = targetLanguageTag,
         };

src/Machine/src/Serval.Machine.Shared/Services/NmtPreprocessBuildJob.cs

@@ -23,6 +23,8 @@ IOptionsMonitor<BuildJobOptions> options
     )
 {
     private readonly ILanguageTagService _languageTagService = languageTagService;
+    private const string ModelName = "NLLB";
+    private const int MinimumTrainCount = 600; //TODO move to options?
 
     private bool ResolveLanguageCode(string languageCode, out string resolvedCode)
     {
@@ -55,28 +57,33 @@ protected override async Task UpdateBuildExecutionData(
         PreprocessStats stats,
         string sourceLanguageTag,
         string targetLanguageTag,
+        bool isNonPersistedTranslationEngine,
         IReadOnlyList<ParallelCorpusContract> parallelCorpora,
         CancellationToken cancellationToken
     )
     {
         bool sourceLanguageHasNativeSupport = ResolveLanguageCode(sourceLanguageTag, out string resolvedSourceLanguage);
         bool targetLanguageHasNativeSupport = ResolveLanguageCode(targetLanguageTag, out string resolvedTargetLanguage);
 
-        if (stats.TrainCount == 0 && (!sourceLanguageHasNativeSupport || !targetLanguageHasNativeSupport))
-        {
-            throw new InvalidOperationException(
-                $"At least one language code in build {buildId} is unknown to the base model, and the data specified for training was empty. Build canceled."
-            );
-        }
-
-        IReadOnlyList<string> warnings = GetWarnings(
+        IReadOnlyList<BuildDiagnostic> diagnostics = GetDiagnostics(
             stats.TrainCount,
             stats.InferenceCount,
             sourceLanguageTag,
             targetLanguageTag,
+            sourceLanguageHasNativeSupport,
+            targetLanguageHasNativeSupport,
+            isNonPersistedTranslationEngine,
             parallelCorpora
         );
 
+        IReadOnlyList<string> warnings = diagnostics.Select(d => d.Message).ToList();
+
+        int maxDiagnostics = BuildJobOptions.MaxDiagnostics;
+        if (diagnostics.Count > maxDiagnostics)
+        {
+            diagnostics = diagnostics.OrderByDescending(d => d.Severity).Take(maxDiagnostics).ToList();
+        }
+
         int maxWarnings = BuildJobOptions.MaxWarnings;
         if (warnings.Count > maxWarnings)
         {
@@ -109,46 +116,134 @@ CancellationToken cancellationToken
             IsTrainFilteredByChapter = stats.IsTrainFilteredByChapter,
             IsInferenceFilteredByChapter = stats.IsInferenceFilteredByChapter,
             Warnings = warnings,
+            Diagnostics = diagnostics,
             EngineSourceLanguageTag = sourceLanguageTag,
             EngineTargetLanguageTag = targetLanguageTag,
             ResolvedSourceLanguage = resolvedSourceLanguage,
             ResolvedTargetLanguage = resolvedTargetLanguage,
         };
         await PlatformService.UpdateBuildExecutionDataAsync(engineId, buildId, executionData, cancellationToken);
+
+        if (stats.TrainCount == 0 && (!sourceLanguageHasNativeSupport || !targetLanguageHasNativeSupport))
+        {
+            throw new InvalidOperationException(
+                $"At least one language code in build {buildId} is unknown to the base model {ModelName}, and no data was specified for training. Build canceled."
+            );
+        }
     }
 
-    protected override IReadOnlyList<string> GetWarnings(
+    protected override IReadOnlyList<BuildDiagnostic> GetDiagnostics(
         int trainCount,
         int inferenceCount,
         string sourceLanguageTag,
         string targetLanguageTag,
+        bool sourceLanguageHasNativeSupport,
+        bool targetLanguageHasNativeSupport,
+        bool isNonPersistedTranslationEngine,
         IReadOnlyList<ParallelCorpusContract> parallelCorpora
     )
     {
-        List<string> warnings =
+        List<BuildDiagnostic> diagnostics =
         [
-            .. base.GetWarnings(trainCount, inferenceCount, sourceLanguageTag, targetLanguageTag, parallelCorpora),
+            .. base.GetDiagnostics(
+                trainCount,
+                inferenceCount,
+                sourceLanguageTag,
+                targetLanguageTag,
+                sourceLanguageHasNativeSupport,
+                targetLanguageHasNativeSupport,
+                isNonPersistedTranslationEngine,
+                parallelCorpora
+            ),
         ];
 
         // Has at least a Gospel of Mark amount of data and not the special case of no data which will be caught elsewhere
         if (trainCount < 600 && trainCount != 0)
         {
-            warnings.Add($"Only {trainCount} segments were selected for training.");
+            diagnostics.Add(
+                new BuildDiagnostic
+                {
+                    Code = "CONFIG-0003",
+                    Category = "CONFIG",
+                    Severity = BuildDiagnosticSeverity.Warn,
+                    Message =
+                        $"Only {trainCount} segments were selected for training. Training on fewer than {MinimumTrainCount} is not recommended.",
+                    Data = new Dictionary<string, object>
+                    {
+                        { "trainCount", trainCount },
+                        { "minimumTrainCount", MinimumTrainCount },
+                    },
+                }
+            );
         }
 
         if (
             _languageTagService.ConvertToFlores200Code(sourceLanguageTag, out string resolvedCode)
             == Flores200Support.None
         )
         {
-            warnings.Add($"The script for the source language '{resolvedCode}' is not in Flores-200");
+            diagnostics.Add(
+                new BuildDiagnostic
+                {
+                    Code = "MODEL-0001",
+                    Category = "MODEL",
+                    Severity = BuildDiagnosticSeverity.Warn,
+                    Message =
+                        $"The script for the source language '{resolvedCode}' is not known to the base model {ModelName}",
+                    Data = new Dictionary<string, object>
+                    {
+                        { "resolvedCode", resolvedCode },
+                        { "modelName", ModelName },
+                    },
+                }
+            );
         }
 
         if (_languageTagService.ConvertToFlores200Code(targetLanguageTag, out resolvedCode) == Flores200Support.None)
         {
-            warnings.Add($"The script for the target language '{resolvedCode}' is not in Flores-200");
+            diagnostics.Add(
+                new BuildDiagnostic
+                {
+                    Code = "MODEL-0002",
+                    Category = "MODEL",
+                    Severity = BuildDiagnosticSeverity.Warn,
+                    Message =
+                        $"The script for the target language '{resolvedCode}' is not known to the base model {ModelName}",
+                    Data = new Dictionary<string, object>
+                    {
+                        { "resolvedCode", resolvedCode },
+                        { "modelName", ModelName },
+                    },
+                }
+            );
+        }
+
+        if (trainCount == 0 && (!sourceLanguageHasNativeSupport || !targetLanguageHasNativeSupport))
+        {
+            List<string> unknownLanguageCodes = new[]
+            {
+                !sourceLanguageHasNativeSupport ? sourceLanguageTag : "",
+                !targetLanguageHasNativeSupport ? targetLanguageTag : "",
+            }
+                .Where(s => !string.IsNullOrEmpty(s))
+                .ToList();
+            diagnostics.Add(
+                new BuildDiagnostic
+                {
+                    Code = "MODEL-0004",
+                    Category = "MODEL",
+                    Severity = BuildDiagnosticSeverity.Error,
+                    Message =
+                        $"The following language codes are unknown to the base model {ModelName}: {string.Join(", ", unknownLanguageCodes)}; and no language data was selected for training.",
+                    Data = new Dictionary<string, object>
+                    {
+                        { "modelName", ModelName },
+                        { "unknownLanguageCodes", unknownLanguageCodes },
+                    },
+                }
+            );
         }
 
-        return warnings;
+        return diagnostics;
     }
 }