diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/BuildJobOptions.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/BuildJobOptions.cs index b6b9da57..efc982f2 100644 --- a/src/Machine/src/Serval.Machine.Shared/Configuration/BuildJobOptions.cs +++ b/src/Machine/src/Serval.Machine.Shared/Configuration/BuildJobOptions.cs @@ -7,4 +7,5 @@ public class BuildJobOptions public IList ClearML { get; set; } = new List(); public bool PreserveBuildFiles { get; set; } = false; public int MaxWarnings { get; set; } = 1000; + public int MaxDiagnostics { get; set; } = 1000; } diff --git a/src/Machine/src/Serval.Machine.Shared/Models/BuildDiagnostic.cs b/src/Machine/src/Serval.Machine.Shared/Models/BuildDiagnostic.cs new file mode 100644 index 00000000..bc973371 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Models/BuildDiagnostic.cs @@ -0,0 +1,17 @@ +namespace Serval.Machine.Shared.Models; + +public record BuildDiagnostic +{ + public required string Code { get; init; } + public required string Category { get; init; } + public required string Message { get; init; } + public required BuildDiagnosticSeverity Severity { get; init; } + public required Dictionary Data { get; init; } +} + +public enum BuildDiagnosticSeverity +{ + Info, + Warn, + Error, +} diff --git a/src/Machine/src/Serval.Machine.Shared/Models/BuildExecutionData.cs b/src/Machine/src/Serval.Machine.Shared/Models/BuildExecutionData.cs index 004ad374..3faddde4 100644 --- a/src/Machine/src/Serval.Machine.Shared/Models/BuildExecutionData.cs +++ b/src/Machine/src/Serval.Machine.Shared/Models/BuildExecutionData.cs @@ -9,6 +9,7 @@ public record BuildExecutionData public IReadOnlyDictionary>? TrainVerseCount { get; init; } public IReadOnlyDictionary>? InferenceVerseCount { get; init; } public IReadOnlyList? Warnings { get; init; } + public IReadOnlyList? Diagnostics { get; init; } public string? EngineSourceLanguageTag { get; init; } public string? EngineTargetLanguageTag { get; init; } public string? ResolvedSourceLanguage { get; init; } diff --git a/src/Machine/src/Serval.Machine.Shared/Services/EchoPreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/EchoPreprocessBuildJob.cs index 2f23ae9a..aabc1e11 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/EchoPreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/EchoPreprocessBuildJob.cs @@ -30,18 +30,30 @@ protected override async Task UpdateBuildExecutionData( PreprocessStats stats, string sourceLanguageTag, string targetLanguageTag, + bool isNonPersistedTranslationEngine, IReadOnlyList parallelCorpora, CancellationToken cancellationToken ) { - IReadOnlyList warnings = GetWarnings( + IReadOnlyList diagnostics = GetDiagnostics( stats.TrainCount, stats.InferenceCount, sourceLanguageTag, targetLanguageTag, + sourceLanguageHasNativeSupport: true, + targetLanguageHasNativeSupport: true, + isNonPersistedTranslationEngine, parallelCorpora ); + IReadOnlyList warnings = diagnostics.Select(d => d.Message).ToList(); + + int maxDiagnostics = BuildJobOptions.MaxDiagnostics; + if (diagnostics.Count > maxDiagnostics) + { + diagnostics = diagnostics.OrderByDescending(d => d.Severity).Take(maxDiagnostics).ToList(); + } + int maxWarnings = BuildJobOptions.MaxWarnings; if (warnings.Count > maxWarnings) { @@ -74,6 +86,7 @@ CancellationToken cancellationToken TrainVerseCount = stats.TrainVerseCount, InferenceVerseCount = stats.InferenceVerseCount, Warnings = warnings, + Diagnostics = diagnostics, EngineSourceLanguageTag = sourceLanguageTag, EngineTargetLanguageTag = targetLanguageTag, ResolvedSourceLanguage = sourceLanguageTag, diff --git a/src/Machine/src/Serval.Machine.Shared/Services/EchoWordAlignmentPreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/EchoWordAlignmentPreprocessBuildJob.cs index 4c444e53..0be276a2 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/EchoWordAlignmentPreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/EchoWordAlignmentPreprocessBuildJob.cs @@ -90,18 +90,38 @@ protected override async Task UpdateBuildExecutionData( PreprocessStats stats, string sourceLanguageTag, string targetLanguageTag, + bool isNonPersistedTranslationEngine, IReadOnlyList parallelCorpora, CancellationToken cancellationToken ) { - IReadOnlyList warnings = GetWarnings( + IReadOnlyList diagnostics = GetDiagnostics( stats.TrainCount, stats.InferenceCount, sourceLanguageTag, targetLanguageTag, + sourceLanguageHasNativeSupport: true, + targetLanguageHasNativeSupport: true, + isNonPersistedTranslationEngine, parallelCorpora ); + IReadOnlyList warnings = diagnostics.Select(d => d.Message).ToList(); + + int maxDiagnostics = BuildJobOptions.MaxDiagnostics; + if (diagnostics.Count > maxDiagnostics) + { + diagnostics = diagnostics.OrderByDescending(d => d.Severity).Take(maxDiagnostics).ToList(); + } + + int maxWarnings = BuildJobOptions.MaxWarnings; + if (warnings.Count > maxWarnings) + { + string tooManyWarningsWarning = + $"There were {warnings.Count} warnings. Only the first {maxWarnings} are shown."; + warnings = [tooManyWarningsWarning, .. warnings.Take(maxWarnings)]; + } + // Log summary of build data var buildPreprocessSummary = new JsonObject { @@ -124,6 +144,7 @@ CancellationToken cancellationToken IsInferenceFilteredByChapter = stats.IsInferenceFilteredByChapter, IsTrainFilteredByChapter = stats.IsTrainFilteredByChapter, Warnings = warnings, + Diagnostics = diagnostics, EngineSourceLanguageTag = sourceLanguageTag, EngineTargetLanguageTag = targetLanguageTag, }; diff --git a/src/Machine/src/Serval.Machine.Shared/Services/NmtPreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/NmtPreprocessBuildJob.cs index dfd38985..bd0691bd 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/NmtPreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/NmtPreprocessBuildJob.cs @@ -23,6 +23,8 @@ IOptionsMonitor options ) { private readonly ILanguageTagService _languageTagService = languageTagService; + private const string ModelName = "NLLB"; + private const int MinimumTrainCount = 600; //TODO move to options? private bool ResolveLanguageCode(string languageCode, out string resolvedCode) { @@ -55,6 +57,7 @@ protected override async Task UpdateBuildExecutionData( PreprocessStats stats, string sourceLanguageTag, string targetLanguageTag, + bool isNonPersistedTranslationEngine, IReadOnlyList parallelCorpora, CancellationToken cancellationToken ) @@ -62,21 +65,25 @@ CancellationToken cancellationToken bool sourceLanguageHasNativeSupport = ResolveLanguageCode(sourceLanguageTag, out string resolvedSourceLanguage); bool targetLanguageHasNativeSupport = ResolveLanguageCode(targetLanguageTag, out string resolvedTargetLanguage); - if (stats.TrainCount == 0 && (!sourceLanguageHasNativeSupport || !targetLanguageHasNativeSupport)) - { - throw new InvalidOperationException( - $"At least one language code in build {buildId} is unknown to the base model, and the data specified for training was empty. Build canceled." - ); - } - - IReadOnlyList warnings = GetWarnings( + IReadOnlyList diagnostics = GetDiagnostics( stats.TrainCount, stats.InferenceCount, sourceLanguageTag, targetLanguageTag, + sourceLanguageHasNativeSupport, + targetLanguageHasNativeSupport, + isNonPersistedTranslationEngine, parallelCorpora ); + IReadOnlyList warnings = diagnostics.Select(d => d.Message).ToList(); + + int maxDiagnostics = BuildJobOptions.MaxDiagnostics; + if (diagnostics.Count > maxDiagnostics) + { + diagnostics = diagnostics.OrderByDescending(d => d.Severity).Take(maxDiagnostics).ToList(); + } + int maxWarnings = BuildJobOptions.MaxWarnings; if (warnings.Count > maxWarnings) { @@ -109,31 +116,65 @@ CancellationToken cancellationToken IsTrainFilteredByChapter = stats.IsTrainFilteredByChapter, IsInferenceFilteredByChapter = stats.IsInferenceFilteredByChapter, Warnings = warnings, + Diagnostics = diagnostics, EngineSourceLanguageTag = sourceLanguageTag, EngineTargetLanguageTag = targetLanguageTag, ResolvedSourceLanguage = resolvedSourceLanguage, ResolvedTargetLanguage = resolvedTargetLanguage, }; await PlatformService.UpdateBuildExecutionDataAsync(engineId, buildId, executionData, cancellationToken); + + if (stats.TrainCount == 0 && (!sourceLanguageHasNativeSupport || !targetLanguageHasNativeSupport)) + { + throw new InvalidOperationException( + $"At least one language code in build {buildId} is unknown to the base model {ModelName}, and no data was specified for training. Build canceled." + ); + } } - protected override IReadOnlyList GetWarnings( + protected override IReadOnlyList GetDiagnostics( int trainCount, int inferenceCount, string sourceLanguageTag, string targetLanguageTag, + bool sourceLanguageHasNativeSupport, + bool targetLanguageHasNativeSupport, + bool isNonPersistedTranslationEngine, IReadOnlyList parallelCorpora ) { - List warnings = + List diagnostics = [ - .. base.GetWarnings(trainCount, inferenceCount, sourceLanguageTag, targetLanguageTag, parallelCorpora), + .. base.GetDiagnostics( + trainCount, + inferenceCount, + sourceLanguageTag, + targetLanguageTag, + sourceLanguageHasNativeSupport, + targetLanguageHasNativeSupport, + isNonPersistedTranslationEngine, + parallelCorpora + ), ]; // Has at least a Gospel of Mark amount of data and not the special case of no data which will be caught elsewhere if (trainCount < 600 && trainCount != 0) { - warnings.Add($"Only {trainCount} segments were selected for training."); + diagnostics.Add( + new BuildDiagnostic + { + Code = "CONFIG-0003", + Category = "CONFIG", + Severity = BuildDiagnosticSeverity.Warn, + Message = + $"Only {trainCount} segments were selected for training. Training on fewer than {MinimumTrainCount} is not recommended.", + Data = new Dictionary + { + { "trainCount", trainCount }, + { "minimumTrainCount", MinimumTrainCount }, + }, + } + ); } if ( @@ -141,14 +182,68 @@ .. base.GetWarnings(trainCount, inferenceCount, sourceLanguageTag, targetLanguag == Flores200Support.None ) { - warnings.Add($"The script for the source language '{resolvedCode}' is not in Flores-200"); + diagnostics.Add( + new BuildDiagnostic + { + Code = "MODEL-0001", + Category = "MODEL", + Severity = BuildDiagnosticSeverity.Warn, + Message = + $"The script for the source language '{resolvedCode}' is not known to the base model {ModelName}", + Data = new Dictionary + { + { "resolvedCode", resolvedCode }, + { "modelName", ModelName }, + }, + } + ); } if (_languageTagService.ConvertToFlores200Code(targetLanguageTag, out resolvedCode) == Flores200Support.None) { - warnings.Add($"The script for the target language '{resolvedCode}' is not in Flores-200"); + diagnostics.Add( + new BuildDiagnostic + { + Code = "MODEL-0002", + Category = "MODEL", + Severity = BuildDiagnosticSeverity.Warn, + Message = + $"The script for the target language '{resolvedCode}' is not known to the base model {ModelName}", + Data = new Dictionary + { + { "resolvedCode", resolvedCode }, + { "modelName", ModelName }, + }, + } + ); + } + + if (trainCount == 0 && (!sourceLanguageHasNativeSupport || !targetLanguageHasNativeSupport)) + { + List unknownLanguageCodes = new[] + { + !sourceLanguageHasNativeSupport ? sourceLanguageTag : "", + !targetLanguageHasNativeSupport ? targetLanguageTag : "", + } + .Where(s => !string.IsNullOrEmpty(s)) + .ToList(); + diagnostics.Add( + new BuildDiagnostic + { + Code = "MODEL-0004", + Category = "MODEL", + Severity = BuildDiagnosticSeverity.Error, + Message = + $"The following language codes are unknown to the base model {ModelName}: {string.Join(", ", unknownLanguageCodes)}; and no language data was selected for training.", + Data = new Dictionary + { + { "modelName", ModelName }, + { "unknownLanguageCodes", unknownLanguageCodes }, + }, + } + ); } - return warnings; + return diagnostics; } } diff --git a/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs index 22607aba..ce550252 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs @@ -46,6 +46,7 @@ CancellationToken cancellationToken throw new OperationCanceledException($"Engine {engineId} does not exist. Build canceled."); PreprocessStats stats = await WriteDataFilesAsync(engineId, buildId, data, buildOptions, cancellationToken); + bool isNonPersistedTranslationEngine = engine is TranslationEngine { IsModelPersisted: false }; await UpdateBuildExecutionData( engineId, @@ -53,16 +54,17 @@ await UpdateBuildExecutionData( stats, engine.SourceLanguage, engine.TargetLanguage, + isNonPersistedTranslationEngine, data, cancellationToken ); await UpdateTargetQuoteConventionAsync(engineId, buildId, data, cancellationToken); - if (stats.InferenceCount == 0 && engine is TranslationEngine { IsModelPersisted: false }) + if (stats.InferenceCount == 0 && isNonPersistedTranslationEngine) { throw new InvalidOperationException( - $"There was no data specified for inferencing in build {buildId}. Build canceled." + $"There was no data specified for inferencing in build {buildId} and the model is not persisted. Build canceled." ); } @@ -87,6 +89,7 @@ protected abstract Task UpdateBuildExecutionData( PreprocessStats stats, string sourceLanguageTag, string targetLanguageTag, + bool isNonPersistedTranslationEngine, IReadOnlyList parallelCorpora, CancellationToken cancellationToken ); @@ -121,53 +124,175 @@ protected override async Task CleanupAsync(string engineId, string buildId, JobC } } - protected virtual IReadOnlyList GetWarnings( + protected virtual IReadOnlyList GetDiagnostics( int trainCount, int inferenceCount, string sourceLanguageTag, string targetLanguageTag, + bool sourceLanguageHasNativeSupport, + bool targetLanguageHasNativeSupport, + bool isNonPersistedTranslationEngine, IReadOnlyList parallelCorpora ) { - List warnings = []; - HashSet versifications = []; + List diagnostics = []; + Dictionary projectVersifications = []; foreach ( ( string parallelCorpusId, string monolingualCorpusId, string projectName, + string projectGuid, string versificationName, - IReadOnlyList diagnostics + IReadOnlyList usfmDiagnostics ) in ParallelCorpusService.AnalyzeUsfmVersification(parallelCorpora) ) { - versifications.Add(versificationName); - foreach (UsfmVersificationDiagnosticContract diagnostic in diagnostics) + projectVersifications[projectGuid] = versificationName; + foreach (UsfmVersificationDiagnosticContract usfmDiagnostic in usfmDiagnostics) { string diagnosticDetails = - $"in project {projectName} at {diagnostic.Filename} " - + (diagnostic.LineNumbers.Count == 1 ? "line " : "lines ") - + $"{string.Join(", ", diagnostic.LineNumbers)}, " - + (diagnostic.NumAffectedVerses == 1 ? "verse " : "verses ") - + $"{string.Join(", ", diagnostic.References)} " + $"in project {projectName} {projectGuid} at {usfmDiagnostic.Filename} " + + (usfmDiagnostic.LineNumbers.Count == 1 ? "line " : "lines ") + + $"{string.Join(", ", usfmDiagnostic.LineNumbers)}, " + + (usfmDiagnostic.NumAffectedVerses == 1 ? "verse " : "verses ") + + $"{string.Join(", ", usfmDiagnostic.References)} " + $"(parallel corpus {parallelCorpusId}, monolingual corpus {monolingualCorpusId})."; - warnings.Add( - diagnostic.Type switch + diagnostics.Add( + usfmDiagnostic.Type switch { - Serval.Shared.Contracts.UsfmVersificationDiagnosticType.InvalidChapter => - $"Invalid chapter number {diagnosticDetails}", - Serval.Shared.Contracts.UsfmVersificationDiagnosticType.InvalidVerse => - $"Invalid verse number {diagnosticDetails}", - Serval.Shared.Contracts.UsfmVersificationDiagnosticType.Extra => - $"{diagnostic.NumAffectedVerses} extra verses {diagnosticDetails}", - Serval.Shared.Contracts.UsfmVersificationDiagnosticType.Missing => - $"Missing {diagnostic.NumAffectedVerses} verses {diagnosticDetails}", + Serval.Shared.Contracts.UsfmVersificationDiagnosticType.InvalidChapter => new BuildDiagnostic + { + Code = "USFM-0001", + Category = "USFM", + Severity = BuildDiagnosticSeverity.Warn, + Message = $"Invalid chapter number {diagnosticDetails}", + Data = new Dictionary + { + { "projectName", projectName }, + { "projectGuid", projectGuid }, + { "usfmFilename", usfmDiagnostic.Filename }, + { + "lineNumber", + usfmDiagnostic.LineNumbers.Count > 0 ? usfmDiagnostic.LineNumbers[0] : -1 + }, + { + "verseReference", + usfmDiagnostic.References.Count > 0 ? usfmDiagnostic.References[0] : "" + }, + { "parallelCorpusId", parallelCorpusId }, + { "monolingualCorpusId", monolingualCorpusId }, + }, + }, + Serval.Shared.Contracts.UsfmVersificationDiagnosticType.InvalidVerse => new BuildDiagnostic + { + Code = "USFM-0002", + Category = "USFM", + Severity = BuildDiagnosticSeverity.Warn, + Message = $"Invalid verse number {diagnosticDetails}", + Data = new Dictionary + { + { "projectName", projectName }, + { "projectGuid", projectGuid }, + { "usfmFilename", usfmDiagnostic.Filename }, + { + "lineNumber", + usfmDiagnostic.LineNumbers.Count > 0 ? usfmDiagnostic.LineNumbers[0] : -1 + }, + { + "verseReference", + usfmDiagnostic.References.Count > 0 ? usfmDiagnostic.References[0] : "" + }, + { "parallelCorpusId", parallelCorpusId }, + { "monolingualCorpusId", monolingualCorpusId }, + }, + }, + Serval.Shared.Contracts.UsfmVersificationDiagnosticType.Extra => new BuildDiagnostic + { + Code = "USFM-0003", + Category = "USFM", + Severity = BuildDiagnosticSeverity.Info, + Message = $"{usfmDiagnostic.NumAffectedVerses} extra verses {diagnosticDetails}", + Data = new Dictionary + { + { "numberOfVerses", usfmDiagnostic.NumAffectedVerses }, + { "projectName", projectName }, + { "projectGuid", projectGuid }, + { "usfmFilename", usfmDiagnostic.Filename }, + { "lineNumbers", usfmDiagnostic.LineNumbers }, + { "verseReferences", usfmDiagnostic.References }, + { "parallelCorpusId", parallelCorpusId }, + { "monolingualCorpusId", monolingualCorpusId }, + }, + }, + Serval.Shared.Contracts.UsfmVersificationDiagnosticType.Missing => new BuildDiagnostic + { + Code = "USFM-0004", + Category = "USFM", + Severity = BuildDiagnosticSeverity.Warn, + Message = $"Missing {usfmDiagnostic.NumAffectedVerses} verses {diagnosticDetails}", + Data = new Dictionary + { + { "numberOfVerses", usfmDiagnostic.NumAffectedVerses }, + { "projectName", projectName }, + { "projectGuid", projectGuid }, + { "usfmFilename", usfmDiagnostic.Filename }, + { "lineNumbers", usfmDiagnostic.LineNumbers }, + { "verseReferences", usfmDiagnostic.References }, + { "parallelCorpusId", parallelCorpusId }, + { "monolingualCorpusId", monolingualCorpusId }, + }, + }, Serval.Shared.Contracts.UsfmVersificationDiagnosticType.IncorrectVerseSegment => - $"Incorrect verse segment {diagnosticDetails}", + new BuildDiagnostic + { + Code = "USFM-0005", + Category = "USFM", + Severity = BuildDiagnosticSeverity.Info, + Message = $"Incorrect verse segment {diagnosticDetails}", + Data = new Dictionary + { + { "projectName", projectName }, + { "projectGuid", projectGuid }, + { "usfmFilename", usfmDiagnostic.Filename }, + { + "lineNumber", + usfmDiagnostic.LineNumbers.Count > 0 ? usfmDiagnostic.LineNumbers[0] : -1 + }, + { + "verseReference", + usfmDiagnostic.References.Count > 0 ? usfmDiagnostic.References[0] : "" + }, + { "parallelCorpusId", parallelCorpusId }, + { "monolingualCorpusId", monolingualCorpusId }, + }, + }, Serval.Shared.Contracts.UsfmVersificationDiagnosticType.UnsupportedVerseRange => - $"Unsupported verse range {diagnosticDetails}", - _ => $"USFM versification issue {diagnosticDetails}", + new BuildDiagnostic + { + Code = "USFM-0006", + Category = "USFM", + Severity = BuildDiagnosticSeverity.Info, + Message = $"Unsupported verse range {diagnosticDetails}", + Data = new Dictionary + { + { "projectName", projectName }, + { "projectGuid", projectGuid }, + { "usfmFilename", usfmDiagnostic.Filename }, + { + "lineNumber", + usfmDiagnostic.LineNumbers.Count > 0 ? usfmDiagnostic.LineNumbers[0] : -1 + }, + { + "verseReference", + usfmDiagnostic.References.Count > 0 ? usfmDiagnostic.References[0] : "" + }, + { "parallelCorpusId", parallelCorpusId }, + { "monolingualCorpusId", monolingualCorpusId }, + }, + }, + _ => throw new InvalidEnumArgumentException(nameof(usfmDiagnostic.Type)), } ); } @@ -181,19 +306,56 @@ MissingParentProjectErrorContract error ) in ParallelCorpusService.FindMissingParentProjects(parallelCorpora) ) { - warnings.Add( - $"Unable to locate parent project {error.ParentProjectName} of daughter project {error.ProjectName} (parallel corpus {parallelCorpusId}, monolingual corpus {monolingualCorpusId})" + diagnostics.Add( + new BuildDiagnostic + { + Code = "CONFIG-0001", + Category = "CONFIG", + Severity = BuildDiagnosticSeverity.Warn, + Message = + $"Unable to locate parent project {error.ParentProjectName} {error.ParentProjectGuid} of daughter project {error.ProjectName} {error.ProjectGuid} (parallel corpus {parallelCorpusId}, monolingual corpus {monolingualCorpusId})", + Data = new Dictionary + { + { "parentProjectName", error.ParentProjectName }, + { "parentProjectGuid", error.ParentProjectGuid }, + { "daughterProjectName", error.ProjectName }, + { "daughterProjectGuid", error.ProjectGuid }, + { "parallelCorpusId", parallelCorpusId }, + { "monolingualCorpusId", monolingualCorpusId }, + }, + } ); } - if (versifications.Count > 1) + if (projectVersifications.Values.Distinct().Count() > 1) { - warnings.Add( - $"Multiple versifications represented among Paratext projects selected for training or inferencing: {string.Join(", ", versifications)}" + diagnostics.Add( + new BuildDiagnostic + { + Code = "CONFIG-0002", + Category = "CONFIG", + Severity = BuildDiagnosticSeverity.Info, + Message = + $"There are multiple versifications represented among Paratext projects selected for training or inferencing: {string.Join(", ", projectVersifications)}.", + Data = new Dictionary { { "projectVersifications", projectVersifications } }, + } ); } - return warnings; + if (inferenceCount == 0 && isNonPersistedTranslationEngine) + { + diagnostics.Add( + new BuildDiagnostic + { + Code = "CONFIG-0004", + Category = "CONFIG", + Severity = BuildDiagnosticSeverity.Error, + Message = "There was no data specified for inferencing and the model is not persisted.", + Data = [], + } + ); + } + return diagnostics; } protected static (bool IsTrainFilteredByChapter, bool IsInferenceFilteredByChapter) CheckChapterFilters( diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationPlatformService.cs b/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationPlatformService.cs index d48b10fd..f27999b9 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationPlatformService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationPlatformService.cs @@ -105,6 +105,16 @@ public Task UpdateBuildExecutionDataAsync( IsTrainFilteredByChapter = executionData.IsTrainFilteredByChapter, IsPretranslateFilteredByChapter = executionData.IsInferenceFilteredByChapter, Warnings = executionData.Warnings, + Diagnostics = executionData + .Diagnostics?.Select(d => new Translation.Contracts.DiagnosticContract + { + Code = d.Code, + Category = d.Category, + Message = d.Message, + Severity = (Translation.Contracts.DiagnosticSeverity)d.Severity, + Data = d.Data, + }) + .ToList(), EngineSourceLanguageTag = executionData.EngineSourceLanguageTag, EngineTargetLanguageTag = executionData.EngineTargetLanguageTag, ResolvedSourceLanguage = executionData.ResolvedSourceLanguage, diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ServalWordAlignmentPlatformService.cs b/src/Machine/src/Serval.Machine.Shared/Services/ServalWordAlignmentPlatformService.cs index 45eb75b1..45f27268 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/ServalWordAlignmentPlatformService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/ServalWordAlignmentPlatformService.cs @@ -104,6 +104,18 @@ public Task UpdateBuildExecutionDataAsync( TrainVerseCount = executionData.TrainVerseCount, WordAlignVerseCount = executionData.InferenceVerseCount, Warnings = executionData.Warnings, + Diagnostics = + executionData + .Diagnostics?.Select(d => new WordAlignment.Contracts.DiagnosticContract + { + Code = d.Code, + Category = d.Category, + Message = d.Message, + Severity = (WordAlignment.Contracts.DiagnosticSeverity)d.Severity, + Data = d.Data, + }) + .ToList() + ?? [], EngineSourceLanguageTag = executionData.EngineSourceLanguageTag, EngineTargetLanguageTag = executionData.EngineTargetLanguageTag, }, diff --git a/src/Machine/src/Serval.Machine.Shared/Services/TranslationPreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/TranslationPreprocessBuildJob.cs index 190ac258..22d0142f 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/TranslationPreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/TranslationPreprocessBuildJob.cs @@ -89,18 +89,38 @@ protected override async Task UpdateBuildExecutionData( PreprocessStats stats, string sourceLanguageTag, string targetLanguageTag, + bool isNonPersistedTranslationEngine, IReadOnlyList parallelCorpora, CancellationToken cancellationToken ) { - IReadOnlyList warnings = GetWarnings( + IReadOnlyList diagnostics = GetDiagnostics( stats.TrainCount, stats.InferenceCount, sourceLanguageTag, targetLanguageTag, + true, + true, + isNonPersistedTranslationEngine, parallelCorpora ); + IReadOnlyList warnings = diagnostics.Select(d => d.Message).ToList(); + + int maxDiagnostics = BuildJobOptions.MaxDiagnostics; + if (diagnostics.Count > maxDiagnostics) + { + diagnostics = diagnostics.OrderByDescending(d => d.Severity).Take(maxDiagnostics).ToList(); + } + + int maxWarnings = BuildJobOptions.MaxWarnings; + if (warnings.Count > maxWarnings) + { + string tooManyWarningsWarning = + $"There were {warnings.Count} warnings. Only the first {maxWarnings} are shown."; + warnings = [tooManyWarningsWarning, .. warnings.Take(maxWarnings)]; + } + // Log summary of build data JsonObject buildPreprocessSummary = new() { @@ -123,6 +143,7 @@ CancellationToken cancellationToken TrainVerseCount = stats.TrainVerseCount, InferenceVerseCount = stats.InferenceVerseCount, Warnings = warnings, + Diagnostics = diagnostics, EngineSourceLanguageTag = sourceLanguageTag, EngineTargetLanguageTag = targetLanguageTag, }; diff --git a/src/Machine/src/Serval.Machine.Shared/Services/WordAlignmentPreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/WordAlignmentPreprocessBuildJob.cs index 2928815a..ee8bafb0 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/WordAlignmentPreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/WordAlignmentPreprocessBuildJob.cs @@ -88,18 +88,38 @@ protected override async Task UpdateBuildExecutionData( PreprocessStats stats, string sourceLanguageTag, string targetLanguageTag, + bool isNonPersistedTranslationEngine, IReadOnlyList parallelCorpora, CancellationToken cancellationToken ) { - IReadOnlyList warnings = GetWarnings( + IReadOnlyList diagnostics = GetDiagnostics( stats.TrainCount, stats.InferenceCount, sourceLanguageTag, targetLanguageTag, + true, + true, + isNonPersistedTranslationEngine, parallelCorpora ); + IReadOnlyList warnings = diagnostics.Select(d => d.Message).ToList(); + + int maxDiagnostics = BuildJobOptions.MaxDiagnostics; + if (diagnostics.Count > maxDiagnostics) + { + diagnostics = diagnostics.OrderByDescending(d => d.Severity).Take(maxDiagnostics).ToList(); + } + + int maxWarnings = BuildJobOptions.MaxWarnings; + if (warnings.Count > maxWarnings) + { + string tooManyWarningsWarning = + $"There were {warnings.Count} warnings. Only the first {maxWarnings} are shown."; + warnings = [tooManyWarningsWarning, .. warnings.Take(maxWarnings)]; + } + // Log summary of build data JsonObject buildPreprocessSummary = new() { @@ -122,6 +142,7 @@ CancellationToken cancellationToken IsInferenceFilteredByChapter = stats.IsInferenceFilteredByChapter, IsTrainFilteredByChapter = stats.IsTrainFilteredByChapter, Warnings = warnings, + Diagnostics = diagnostics, EngineSourceLanguageTag = sourceLanguageTag, EngineTargetLanguageTag = targetLanguageTag, }; diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs index 37102092..4babe23a 100644 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs @@ -46,6 +46,7 @@ public async Task RunAsync_BuildWarnings() ( "corpusId1", "src_1", + "0000", "pt-source1", "Original", [ diff --git a/src/Serval/src/Serval.Client/Client.g.cs b/src/Serval/src/Serval.Client/Client.g.cs index 34a76f00..2b96eb61 100644 --- a/src/Serval/src/Serval.Client/Client.g.cs +++ b/src/Serval/src/Serval.Client/Client.g.cs @@ -10942,8 +10942,12 @@ public partial class ExecutionData [Newtonsoft.Json.JsonProperty("warnings", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required] + [System.Obsolete] public System.Collections.Generic.IList Warnings { get; set; } = new System.Collections.ObjectModel.Collection(); + [Newtonsoft.Json.JsonProperty("diagnostics", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IList? Diagnostics { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("engineSourceLanguageTag", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] public string? EngineSourceLanguageTag { get; set; } = default!; @@ -10961,6 +10965,48 @@ public partial class ExecutionData } + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.7.1.0 (NJsonSchema v11.6.1.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class TranslationDiagnostic + { + + [Newtonsoft.Json.JsonProperty("code", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Code { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("category", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Category { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("message", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Message { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("severity", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + [Newtonsoft.Json.JsonConverter(typeof(Newtonsoft.Json.Converters.StringEnumConverter))] + public TranslationDiagnosticSeverity Severity { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("data", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public System.Collections.Generic.IDictionary Data { get; set; } = new System.Collections.Generic.Dictionary(); + + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.7.1.0 (NJsonSchema v11.6.1.0 (Newtonsoft.Json v13.0.0.0))")] + public enum TranslationDiagnosticSeverity + { + + [System.Runtime.Serialization.EnumMember(Value = @"Info")] + Info = 0, + + [System.Runtime.Serialization.EnumMember(Value = @"Warn")] + Warn = 1, + + [System.Runtime.Serialization.EnumMember(Value = @"Error")] + Error = 2, + + } + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.7.1.0 (NJsonSchema v11.6.1.0 (Newtonsoft.Json v13.0.0.0))")] public partial class Phase { @@ -11694,8 +11740,12 @@ public partial class WordAlignmentExecutionData [Newtonsoft.Json.JsonProperty("warnings", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required] + [System.Obsolete] public System.Collections.Generic.IList Warnings { get; set; } = new System.Collections.ObjectModel.Collection(); + [Newtonsoft.Json.JsonProperty("diagnostics", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IList? Diagnostics { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("engineSourceLanguageTag", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] public string? EngineSourceLanguageTag { get; set; } = default!; @@ -11704,6 +11754,48 @@ public partial class WordAlignmentExecutionData } + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.7.1.0 (NJsonSchema v11.6.1.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class WordAlignmentDiagnostic + { + + [Newtonsoft.Json.JsonProperty("code", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Code { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("category", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Category { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("message", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Message { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("severity", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + [Newtonsoft.Json.JsonConverter(typeof(Newtonsoft.Json.Converters.StringEnumConverter))] + public WordAlignmentDiagnosticSeverity Severity { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("data", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public System.Collections.Generic.IDictionary Data { get; set; } = new System.Collections.Generic.Dictionary(); + + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.7.1.0 (NJsonSchema v11.6.1.0 (Newtonsoft.Json v13.0.0.0))")] + public enum WordAlignmentDiagnosticSeverity + { + + [System.Runtime.Serialization.EnumMember(Value = @"Info")] + Info = 0, + + [System.Runtime.Serialization.EnumMember(Value = @"Warn")] + Warn = 1, + + [System.Runtime.Serialization.EnumMember(Value = @"Error")] + Error = 2, + + } + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.7.1.0 (NJsonSchema v11.6.1.0 (Newtonsoft.Json v13.0.0.0))")] public partial class WordAlignmentEngine { diff --git a/src/Serval/src/Serval.Shared.Contracts/IParallelCorpusService.cs b/src/Serval/src/Serval.Shared.Contracts/IParallelCorpusService.cs index 6f1071fe..59c94bf1 100644 --- a/src/Serval/src/Serval.Shared.Contracts/IParallelCorpusService.cs +++ b/src/Serval/src/Serval.Shared.Contracts/IParallelCorpusService.cs @@ -8,6 +8,7 @@ public interface IParallelCorpusService string ParallelCorpusId, string MonolingualCorpusId, string ProjectName, + string ProjectGuid, string VersificationName, IReadOnlyList Diagnostics )> AnalyzeUsfmVersification(IEnumerable parallelCorpora); diff --git a/src/Serval/src/Serval.Shared.Contracts/MissingParentProjectErrorContract.cs b/src/Serval/src/Serval.Shared.Contracts/MissingParentProjectErrorContract.cs index bb382db1..52d843ef 100644 --- a/src/Serval/src/Serval.Shared.Contracts/MissingParentProjectErrorContract.cs +++ b/src/Serval/src/Serval.Shared.Contracts/MissingParentProjectErrorContract.cs @@ -3,5 +3,7 @@ namespace Serval.Shared.Contracts; public record MissingParentProjectErrorContract { public required string ProjectName { get; init; } + public required string ProjectGuid { get; init; } public required string ParentProjectName { get; init; } + public required string ParentProjectGuid { get; init; } } diff --git a/src/Serval/src/Serval.Shared/Services/ParallelCorpusService.cs b/src/Serval/src/Serval.Shared/Services/ParallelCorpusService.cs index d0d4f131..e782473f 100644 --- a/src/Serval/src/Serval.Shared/Services/ParallelCorpusService.cs +++ b/src/Serval/src/Serval.Shared/Services/ParallelCorpusService.cs @@ -11,6 +11,7 @@ public class ParallelCorpusService : IParallelCorpusService string ParallelCorpusId, string MonolingualCorpusId, string ProjectName, + string ProjectGuid, string VersificationName, IReadOnlyList Diagnostics )> AnalyzeUsfmVersification(IEnumerable parallelCorpora) @@ -20,6 +21,7 @@ IReadOnlyList Diagnostics string ParallelCorpusId, string MonolingualCorpusId, string ProjectName, + string ProjectGuid, string VersificationName, IReadOnlyList Diagnostics )> diagnosticsPerCorpus = []; @@ -44,6 +46,7 @@ IReadOnlyList Diagnostics parallelCorpus.Id, monolingualCorpus.Id, analysis.ProjectSettings.Name, + analysis.ProjectSettings.Guid, analysis.ProjectSettings.Versification.Name, [ .. analysis.Diagnostics.Select(d => new UsfmVersificationDiagnosticContract @@ -167,7 +170,13 @@ MissingParentProjectErrorContract Error ( parallelCorpus.Id, monolingualCorpus.Id, - new() { ProjectName = settings.Name, ParentProjectName = settings.ParentName } + new() + { + ProjectName = settings.Name, + ProjectGuid = settings.Guid, + ParentProjectName = settings.ParentName, + ParentProjectGuid = settings.ParentGuid, + } ) ); } diff --git a/src/Serval/src/Serval.Translation.Contracts/DiagnosticContract.cs b/src/Serval/src/Serval.Translation.Contracts/DiagnosticContract.cs new file mode 100644 index 00000000..f3157ef1 --- /dev/null +++ b/src/Serval/src/Serval.Translation.Contracts/DiagnosticContract.cs @@ -0,0 +1,17 @@ +namespace Serval.Translation.Contracts; + +public record DiagnosticContract +{ + public required string Code { get; init; } + public required string Category { get; init; } + public required string Message { get; init; } + public required DiagnosticSeverity Severity { get; init; } + public required Dictionary Data { get; init; } +} + +public enum DiagnosticSeverity +{ + Info, + Warn, + Error, +} diff --git a/src/Serval/src/Serval.Translation.Contracts/ExecutionDataContract.cs b/src/Serval/src/Serval.Translation.Contracts/ExecutionDataContract.cs index d7fd7422..e0b67f58 100644 --- a/src/Serval/src/Serval.Translation.Contracts/ExecutionDataContract.cs +++ b/src/Serval/src/Serval.Translation.Contracts/ExecutionDataContract.cs @@ -9,6 +9,7 @@ public record ExecutionDataContract public IReadOnlyDictionary>? TrainVerseCount { get; init; } public IReadOnlyDictionary>? PretranslateVerseCount { get; init; } public IReadOnlyList? Warnings { get; init; } + public IReadOnlyList? Diagnostics { get; init; } public string? EngineSourceLanguageTag { get; init; } public string? EngineTargetLanguageTag { get; init; } public string? ResolvedSourceLanguage { get; init; } diff --git a/src/Serval/src/Serval.Translation/Dtos/ExecutionDataDto.cs b/src/Serval/src/Serval.Translation/Dtos/ExecutionDataDto.cs index ce427261..c9ce06b5 100644 --- a/src/Serval/src/Serval.Translation/Dtos/ExecutionDataDto.cs +++ b/src/Serval/src/Serval.Translation/Dtos/ExecutionDataDto.cs @@ -8,7 +8,10 @@ public record ExecutionDataDto public bool? IsPretranslateFilteredByChapter { get; init; } public IReadOnlyDictionary>? TrainVerseCount { get; init; } public IReadOnlyDictionary>? PretranslateVerseCount { get; init; } + + [Obsolete] public IReadOnlyList Warnings { get; init; } = []; + public IReadOnlyList? Diagnostics { get; init; } public string? EngineSourceLanguageTag { get; init; } public string? EngineTargetLanguageTag { get; init; } public string? ResolvedSourceLanguage { get; init; } diff --git a/src/Serval/src/Serval.Translation/Dtos/TranslationDiagnosticDto.cs b/src/Serval/src/Serval.Translation/Dtos/TranslationDiagnosticDto.cs new file mode 100644 index 00000000..21111c2d --- /dev/null +++ b/src/Serval/src/Serval.Translation/Dtos/TranslationDiagnosticDto.cs @@ -0,0 +1,17 @@ +namespace Serval.Translation.Dtos; + +public record TranslationDiagnosticDto +{ + public required string Code { get; init; } + public required string Category { get; init; } + public required string Message { get; init; } + public required TranslationDiagnosticSeverity Severity { get; init; } + public required Dictionary Data { get; init; } +} + +public enum TranslationDiagnosticSeverity +{ + Info, + Warn, + Error, +} diff --git a/src/Serval/src/Serval.Translation/Models/Diagnostic.cs b/src/Serval/src/Serval.Translation/Models/Diagnostic.cs new file mode 100644 index 00000000..6572a7c0 --- /dev/null +++ b/src/Serval/src/Serval.Translation/Models/Diagnostic.cs @@ -0,0 +1,17 @@ +namespace Serval.Translation.Models; + +public record Diagnostic +{ + public required string Code { get; init; } + public required string Category { get; init; } + public required string Message { get; init; } + public required DiagnosticSeverity Severity { get; init; } + public required Dictionary Data { get; init; } +} + +public enum DiagnosticSeverity +{ + Info, + Warn, + Error, +} diff --git a/src/Serval/src/Serval.Translation/Models/ExecutionData.cs b/src/Serval/src/Serval.Translation/Models/ExecutionData.cs index feb9a2af..a7fe4c19 100644 --- a/src/Serval/src/Serval.Translation/Models/ExecutionData.cs +++ b/src/Serval/src/Serval.Translation/Models/ExecutionData.cs @@ -9,6 +9,7 @@ public record ExecutionData public IReadOnlyDictionary>? TrainVerseCount { get; init; } public IReadOnlyDictionary>? PretranslateVerseCount { get; init; } public IReadOnlyList? Warnings { get; init; } + public IReadOnlyList? Diagnostics { get; init; } public string? EngineSourceLanguageTag { get; init; } public string? EngineTargetLanguageTag { get; init; } public string? ResolvedSourceLanguage { get; init; } diff --git a/src/Serval/src/Serval.Translation/Services/DtoMapper.cs b/src/Serval/src/Serval.Translation/Services/DtoMapper.cs index ad09d355..721a08fe 100644 --- a/src/Serval/src/Serval.Translation/Services/DtoMapper.cs +++ b/src/Serval/src/Serval.Translation/Services/DtoMapper.cs @@ -236,12 +236,25 @@ private static ExecutionDataDto Map(ExecutionData source) => IsPretranslateFilteredByChapter = source.IsPretranslateFilteredByChapter ?? false, IsTrainFilteredByChapter = source.IsTrainFilteredByChapter ?? false, Warnings = source.Warnings ?? [], + Diagnostics = source.Diagnostics?.Select(Map).ToList() ?? [], EngineSourceLanguageTag = source.EngineSourceLanguageTag, EngineTargetLanguageTag = source.EngineTargetLanguageTag, ResolvedSourceLanguage = source.ResolvedSourceLanguage, ResolvedTargetLanguage = source.ResolvedTargetLanguage, AveragePretranslationConfidence = source.AveragePretranslationConfidence, }; + + private static TranslationDiagnosticDto Map(Diagnostic source) + { + return new TranslationDiagnosticDto + { + Code = source.Code, + Category = source.Category, + Message = source.Message, + Severity = (Dtos.TranslationDiagnosticSeverity)source.Severity, + Data = source.Data, + }; + } } #pragma warning restore CS0612 // Type or member is obsolete diff --git a/src/Serval/src/Serval.Translation/Services/PlatformService.cs b/src/Serval/src/Serval.Translation/Services/PlatformService.cs index 88b6e694..4475cc98 100644 --- a/src/Serval/src/Serval.Translation/Services/PlatformService.cs +++ b/src/Serval/src/Serval.Translation/Services/PlatformService.cs @@ -1,3 +1,5 @@ +using SIL.Machine.Corpora; + namespace Serval.Translation.Services; public class PlatformService( @@ -10,6 +12,9 @@ IEventRouter eventRouter { private const int PretranslationInsertBatchSize = 128; + private const double BadBookConfidenceThreshold = 0.35; + private const string ModelName = "NLLB"; //TODO where can we store this since it's used in multiple classes? In a constants.cs? On the build object itself? + private readonly IRepository _builds = builds; private readonly IRepository _engines = engines; private readonly IRepository _pretranslations = pretranslations; @@ -316,6 +321,16 @@ await _builds.UpdateAsync( IsTrainFilteredByChapter = executionData.IsTrainFilteredByChapter, IsPretranslateFilteredByChapter = executionData.IsPretranslateFilteredByChapter, Warnings = executionData.Warnings?.ToList() ?? [], + Diagnostics = executionData + .Diagnostics?.Select(d => new Diagnostic + { + Code = d.Code, + Category = d.Category, + Message = d.Message, + Severity = (Models.DiagnosticSeverity)d.Severity, + Data = d.Data, + }) + .ToList(), EngineSourceLanguageTag = executionData.EngineSourceLanguageTag, EngineTargetLanguageTag = executionData.EngineTargetLanguageTag, ResolvedSourceLanguage = executionData.ResolvedSourceLanguage, @@ -383,6 +398,8 @@ public async Task InsertPretranslationsAsync( double logConfidenceTotal = 0.0; int confidenceCount = 0; int numPretranslations = 0; + Dictionary logConfidenceTotalPerBook = []; + Dictionary confidenceCountPerBook = []; await foreach (PretranslationContract item in pretranslations.WithCancellation(cancellationToken)) { batch.Add( @@ -399,7 +416,7 @@ public async Task InsertPretranslationsAsync( SourceTokens = item.SourceTokens, TranslationTokens = item.TranslationTokens, Alignment = item - .Alignment?.Select(a => new AlignedWordPair + .Alignment?.Select(a => new Shared.Models.AlignedWordPair { SourceIndex = a.SourceIndex, TargetIndex = a.TargetIndex, @@ -412,8 +429,25 @@ public async Task InsertPretranslationsAsync( double? confidence = item.Confidence; if (confidence != null && confidence > 0.0) { - logConfidenceTotal += Math.Log((double)confidence); + double logConfidence = Math.Log((double)confidence); + logConfidenceTotal += logConfidence; confidenceCount++; + + if ( + item.TargetRefs.Count > 0 + && ScriptureRef.TryParse(item.TargetRefs[0], out ScriptureRef scriptureRef) + ) + { + string bookId = scriptureRef.Book; + + if (!logConfidenceTotalPerBook.ContainsKey(bookId)) + logConfidenceTotalPerBook[bookId] = 0.0; + logConfidenceTotalPerBook[bookId] += logConfidence; + + if (!confidenceCountPerBook.ContainsKey(bookId)) + confidenceCountPerBook[bookId] = 0; + confidenceCountPerBook[bookId]++; + } } numPretranslations++; @@ -426,16 +460,68 @@ public async Task InsertPretranslationsAsync( if (batch.Count > 0) await _pretranslations.InsertAllAsync(batch, CancellationToken.None); + List badBookConfidences = logConfidenceTotalPerBook + .Select(kvp => + { + string bookId = kvp.Key; + double logTotal = kvp.Value; + int count = confidenceCountPerBook[bookId]; + double averageConfidence = count > 0 ? Math.Exp(logTotal / count) : 0.0; + return (bookId, averageConfidence); + }) + .Where(b => b.averageConfidence < BadBookConfidenceThreshold) + .Select(b => new Diagnostic + { + Code = "MODEL-0003", + Category = "MODEL", + Severity = Models.DiagnosticSeverity.Warn, + Message = + $"The average pretranslation model confidence {b.averageConfidence:f4} in book {b.bookId} is unusually low for the base model {ModelName}", + Data = new Dictionary + { + { "bookId", b.bookId }, + { "averagePretranslationConfidence", b.averageConfidence }, + { "modelName", ModelName }, + }, + }) + .ToList(); + + Build? currentBuild = null; + if (badBookConfidences.Count > 0) + { + currentBuild = await _builds.GetAsync(b => b.Id == buildId, cancellationToken); + + await _builds.UpdateAsync( + b => b.Id == buildId, + u => + u.Set( + b => b.ExecutionData.Diagnostics, + currentBuild?.ExecutionData.Diagnostics is null + ? [.. badBookConfidences] + : [.. currentBuild.ExecutionData.Diagnostics, .. badBookConfidences] + ), + cancellationToken: cancellationToken + ); + } + await _builds.UpdateAsync( b => b.Id == buildId, u => + { u.Set( b => b.ExecutionData.AveragePretranslationConfidence, // Calculate the geometric mean of the pretranslation confidences confidenceCount > 0 ? Math.Exp(logConfidenceTotal / confidenceCount) : 0.0 - ), + ); + u.Set( + b => b.ExecutionData.Diagnostics, + currentBuild?.ExecutionData.Diagnostics is null + ? [.. badBookConfidences] + : [.. currentBuild.ExecutionData.Diagnostics, .. badBookConfidences] + ); + }, cancellationToken: cancellationToken ); } diff --git a/src/Serval/src/Serval.WordAlignment.Contracts/DiagnosticContract.cs b/src/Serval/src/Serval.WordAlignment.Contracts/DiagnosticContract.cs new file mode 100644 index 00000000..1c6a751a --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment.Contracts/DiagnosticContract.cs @@ -0,0 +1,17 @@ +namespace Serval.WordAlignment.Contracts; + +public record DiagnosticContract +{ + public required string Code { get; init; } + public required string Category { get; init; } + public required string Message { get; init; } + public required DiagnosticSeverity Severity { get; init; } + public required Dictionary Data { get; init; } +} + +public enum DiagnosticSeverity +{ + Info, + Warn, + Error, +} diff --git a/src/Serval/src/Serval.WordAlignment.Contracts/ExecutionDataContract.cs b/src/Serval/src/Serval.WordAlignment.Contracts/ExecutionDataContract.cs index ac0d1dc0..fd89367e 100644 --- a/src/Serval/src/Serval.WordAlignment.Contracts/ExecutionDataContract.cs +++ b/src/Serval/src/Serval.WordAlignment.Contracts/ExecutionDataContract.cs @@ -9,6 +9,7 @@ public record ExecutionDataContract public IReadOnlyDictionary>? TrainVerseCount { get; init; } public IReadOnlyDictionary>? WordAlignVerseCount { get; init; } public IReadOnlyList? Warnings { get; init; } + public IReadOnlyList? Diagnostics { get; init; } public string? EngineSourceLanguageTag { get; init; } public string? EngineTargetLanguageTag { get; init; } } diff --git a/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentDiagnosticDto.cs b/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentDiagnosticDto.cs new file mode 100644 index 00000000..7b602d51 --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentDiagnosticDto.cs @@ -0,0 +1,17 @@ +namespace Serval.WordAlignment.Dtos; + +public record WordAlignmentDiagnosticDto +{ + public required string Code { get; init; } + public required string Category { get; init; } + public required string Message { get; init; } + public required WordAlignmentDiagnosticSeverity Severity { get; init; } + public required Dictionary Data { get; init; } +} + +public enum WordAlignmentDiagnosticSeverity +{ + Info, + Warn, + Error, +} diff --git a/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentExecutionDataDto.cs b/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentExecutionDataDto.cs index 9ed2a182..75670eb0 100644 --- a/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentExecutionDataDto.cs +++ b/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentExecutionDataDto.cs @@ -8,7 +8,10 @@ public record WordAlignmentExecutionDataDto public bool? IsWordAlignFilteredByChapter { get; init; } public IReadOnlyDictionary>? TrainVerseCount { get; init; } public IReadOnlyDictionary>? WordAlignVerseCount { get; init; } + + [Obsolete] public IReadOnlyList Warnings { get; init; } = []; + public IReadOnlyList? Diagnostics { get; init; } public string? EngineSourceLanguageTag { get; init; } public string? EngineTargetLanguageTag { get; init; } } diff --git a/src/Serval/src/Serval.WordAlignment/Models/Diagnostic.cs b/src/Serval/src/Serval.WordAlignment/Models/Diagnostic.cs new file mode 100644 index 00000000..f8657779 --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Models/Diagnostic.cs @@ -0,0 +1,17 @@ +namespace Serval.WordAlignment.Models; + +public record Diagnostic +{ + public required string Code { get; init; } + public required string Category { get; init; } + public required string Message { get; init; } + public required DiagnosticSeverity Severity { get; init; } + public required Dictionary Data { get; init; } +} + +public enum DiagnosticSeverity +{ + Info, + Warn, + Error, +} diff --git a/src/Serval/src/Serval.WordAlignment/Models/ExecutionData.cs b/src/Serval/src/Serval.WordAlignment/Models/ExecutionData.cs index 824e80b2..3bd6cc28 100644 --- a/src/Serval/src/Serval.WordAlignment/Models/ExecutionData.cs +++ b/src/Serval/src/Serval.WordAlignment/Models/ExecutionData.cs @@ -9,6 +9,7 @@ public record ExecutionData public IReadOnlyDictionary>? TrainVerseCount { get; init; } public IReadOnlyDictionary>? WordAlignVerseCount { get; init; } public IReadOnlyList? Warnings { get; init; } + public IReadOnlyList? Diagnostics { get; init; } public string? EngineSourceLanguageTag { get; init; } public string? EngineTargetLanguageTag { get; init; } } diff --git a/src/Serval/src/Serval.WordAlignment/Services/DtoMapper.cs b/src/Serval/src/Serval.WordAlignment/Services/DtoMapper.cs index 1b368ac2..60a38a3b 100644 --- a/src/Serval/src/Serval.WordAlignment/Services/DtoMapper.cs +++ b/src/Serval/src/Serval.WordAlignment/Services/DtoMapper.cs @@ -195,8 +195,21 @@ private static WordAlignmentExecutionDataDto Map(ExecutionData source) IsTrainFilteredByChapter = source.IsTrainFilteredByChapter, IsWordAlignFilteredByChapter = source.IsWordAlignFilteredByChapter, Warnings = source.Warnings ?? [], + Diagnostics = source.Diagnostics?.Select(Map).ToList() ?? [], EngineSourceLanguageTag = source.EngineSourceLanguageTag, EngineTargetLanguageTag = source.EngineTargetLanguageTag, }; } + + private static WordAlignmentDiagnosticDto Map(Diagnostic source) + { + return new WordAlignmentDiagnosticDto + { + Code = source.Code, + Category = source.Category, + Message = source.Message, + Severity = (Dtos.WordAlignmentDiagnosticSeverity)source.Severity, + Data = source.Data, + }; + } } diff --git a/src/Serval/test/Serval.Shared.Tests/Services/ParallelCorpusServiceTests.cs b/src/Serval/test/Serval.Shared.Tests/Services/ParallelCorpusServiceTests.cs index 98780e33..e9e36494 100644 --- a/src/Serval/test/Serval.Shared.Tests/Services/ParallelCorpusServiceTests.cs +++ b/src/Serval/test/Serval.Shared.Tests/Services/ParallelCorpusServiceTests.cs @@ -218,6 +218,7 @@ public void AnalyzeUsfmVersification() string ParallelCorpusId, string MonolingualCorpusId, string ProjectName, + string ProjectGuid, string VersificationName, IReadOnlyList Diagnostics )> analysis = env.Processor.AnalyzeUsfmVersification([parallelCorpus]);