From 2c3302f685c6a90428a10e86dbf6a643b7155811 Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Tue, 28 Jul 2026 11:43:44 -0400 Subject: [PATCH 1/9] Add all diagnostics from document --- .../Models/BuildDiagnostic.cs | 17 ++ .../Models/BuildExecutionData.cs | 1 + .../Services/NmtPreprocessBuildJob.cs | 152 ++++++++++- .../Services/PreprocessBuildJob.cs | 244 +++++++++++++++++- .../Services/PreprocessBuildJobTests.cs | 1 + .../IParallelCorpusService.cs | 1 + .../MissingParentProjectErrorContract.cs | 2 + .../Services/ParallelCorpusService.cs | 11 +- .../Services/ParallelCorpusServiceTests.cs | 1 + 9 files changed, 417 insertions(+), 13 deletions(-) create mode 100644 src/Machine/src/Serval.Machine.Shared/Models/BuildDiagnostic.cs diff --git a/src/Machine/src/Serval.Machine.Shared/Models/BuildDiagnostic.cs b/src/Machine/src/Serval.Machine.Shared/Models/BuildDiagnostic.cs new file mode 100644 index 00000000..c561810b --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Models/BuildDiagnostic.cs @@ -0,0 +1,17 @@ +namespace Serval.Machine.Shared.Models; + +public class BuildDiagnostic +{ + public required string Code { get; init; } + public required string Category { get; init; } + public required string Message { get; init; } + public required BuildDiagnosticSeverity Severity { get; init; } + public required Dictionary Data { get; init; } +} + +public enum BuildDiagnosticSeverity +{ + Info, + Warn, + Error, +} diff --git a/src/Machine/src/Serval.Machine.Shared/Models/BuildExecutionData.cs b/src/Machine/src/Serval.Machine.Shared/Models/BuildExecutionData.cs index 004ad374..3faddde4 100644 --- a/src/Machine/src/Serval.Machine.Shared/Models/BuildExecutionData.cs +++ b/src/Machine/src/Serval.Machine.Shared/Models/BuildExecutionData.cs @@ -9,6 +9,7 @@ public record BuildExecutionData public IReadOnlyDictionary>? TrainVerseCount { get; init; } public IReadOnlyDictionary>? InferenceVerseCount { get; init; } public IReadOnlyList? Warnings { get; init; } + public IReadOnlyList? Diagnostics { get; init; } public string? EngineSourceLanguageTag { get; init; } public string? EngineTargetLanguageTag { get; init; } public string? ResolvedSourceLanguage { get; init; } diff --git a/src/Machine/src/Serval.Machine.Shared/Services/NmtPreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/NmtPreprocessBuildJob.cs index dfd38985..ac70f612 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/NmtPreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/NmtPreprocessBuildJob.cs @@ -23,6 +23,8 @@ IOptionsMonitor options ) { private readonly ILanguageTagService _languageTagService = languageTagService; + private const string ModelName = "NLLB"; + private const string MinimumTrainCount = "600"; //TODO move to options? private bool ResolveLanguageCode(string languageCode, out string resolvedCode) { @@ -55,6 +57,7 @@ protected override async Task UpdateBuildExecutionData( PreprocessStats stats, string sourceLanguageTag, string targetLanguageTag, + bool isNonPersistedTranslationEngine, IReadOnlyList parallelCorpora, CancellationToken cancellationToken ) @@ -62,13 +65,6 @@ CancellationToken cancellationToken bool sourceLanguageHasNativeSupport = ResolveLanguageCode(sourceLanguageTag, out string resolvedSourceLanguage); bool targetLanguageHasNativeSupport = ResolveLanguageCode(targetLanguageTag, out string resolvedTargetLanguage); - if (stats.TrainCount == 0 && (!sourceLanguageHasNativeSupport || !targetLanguageHasNativeSupport)) - { - throw new InvalidOperationException( - $"At least one language code in build {buildId} is unknown to the base model, and the data specified for training was empty. Build canceled." - ); - } - IReadOnlyList warnings = GetWarnings( stats.TrainCount, stats.InferenceCount, @@ -85,6 +81,17 @@ CancellationToken cancellationToken warnings = [tooManyWarningsWarning, .. warnings.Take(maxWarnings)]; } + IReadOnlyList diagnostics = GetDiagnostics( + stats.TrainCount, + stats.InferenceCount, + sourceLanguageTag, + targetLanguageTag, + sourceLanguageHasNativeSupport, + targetLanguageHasNativeSupport, + isNonPersistedTranslationEngine, + parallelCorpora + ); + // Log summary of build data JsonObject buildPreprocessSummary = new() { @@ -109,12 +116,20 @@ CancellationToken cancellationToken IsTrainFilteredByChapter = stats.IsTrainFilteredByChapter, IsInferenceFilteredByChapter = stats.IsInferenceFilteredByChapter, Warnings = warnings, + Diagnostics = diagnostics, EngineSourceLanguageTag = sourceLanguageTag, EngineTargetLanguageTag = targetLanguageTag, ResolvedSourceLanguage = resolvedSourceLanguage, ResolvedTargetLanguage = resolvedTargetLanguage, }; await PlatformService.UpdateBuildExecutionDataAsync(engineId, buildId, executionData, cancellationToken); + + if (stats.TrainCount == 0 && (!sourceLanguageHasNativeSupport || !targetLanguageHasNativeSupport)) + { + throw new InvalidOperationException( + $"At least one language code in build {buildId} is unknown to the base model {ModelName}, and no data was specified for training. Build canceled." + ); + } } protected override IReadOnlyList GetWarnings( @@ -141,14 +156,133 @@ .. base.GetWarnings(trainCount, inferenceCount, sourceLanguageTag, targetLanguag == Flores200Support.None ) { - warnings.Add($"The script for the source language '{resolvedCode}' is not in Flores-200"); + warnings.Add( + $"The script for the source language '{resolvedCode}' is not known to the base model {ModelName}" + ); } if (_languageTagService.ConvertToFlores200Code(targetLanguageTag, out resolvedCode) == Flores200Support.None) { - warnings.Add($"The script for the target language '{resolvedCode}' is not in Flores-200"); + warnings.Add( + $"The script for the target language '{resolvedCode}' is not known to the base model {ModelName}" + ); } return warnings; } + + protected override IReadOnlyList GetDiagnostics( + int trainCount, + int inferenceCount, + string sourceLanguageTag, + string targetLanguageTag, + bool sourceLanguageHasNativeSupport, + bool targetLanguageHasNativeSupport, + bool isNonPersistedTranslationEngine, + IReadOnlyList parallelCorpora + ) + { + List diagnostics = + [ + .. base.GetDiagnostics( + trainCount, + inferenceCount, + sourceLanguageTag, + targetLanguageTag, + sourceLanguageHasNativeSupport, + targetLanguageHasNativeSupport, + isNonPersistedTranslationEngine, + parallelCorpora + ), + ]; + + // Has at least a Gospel of Mark amount of data and not the special case of no data which will be caught elsewhere + if (trainCount < 600 && trainCount != 0) + { + diagnostics.Add( + new BuildDiagnostic + { + Code = "CONFIG-003", + Category = "CONFIG", + Severity = BuildDiagnosticSeverity.Warn, + Message = + $"Only {trainCount} segments were selected for training. Training on fewer than {MinimumTrainCount} is not recommended.", + Data = new Dictionary + { + { "trainCount", trainCount }, + { "minimumTrainCount", MinimumTrainCount }, + }, + } + ); + } + + if ( + _languageTagService.ConvertToFlores200Code(sourceLanguageTag, out string resolvedCode) + == Flores200Support.None + ) + { + diagnostics.Add( + new BuildDiagnostic + { + Code = "MODEL-001", + Category = "MODEL", + Severity = BuildDiagnosticSeverity.Warn, + Message = + $"The script for the source language '{resolvedCode}' is not known to the base model {ModelName}", + Data = new Dictionary + { + { "resolvedCode", sourceLanguageTag }, + { "modelName", ModelName }, + }, + } + ); + } + + if (_languageTagService.ConvertToFlores200Code(targetLanguageTag, out resolvedCode) == Flores200Support.None) + { + diagnostics.Add( + new BuildDiagnostic + { + Code = "MODEL-002", + Category = "MODEL", + Severity = BuildDiagnosticSeverity.Warn, + Message = + $"The script for the target language '{resolvedCode}' is not known to the base model {ModelName}", + Data = new Dictionary + { + { "resolvedCode", targetLanguageTag }, + { "modelName", ModelName }, + }, + } + ); + } + + if (trainCount == 0 && (!sourceLanguageHasNativeSupport || !targetLanguageHasNativeSupport)) + { + List unknownLanguageCodes = new[] + { + !sourceLanguageHasNativeSupport ? sourceLanguageTag : "", + !targetLanguageHasNativeSupport ? targetLanguageTag : "", + } + .Where(s => !string.IsNullOrEmpty(s)) + .ToList(); + diagnostics.Add( + new BuildDiagnostic + { + Code = "MODEL-004", + Category = "MODEL", + Severity = BuildDiagnosticSeverity.Error, + Message = + $"The following language codes are unknown to the base model {ModelName}: {string.Join(", ", unknownLanguageCodes)}; and no language data was selected for training.", + Data = new Dictionary + { + { "modelName", ModelName }, + { "unknownLanguageCodes", unknownLanguageCodes }, + }, + } + ); + } + + return diagnostics; + } } diff --git a/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs index 22607aba..af26582a 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs @@ -46,6 +46,7 @@ CancellationToken cancellationToken throw new OperationCanceledException($"Engine {engineId} does not exist. Build canceled."); PreprocessStats stats = await WriteDataFilesAsync(engineId, buildId, data, buildOptions, cancellationToken); + bool isNonPersistedTranslationEngine = engine is TranslationEngine { IsModelPersisted: false }; await UpdateBuildExecutionData( engineId, @@ -53,13 +54,14 @@ await UpdateBuildExecutionData( stats, engine.SourceLanguage, engine.TargetLanguage, + isNonPersistedTranslationEngine, data, cancellationToken ); await UpdateTargetQuoteConventionAsync(engineId, buildId, data, cancellationToken); - if (stats.InferenceCount == 0 && engine is TranslationEngine { IsModelPersisted: false }) + if (stats.InferenceCount == 0 && isNonPersistedTranslationEngine) { throw new InvalidOperationException( $"There was no data specified for inferencing in build {buildId}. Build canceled." @@ -87,6 +89,7 @@ protected abstract Task UpdateBuildExecutionData( PreprocessStats stats, string sourceLanguageTag, string targetLanguageTag, + bool isNonPersistedTranslationEngine, IReadOnlyList parallelCorpora, CancellationToken cancellationToken ); @@ -137,6 +140,7 @@ IReadOnlyList parallelCorpora string parallelCorpusId, string monolingualCorpusId, string projectName, + string projectGuid, string versificationName, IReadOnlyList diagnostics ) in ParallelCorpusService.AnalyzeUsfmVersification(parallelCorpora) @@ -146,7 +150,7 @@ IReadOnlyList diagnostics foreach (UsfmVersificationDiagnosticContract diagnostic in diagnostics) { string diagnosticDetails = - $"in project {projectName} at {diagnostic.Filename} " + $"in project {projectName} {projectGuid} at {diagnostic.Filename} " + (diagnostic.LineNumbers.Count == 1 ? "line " : "lines ") + $"{string.Join(", ", diagnostic.LineNumbers)}, " + (diagnostic.NumAffectedVerses == 1 ? "verse " : "verses ") @@ -182,7 +186,7 @@ MissingParentProjectErrorContract error ) { warnings.Add( - $"Unable to locate parent project {error.ParentProjectName} of daughter project {error.ProjectName} (parallel corpus {parallelCorpusId}, monolingual corpus {monolingualCorpusId})" + $"Unable to locate parent project {error.ParentProjectName} {error.ParentProjectGuid} of daughter project {error.ProjectName} {error.ProjectGuid} (parallel corpus {parallelCorpusId}, monolingual corpus {monolingualCorpusId})" ); } @@ -196,6 +200,240 @@ MissingParentProjectErrorContract error return warnings; } + protected virtual IReadOnlyList GetDiagnostics( + int trainCount, + int inferenceCount, + string sourceLanguageTag, + string targetLanguageTag, + bool sourceLanguageHasNativeSupport, + bool targetLanguageHasNativeSupport, + bool isNonPersistedTranslationEngine, + IReadOnlyList parallelCorpora + ) + { + List diagnostics = []; + Dictionary projectVersifications = []; + + foreach ( + ( + string parallelCorpusId, + string monolingualCorpusId, + string projectName, + string projectGuid, + string versificationName, + IReadOnlyList usfmDiagnostics + ) in ParallelCorpusService.AnalyzeUsfmVersification(parallelCorpora) + ) + { + projectVersifications[projectGuid] = versificationName; + foreach (UsfmVersificationDiagnosticContract usfmDiagnostic in usfmDiagnostics) + { + string diagnosticDetails = + $"in project {projectName} {projectGuid} at {usfmDiagnostic.Filename} " + + (usfmDiagnostic.LineNumbers.Count == 1 ? "line " : "lines ") + + $"{string.Join(", ", usfmDiagnostic.LineNumbers)}, " + + (usfmDiagnostic.NumAffectedVerses == 1 ? "verse " : "verses ") + + $"{string.Join(", ", usfmDiagnostic.References)} " + + $"(parallel corpus {parallelCorpusId}, monolingual corpus {monolingualCorpusId})."; + diagnostics.Add( + usfmDiagnostic.Type switch + { + Serval.Shared.Contracts.UsfmVersificationDiagnosticType.InvalidChapter => new BuildDiagnostic + { + Code = "USFM-001", + Category = "USFM", + Severity = BuildDiagnosticSeverity.Warn, + Message = $"Invalid chapter number {diagnosticDetails}", + Data = new Dictionary + { + { "projectName", projectName }, + { "projectGuid", projectGuid }, + { "usfmFilename", usfmDiagnostic.Filename }, + { + "lineNumber", + usfmDiagnostic.LineNumbers.Count > 0 ? usfmDiagnostic.LineNumbers[0] : -1 + }, + { + "verseReference", + usfmDiagnostic.References.Count > 0 ? usfmDiagnostic.References[0] : "" + }, + { "parallelCorpusId", parallelCorpusId }, + { "monolingualCorpusId", monolingualCorpusId }, + }, + }, + Serval.Shared.Contracts.UsfmVersificationDiagnosticType.InvalidVerse => new BuildDiagnostic + { + Code = "USFM-002", + Category = "USFM", + Severity = BuildDiagnosticSeverity.Warn, + Message = $"Invalid verse number {diagnosticDetails}", + Data = new Dictionary + { + { "projectName", projectName }, + { "projectGuid", projectGuid }, + { "usfmFilename", usfmDiagnostic.Filename }, + { + "lineNumber", + usfmDiagnostic.LineNumbers.Count > 0 ? usfmDiagnostic.LineNumbers[0] : -1 + }, + { + "verseReference", + usfmDiagnostic.References.Count > 0 ? usfmDiagnostic.References[0] : "" + }, + { "parallelCorpusId", parallelCorpusId }, + { "monolingualCorpusId", monolingualCorpusId }, + }, + }, + Serval.Shared.Contracts.UsfmVersificationDiagnosticType.Extra => new BuildDiagnostic + { + Code = "USFM-003", + Category = "USFM", + Severity = BuildDiagnosticSeverity.Info, + Message = $"{usfmDiagnostic.NumAffectedVerses} extra verses {diagnosticDetails}", + Data = new Dictionary + { + { "numberOfVerses", usfmDiagnostic.NumAffectedVerses }, + { "projectName", projectName }, + { "projectGuid", projectGuid }, + { "usfmFilename", usfmDiagnostic.Filename }, + { "lineNumbers", usfmDiagnostic.LineNumbers }, + { "verseReferences", usfmDiagnostic.References }, + { "parallelCorpusId", parallelCorpusId }, + { "monolingualCorpusId", monolingualCorpusId }, + }, + }, + Serval.Shared.Contracts.UsfmVersificationDiagnosticType.Missing => new BuildDiagnostic + { + Code = "USFM-004", + Category = "USFM", + Severity = BuildDiagnosticSeverity.Warn, + Message = $"Missing {usfmDiagnostic.NumAffectedVerses} verses {diagnosticDetails}", + Data = new Dictionary + { + { "numberOfVerses", usfmDiagnostic.NumAffectedVerses }, + { "projectName", projectName }, + { "projectGuid", projectGuid }, + { "usfmFilename", usfmDiagnostic.Filename }, + { "lineNumbers", usfmDiagnostic.LineNumbers }, + { "verseReferences", usfmDiagnostic.References }, + { "parallelCorpusId", parallelCorpusId }, + { "monolingualCorpusId", monolingualCorpusId }, + }, + }, + Serval.Shared.Contracts.UsfmVersificationDiagnosticType.IncorrectVerseSegment => + new BuildDiagnostic + { + Code = "USFM-005", + Category = "USFM", + Severity = BuildDiagnosticSeverity.Info, + Message = $"Incorrect verse segment {diagnosticDetails}", + Data = new Dictionary + { + { "projectName", projectName }, + { "projectGuid", projectGuid }, + { "usfmFilename", usfmDiagnostic.Filename }, + { + "lineNumber", + usfmDiagnostic.LineNumbers.Count > 0 ? usfmDiagnostic.LineNumbers[0] : -1 + }, + { + "verseReference", + usfmDiagnostic.References.Count > 0 ? usfmDiagnostic.References[0] : "" + }, + { "parallelCorpusId", parallelCorpusId }, + { "monolingualCorpusId", monolingualCorpusId }, + }, + }, + Serval.Shared.Contracts.UsfmVersificationDiagnosticType.UnsupportedVerseRange => + new BuildDiagnostic + { + Code = "USFM-006", + Category = "USFM", + Severity = BuildDiagnosticSeverity.Info, + Message = $"Unsupported verse range {diagnosticDetails}", + Data = new Dictionary + { + { "projectName", projectName }, + { "projectGuid", projectGuid }, + { "usfmFilename", usfmDiagnostic.Filename }, + { + "lineNumber", + usfmDiagnostic.LineNumbers.Count > 0 ? usfmDiagnostic.LineNumbers[0] : -1 + }, + { + "verseReference", + usfmDiagnostic.References.Count > 0 ? usfmDiagnostic.References[0] : "" + }, + { "parallelCorpusId", parallelCorpusId }, + { "monolingualCorpusId", monolingualCorpusId }, + }, + }, + _ => throw new InvalidEnumArgumentException(nameof(usfmDiagnostic.Type)), + } + ); + } + } + + foreach ( + ( + string parallelCorpusId, + string monolingualCorpusId, + MissingParentProjectErrorContract error + ) in ParallelCorpusService.FindMissingParentProjects(parallelCorpora) + ) + { + diagnostics.Add( + new BuildDiagnostic + { + Code = "CONFIG-001", + Category = "CONFIG", + Severity = BuildDiagnosticSeverity.Warn, + Message = + $"Unable to locate parent project {error.ParentProjectName} {error.ParentProjectGuid} of daughter project {error.ProjectName} {error.ProjectGuid} (parallel corpus {parallelCorpusId}, monolingual corpus {monolingualCorpusId})", + Data = new Dictionary + { + { "parentProjectName", error.ParentProjectName }, + { "parentProjectGuid", error.ParentProjectGuid }, + { "daughterProjectName", error.ProjectName }, + { "daughterProjectGuid", error.ProjectGuid }, + { "parallelCorpusId", parallelCorpusId }, + { "monolingualCorpusId", monolingualCorpusId }, + }, + } + ); + } + + if (projectVersifications.Count > 1) + { + diagnostics.Add( + new BuildDiagnostic + { + Code = "CONFIG-002", + Category = "CONFIG", + Severity = BuildDiagnosticSeverity.Info, + Message = + $"Multiple versifications represented among Paratext projects selected for training or inferencing: {string.Join(", ", projectVersifications)}", + Data = new Dictionary { { "projectVersifications", projectVersifications } }, + } + ); + } + + if (inferenceCount == 0 && isNonPersistedTranslationEngine) + { + diagnostics.Add( + new BuildDiagnostic + { + Code = "CONFIG-004", + Category = "CONFIG", + Severity = BuildDiagnosticSeverity.Error, + Message = "There was no data specified for inferencing and the model is not persisted.", + Data = [], + } + ); + } + return diagnostics; + } + protected static (bool IsTrainFilteredByChapter, bool IsInferenceFilteredByChapter) CheckChapterFilters( IReadOnlyList parallelCorpora ) diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs index 37102092..4babe23a 100644 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs @@ -46,6 +46,7 @@ public async Task RunAsync_BuildWarnings() ( "corpusId1", "src_1", + "0000", "pt-source1", "Original", [ diff --git a/src/Serval/src/Serval.Shared.Contracts/IParallelCorpusService.cs b/src/Serval/src/Serval.Shared.Contracts/IParallelCorpusService.cs index 6f1071fe..59c94bf1 100644 --- a/src/Serval/src/Serval.Shared.Contracts/IParallelCorpusService.cs +++ b/src/Serval/src/Serval.Shared.Contracts/IParallelCorpusService.cs @@ -8,6 +8,7 @@ public interface IParallelCorpusService string ParallelCorpusId, string MonolingualCorpusId, string ProjectName, + string ProjectGuid, string VersificationName, IReadOnlyList Diagnostics )> AnalyzeUsfmVersification(IEnumerable parallelCorpora); diff --git a/src/Serval/src/Serval.Shared.Contracts/MissingParentProjectErrorContract.cs b/src/Serval/src/Serval.Shared.Contracts/MissingParentProjectErrorContract.cs index bb382db1..52d843ef 100644 --- a/src/Serval/src/Serval.Shared.Contracts/MissingParentProjectErrorContract.cs +++ b/src/Serval/src/Serval.Shared.Contracts/MissingParentProjectErrorContract.cs @@ -3,5 +3,7 @@ namespace Serval.Shared.Contracts; public record MissingParentProjectErrorContract { public required string ProjectName { get; init; } + public required string ProjectGuid { get; init; } public required string ParentProjectName { get; init; } + public required string ParentProjectGuid { get; init; } } diff --git a/src/Serval/src/Serval.Shared/Services/ParallelCorpusService.cs b/src/Serval/src/Serval.Shared/Services/ParallelCorpusService.cs index d0d4f131..e782473f 100644 --- a/src/Serval/src/Serval.Shared/Services/ParallelCorpusService.cs +++ b/src/Serval/src/Serval.Shared/Services/ParallelCorpusService.cs @@ -11,6 +11,7 @@ public class ParallelCorpusService : IParallelCorpusService string ParallelCorpusId, string MonolingualCorpusId, string ProjectName, + string ProjectGuid, string VersificationName, IReadOnlyList Diagnostics )> AnalyzeUsfmVersification(IEnumerable parallelCorpora) @@ -20,6 +21,7 @@ IReadOnlyList Diagnostics string ParallelCorpusId, string MonolingualCorpusId, string ProjectName, + string ProjectGuid, string VersificationName, IReadOnlyList Diagnostics )> diagnosticsPerCorpus = []; @@ -44,6 +46,7 @@ IReadOnlyList Diagnostics parallelCorpus.Id, monolingualCorpus.Id, analysis.ProjectSettings.Name, + analysis.ProjectSettings.Guid, analysis.ProjectSettings.Versification.Name, [ .. analysis.Diagnostics.Select(d => new UsfmVersificationDiagnosticContract @@ -167,7 +170,13 @@ MissingParentProjectErrorContract Error ( parallelCorpus.Id, monolingualCorpus.Id, - new() { ProjectName = settings.Name, ParentProjectName = settings.ParentName } + new() + { + ProjectName = settings.Name, + ProjectGuid = settings.Guid, + ParentProjectName = settings.ParentName, + ParentProjectGuid = settings.ParentGuid, + } ) ); } diff --git a/src/Serval/test/Serval.Shared.Tests/Services/ParallelCorpusServiceTests.cs b/src/Serval/test/Serval.Shared.Tests/Services/ParallelCorpusServiceTests.cs index 98780e33..e9e36494 100644 --- a/src/Serval/test/Serval.Shared.Tests/Services/ParallelCorpusServiceTests.cs +++ b/src/Serval/test/Serval.Shared.Tests/Services/ParallelCorpusServiceTests.cs @@ -218,6 +218,7 @@ public void AnalyzeUsfmVersification() string ParallelCorpusId, string MonolingualCorpusId, string ProjectName, + string ProjectGuid, string VersificationName, IReadOnlyList Diagnostics )> analysis = env.Processor.AnalyzeUsfmVersification([parallelCorpus]); From 6da1e6499d365816434417e270fe2d584c5da8fe Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Tue, 28 Jul 2026 15:27:04 -0400 Subject: [PATCH 2/9] Add low confidence diagnostic --- .../Models/BuildDiagnostic.cs | 2 +- .../Services/EchoPreprocessBuildJob.cs | 13 +++ .../EchoWordAlignmentPreprocessBuildJob.cs | 1 + .../Services/PreprocessBuildJob.cs | 2 +- .../ServalTranslationPlatformService.cs | 10 +++ .../Services/TranslationPreprocessBuildJob.cs | 21 +++++ .../WordAlignmentPreprocessBuildJob.cs | 21 +++++ src/Serval/src/Serval.Client/Client.g.cs | 50 ++++++++++- .../DiagnosticContract.cs | 17 ++++ .../ExecutionDataContract.cs | 1 + .../Serval.Translation/Dtos/DiagnosticDto.cs | 17 ++++ .../Dtos/ExecutionDataDto.cs | 3 +- .../Serval.Translation/Models/Diagnostic.cs | 17 ++++ .../Models/ExecutionData.cs | 3 +- .../Serval.Translation/Services/DtoMapper.cs | 15 +++- .../Services/PlatformService.cs | 85 +++++++++++++++++-- .../test/Serval.E2ETests/ServalApiTests.cs | 2 +- .../Services/PlatformServiceTests.cs | 8 +- 18 files changed, 270 insertions(+), 18 deletions(-) create mode 100644 src/Serval/src/Serval.Translation.Contracts/DiagnosticContract.cs create mode 100644 src/Serval/src/Serval.Translation/Dtos/DiagnosticDto.cs create mode 100644 src/Serval/src/Serval.Translation/Models/Diagnostic.cs diff --git a/src/Machine/src/Serval.Machine.Shared/Models/BuildDiagnostic.cs b/src/Machine/src/Serval.Machine.Shared/Models/BuildDiagnostic.cs index c561810b..bc973371 100644 --- a/src/Machine/src/Serval.Machine.Shared/Models/BuildDiagnostic.cs +++ b/src/Machine/src/Serval.Machine.Shared/Models/BuildDiagnostic.cs @@ -1,6 +1,6 @@ namespace Serval.Machine.Shared.Models; -public class BuildDiagnostic +public record BuildDiagnostic { public required string Code { get; init; } public required string Category { get; init; } diff --git a/src/Machine/src/Serval.Machine.Shared/Services/EchoPreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/EchoPreprocessBuildJob.cs index 2f23ae9a..7c0b70db 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/EchoPreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/EchoPreprocessBuildJob.cs @@ -30,6 +30,7 @@ protected override async Task UpdateBuildExecutionData( PreprocessStats stats, string sourceLanguageTag, string targetLanguageTag, + bool isNonPersistedTranslationEngine, IReadOnlyList parallelCorpora, CancellationToken cancellationToken ) @@ -50,6 +51,17 @@ CancellationToken cancellationToken warnings = [tooManyWarningsWarning, .. warnings.Take(maxWarnings)]; } + IReadOnlyList diagnostics = GetDiagnostics( + stats.TrainCount, + stats.InferenceCount, + sourceLanguageTag, + targetLanguageTag, + true, + true, + isNonPersistedTranslationEngine, + parallelCorpora + ); + // Log summary of build data var buildPreprocessSummary = new JsonObject { @@ -74,6 +86,7 @@ CancellationToken cancellationToken TrainVerseCount = stats.TrainVerseCount, InferenceVerseCount = stats.InferenceVerseCount, Warnings = warnings, + Diagnostics = diagnostics, EngineSourceLanguageTag = sourceLanguageTag, EngineTargetLanguageTag = targetLanguageTag, ResolvedSourceLanguage = sourceLanguageTag, diff --git a/src/Machine/src/Serval.Machine.Shared/Services/EchoWordAlignmentPreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/EchoWordAlignmentPreprocessBuildJob.cs index 4c444e53..6f4e64ef 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/EchoWordAlignmentPreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/EchoWordAlignmentPreprocessBuildJob.cs @@ -90,6 +90,7 @@ protected override async Task UpdateBuildExecutionData( PreprocessStats stats, string sourceLanguageTag, string targetLanguageTag, + bool isNonPersistedTranslationEngine, IReadOnlyList parallelCorpora, CancellationToken cancellationToken ) diff --git a/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs index af26582a..e9906a1e 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs @@ -64,7 +64,7 @@ await UpdateBuildExecutionData( if (stats.InferenceCount == 0 && isNonPersistedTranslationEngine) { throw new InvalidOperationException( - $"There was no data specified for inferencing in build {buildId}. Build canceled." + $"There was no data specified for inferencing in build {buildId} and the model is not persisted. Build canceled." ); } diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationPlatformService.cs b/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationPlatformService.cs index d48b10fd..f134e1ee 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationPlatformService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationPlatformService.cs @@ -105,6 +105,16 @@ public Task UpdateBuildExecutionDataAsync( IsTrainFilteredByChapter = executionData.IsTrainFilteredByChapter, IsPretranslateFilteredByChapter = executionData.IsInferenceFilteredByChapter, Warnings = executionData.Warnings, + Diagnostics = executionData + .Diagnostics?.Select(d => new DiagnosticContract + { + Code = d.Code, + Category = d.Category, + Message = d.Message, + Severity = (DiagnosticSeverity)d.Severity, + Data = d.Data, + }) + .ToList(), EngineSourceLanguageTag = executionData.EngineSourceLanguageTag, EngineTargetLanguageTag = executionData.EngineTargetLanguageTag, ResolvedSourceLanguage = executionData.ResolvedSourceLanguage, diff --git a/src/Machine/src/Serval.Machine.Shared/Services/TranslationPreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/TranslationPreprocessBuildJob.cs index 190ac258..c03ba7f9 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/TranslationPreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/TranslationPreprocessBuildJob.cs @@ -89,6 +89,7 @@ protected override async Task UpdateBuildExecutionData( PreprocessStats stats, string sourceLanguageTag, string targetLanguageTag, + bool isNonPersistedTranslationEngine, IReadOnlyList parallelCorpora, CancellationToken cancellationToken ) @@ -101,6 +102,25 @@ CancellationToken cancellationToken parallelCorpora ); + int maxWarnings = BuildJobOptions.MaxWarnings; + if (warnings.Count > maxWarnings) + { + string tooManyWarningsWarning = + $"There were {warnings.Count} warnings. Only the first {maxWarnings} are shown."; + warnings = [tooManyWarningsWarning, .. warnings.Take(maxWarnings)]; + } + + IReadOnlyList diagnostics = GetDiagnostics( + stats.TrainCount, + stats.InferenceCount, + sourceLanguageTag, + targetLanguageTag, + true, + true, + isNonPersistedTranslationEngine, + parallelCorpora + ); + // Log summary of build data JsonObject buildPreprocessSummary = new() { @@ -123,6 +143,7 @@ CancellationToken cancellationToken TrainVerseCount = stats.TrainVerseCount, InferenceVerseCount = stats.InferenceVerseCount, Warnings = warnings, + Diagnostics = diagnostics, EngineSourceLanguageTag = sourceLanguageTag, EngineTargetLanguageTag = targetLanguageTag, }; diff --git a/src/Machine/src/Serval.Machine.Shared/Services/WordAlignmentPreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/WordAlignmentPreprocessBuildJob.cs index 2928815a..069fdba1 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/WordAlignmentPreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/WordAlignmentPreprocessBuildJob.cs @@ -88,6 +88,7 @@ protected override async Task UpdateBuildExecutionData( PreprocessStats stats, string sourceLanguageTag, string targetLanguageTag, + bool isNonPersistedTranslationEngine, IReadOnlyList parallelCorpora, CancellationToken cancellationToken ) @@ -100,6 +101,25 @@ CancellationToken cancellationToken parallelCorpora ); + int maxWarnings = BuildJobOptions.MaxWarnings; + if (warnings.Count > maxWarnings) + { + string tooManyWarningsWarning = + $"There were {warnings.Count} warnings. Only the first {maxWarnings} are shown."; + warnings = [tooManyWarningsWarning, .. warnings.Take(maxWarnings)]; + } + + IReadOnlyList diagnostics = GetDiagnostics( + stats.TrainCount, + stats.InferenceCount, + sourceLanguageTag, + targetLanguageTag, + true, + true, + isNonPersistedTranslationEngine, + parallelCorpora + ); + // Log summary of build data JsonObject buildPreprocessSummary = new() { @@ -122,6 +142,7 @@ CancellationToken cancellationToken IsInferenceFilteredByChapter = stats.IsInferenceFilteredByChapter, IsTrainFilteredByChapter = stats.IsTrainFilteredByChapter, Warnings = warnings, + Diagnostics = diagnostics, EngineSourceLanguageTag = sourceLanguageTag, EngineTargetLanguageTag = targetLanguageTag, }; diff --git a/src/Serval/src/Serval.Client/Client.g.cs b/src/Serval/src/Serval.Client/Client.g.cs index 34a76f00..829ae5d2 100644 --- a/src/Serval/src/Serval.Client/Client.g.cs +++ b/src/Serval/src/Serval.Client/Client.g.cs @@ -10944,6 +10944,10 @@ public partial class ExecutionData [System.ComponentModel.DataAnnotations.Required] public System.Collections.Generic.IList Warnings { get; set; } = new System.Collections.ObjectModel.Collection(); + [Newtonsoft.Json.JsonProperty("diagnostics", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public System.Collections.Generic.IList Diagnostics { get; set; } = new System.Collections.ObjectModel.Collection(); + [Newtonsoft.Json.JsonProperty("engineSourceLanguageTag", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] public string? EngineSourceLanguageTag { get; set; } = default!; @@ -10956,8 +10960,50 @@ public partial class ExecutionData [Newtonsoft.Json.JsonProperty("resolvedTargetLanguage", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] public string? ResolvedTargetLanguage { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("averagePretranslationConfidence", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public double? AveragePretranslationConfidence { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("averageVersePretranslationConfidence", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public double? AverageVersePretranslationConfidence { get; set; } = default!; + + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.7.1.0 (NJsonSchema v11.6.1.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class Diagnostic + { + + [Newtonsoft.Json.JsonProperty("code", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Code { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("category", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Category { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("message", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Message { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("severity", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + [Newtonsoft.Json.JsonConverter(typeof(Newtonsoft.Json.Converters.StringEnumConverter))] + public DiagnosticSeverity Severity { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("data", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public System.Collections.Generic.IDictionary Data { get; set; } = new System.Collections.Generic.Dictionary(); + + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.7.1.0 (NJsonSchema v11.6.1.0 (Newtonsoft.Json v13.0.0.0))")] + public enum DiagnosticSeverity + { + + [System.Runtime.Serialization.EnumMember(Value = @"Info")] + Info = 0, + + [System.Runtime.Serialization.EnumMember(Value = @"Warn")] + Warn = 1, + + [System.Runtime.Serialization.EnumMember(Value = @"Error")] + Error = 2, } diff --git a/src/Serval/src/Serval.Translation.Contracts/DiagnosticContract.cs b/src/Serval/src/Serval.Translation.Contracts/DiagnosticContract.cs new file mode 100644 index 00000000..f3157ef1 --- /dev/null +++ b/src/Serval/src/Serval.Translation.Contracts/DiagnosticContract.cs @@ -0,0 +1,17 @@ +namespace Serval.Translation.Contracts; + +public record DiagnosticContract +{ + public required string Code { get; init; } + public required string Category { get; init; } + public required string Message { get; init; } + public required DiagnosticSeverity Severity { get; init; } + public required Dictionary Data { get; init; } +} + +public enum DiagnosticSeverity +{ + Info, + Warn, + Error, +} diff --git a/src/Serval/src/Serval.Translation.Contracts/ExecutionDataContract.cs b/src/Serval/src/Serval.Translation.Contracts/ExecutionDataContract.cs index d7fd7422..e0b67f58 100644 --- a/src/Serval/src/Serval.Translation.Contracts/ExecutionDataContract.cs +++ b/src/Serval/src/Serval.Translation.Contracts/ExecutionDataContract.cs @@ -9,6 +9,7 @@ public record ExecutionDataContract public IReadOnlyDictionary>? TrainVerseCount { get; init; } public IReadOnlyDictionary>? PretranslateVerseCount { get; init; } public IReadOnlyList? Warnings { get; init; } + public IReadOnlyList? Diagnostics { get; init; } public string? EngineSourceLanguageTag { get; init; } public string? EngineTargetLanguageTag { get; init; } public string? ResolvedSourceLanguage { get; init; } diff --git a/src/Serval/src/Serval.Translation/Dtos/DiagnosticDto.cs b/src/Serval/src/Serval.Translation/Dtos/DiagnosticDto.cs new file mode 100644 index 00000000..a45b1d88 --- /dev/null +++ b/src/Serval/src/Serval.Translation/Dtos/DiagnosticDto.cs @@ -0,0 +1,17 @@ +namespace Serval.Translation.Dtos; + +public record DiagnosticDto +{ + public required string Code { get; init; } + public required string Category { get; init; } + public required string Message { get; init; } + public required DiagnosticSeverity Severity { get; init; } + public required Dictionary Data { get; init; } +} + +public enum DiagnosticSeverity +{ + Info, + Warn, + Error, +} diff --git a/src/Serval/src/Serval.Translation/Dtos/ExecutionDataDto.cs b/src/Serval/src/Serval.Translation/Dtos/ExecutionDataDto.cs index ce427261..92b89461 100644 --- a/src/Serval/src/Serval.Translation/Dtos/ExecutionDataDto.cs +++ b/src/Serval/src/Serval.Translation/Dtos/ExecutionDataDto.cs @@ -9,9 +9,10 @@ public record ExecutionDataDto public IReadOnlyDictionary>? TrainVerseCount { get; init; } public IReadOnlyDictionary>? PretranslateVerseCount { get; init; } public IReadOnlyList Warnings { get; init; } = []; + public IReadOnlyList Diagnostics { get; init; } = []; public string? EngineSourceLanguageTag { get; init; } public string? EngineTargetLanguageTag { get; init; } public string? ResolvedSourceLanguage { get; init; } public string? ResolvedTargetLanguage { get; init; } - public double? AveragePretranslationConfidence { get; init; } + public double? AverageVersePretranslationConfidence { get; init; } } diff --git a/src/Serval/src/Serval.Translation/Models/Diagnostic.cs b/src/Serval/src/Serval.Translation/Models/Diagnostic.cs new file mode 100644 index 00000000..6572a7c0 --- /dev/null +++ b/src/Serval/src/Serval.Translation/Models/Diagnostic.cs @@ -0,0 +1,17 @@ +namespace Serval.Translation.Models; + +public record Diagnostic +{ + public required string Code { get; init; } + public required string Category { get; init; } + public required string Message { get; init; } + public required DiagnosticSeverity Severity { get; init; } + public required Dictionary Data { get; init; } +} + +public enum DiagnosticSeverity +{ + Info, + Warn, + Error, +} diff --git a/src/Serval/src/Serval.Translation/Models/ExecutionData.cs b/src/Serval/src/Serval.Translation/Models/ExecutionData.cs index feb9a2af..e12496f7 100644 --- a/src/Serval/src/Serval.Translation/Models/ExecutionData.cs +++ b/src/Serval/src/Serval.Translation/Models/ExecutionData.cs @@ -9,9 +9,10 @@ public record ExecutionData public IReadOnlyDictionary>? TrainVerseCount { get; init; } public IReadOnlyDictionary>? PretranslateVerseCount { get; init; } public IReadOnlyList? Warnings { get; init; } + public IReadOnlyList? Diagnostics { get; init; } public string? EngineSourceLanguageTag { get; init; } public string? EngineTargetLanguageTag { get; init; } public string? ResolvedSourceLanguage { get; init; } public string? ResolvedTargetLanguage { get; init; } - public double? AveragePretranslationConfidence { get; init; } + public double? AverageVersePretranslationConfidence { get; init; } } diff --git a/src/Serval/src/Serval.Translation/Services/DtoMapper.cs b/src/Serval/src/Serval.Translation/Services/DtoMapper.cs index ad09d355..fe87f818 100644 --- a/src/Serval/src/Serval.Translation/Services/DtoMapper.cs +++ b/src/Serval/src/Serval.Translation/Services/DtoMapper.cs @@ -236,12 +236,25 @@ private static ExecutionDataDto Map(ExecutionData source) => IsPretranslateFilteredByChapter = source.IsPretranslateFilteredByChapter ?? false, IsTrainFilteredByChapter = source.IsTrainFilteredByChapter ?? false, Warnings = source.Warnings ?? [], + Diagnostics = source.Diagnostics?.Select(Map).ToList() ?? [], EngineSourceLanguageTag = source.EngineSourceLanguageTag, EngineTargetLanguageTag = source.EngineTargetLanguageTag, ResolvedSourceLanguage = source.ResolvedSourceLanguage, ResolvedTargetLanguage = source.ResolvedTargetLanguage, - AveragePretranslationConfidence = source.AveragePretranslationConfidence, + AverageVersePretranslationConfidence = source.AverageVersePretranslationConfidence, }; + + private static DiagnosticDto Map(Diagnostic source) + { + return new DiagnosticDto + { + Code = source.Code, + Category = source.Category, + Message = source.Message, + Severity = (Dtos.DiagnosticSeverity)source.Severity, + Data = source.Data, + }; + } } #pragma warning restore CS0612 // Type or member is obsolete diff --git a/src/Serval/src/Serval.Translation/Services/PlatformService.cs b/src/Serval/src/Serval.Translation/Services/PlatformService.cs index 88b6e694..6684354d 100644 --- a/src/Serval/src/Serval.Translation/Services/PlatformService.cs +++ b/src/Serval/src/Serval.Translation/Services/PlatformService.cs @@ -1,3 +1,5 @@ +using SIL.Machine.Corpora; + namespace Serval.Translation.Services; public class PlatformService( @@ -10,6 +12,9 @@ IEventRouter eventRouter { private const int PretranslationInsertBatchSize = 128; + private const double BadBookConfidenceThreshold = 0.35; + private const string ModelName = "NLLB"; + private readonly IRepository _builds = builds; private readonly IRepository _engines = engines; private readonly IRepository _pretranslations = pretranslations; @@ -316,6 +321,16 @@ await _builds.UpdateAsync( IsTrainFilteredByChapter = executionData.IsTrainFilteredByChapter, IsPretranslateFilteredByChapter = executionData.IsPretranslateFilteredByChapter, Warnings = executionData.Warnings?.ToList() ?? [], + Diagnostics = executionData + .Diagnostics?.Select(d => new Diagnostic + { + Code = d.Code, + Category = d.Category, + Message = d.Message, + Severity = (Models.DiagnosticSeverity)d.Severity, + Data = d.Data, + }) + .ToList(), EngineSourceLanguageTag = executionData.EngineSourceLanguageTag, EngineTargetLanguageTag = executionData.EngineTargetLanguageTag, ResolvedSourceLanguage = executionData.ResolvedSourceLanguage, @@ -383,6 +398,8 @@ public async Task InsertPretranslationsAsync( double logConfidenceTotal = 0.0; int confidenceCount = 0; int numPretranslations = 0; + Dictionary logConfidenceTotalPerBook = []; + Dictionary confidenceCountPerBook = []; await foreach (PretranslationContract item in pretranslations.WithCancellation(cancellationToken)) { batch.Add( @@ -399,7 +416,7 @@ public async Task InsertPretranslationsAsync( SourceTokens = item.SourceTokens, TranslationTokens = item.TranslationTokens, Alignment = item - .Alignment?.Select(a => new AlignedWordPair + .Alignment?.Select(a => new Shared.Models.AlignedWordPair { SourceIndex = a.SourceIndex, TargetIndex = a.TargetIndex, @@ -409,11 +426,25 @@ public async Task InsertPretranslationsAsync( Confidence = item.Confidence, } ); - double? confidence = item.Confidence; - if (confidence != null && confidence > 0.0) + + if (item.TargetRefs.Count > 0 && ScriptureRef.TryParse(item.TargetRefs[0], out ScriptureRef scriptureRef)) { - logConfidenceTotal += Math.Log((double)confidence); - confidenceCount++; + string bookId = scriptureRef.Book; + double? confidence = item.Confidence; + if (confidence != null && confidence > 0.0) + { + double logConfidence = Math.Log((double)confidence); + logConfidenceTotal += logConfidence; + confidenceCount++; + + if (!logConfidenceTotalPerBook.ContainsKey(bookId)) + logConfidenceTotalPerBook[bookId] = 0.0; + logConfidenceTotalPerBook[bookId] += logConfidence; + + if (!confidenceCountPerBook.ContainsKey(bookId)) + confidenceCountPerBook[bookId] = 0; + confidenceCountPerBook[bookId]++; + } } numPretranslations++; @@ -426,11 +457,53 @@ public async Task InsertPretranslationsAsync( if (batch.Count > 0) await _pretranslations.InsertAllAsync(batch, CancellationToken.None); + IEnumerable badBookConfidences = logConfidenceTotalPerBook + .Select(kvp => + { + string bookId = kvp.Key; + double logTotal = kvp.Value; + int count = confidenceCountPerBook[bookId]; + double averageConfidence = count > 0 ? Math.Exp(logTotal / count) : 0.0; + return (bookId, averageConfidence); + }) + .Where(b => b.averageConfidence < BadBookConfidenceThreshold) + .Select(b => new Diagnostic + { + Code = "MODEL-004", + Category = "MODEL", + Severity = Models.DiagnosticSeverity.Warn, + Message = + $"The average pretranslation model confidence {b.averageConfidence} in book {b.bookId} is unusually low for the base model {ModelName}", + Data = new Dictionary + { + { "bookId", b.bookId }, + { "averagePretranslationConfidence", b.averageConfidence }, + { "modelName", ModelName }, + }, + }); + + if (badBookConfidences.Any()) + { + Build? currentBuild = await _builds.GetAsync(b => b.Id == buildId, cancellationToken); + + await _builds.UpdateAsync( + b => b.Id == buildId, + u => + u.Set( + b => b.ExecutionData.Diagnostics, + currentBuild?.ExecutionData.Diagnostics is null + ? [.. badBookConfidences] + : [.. currentBuild.ExecutionData.Diagnostics, .. badBookConfidences] + ), + cancellationToken: cancellationToken + ); + } + await _builds.UpdateAsync( b => b.Id == buildId, u => u.Set( - b => b.ExecutionData.AveragePretranslationConfidence, + b => b.ExecutionData.AverageVersePretranslationConfidence, // Calculate the geometric mean of the pretranslation confidences confidenceCount > 0 ? Math.Exp(logConfidenceTotal / confidenceCount) diff --git a/src/Serval/test/Serval.E2ETests/ServalApiTests.cs b/src/Serval/test/Serval.E2ETests/ServalApiTests.cs index 1e80410a..4d42818c 100644 --- a/src/Serval/test/Serval.E2ETests/ServalApiTests.cs +++ b/src/Serval/test/Serval.E2ETests/ServalApiTests.cs @@ -292,7 +292,7 @@ public async Task Nmt_Paratext(bool withAdditionalFiles) string buildId = await _helperClient.BuildEngineAsync(engineId); TranslationBuild build = await _helperClient.TranslationEnginesClient.GetBuildAsync(engineId, buildId); Assert.That(build.State, Is.EqualTo(JobState.Completed), JsonSerializer.Serialize(build)); - Assert.That(build.ExecutionData.AveragePretranslationConfidence, Is.GreaterThan(0.2)); + Assert.That(build.ExecutionData.AverageVersePretranslationConfidence, Is.GreaterThan(0.2)); IList translations = await _helperClient.TranslationEnginesClient.GetAllPretranslationsAsync( engineId, diff --git a/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs b/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs index 23c032f4..8dfc19e7 100644 --- a/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs +++ b/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs @@ -58,7 +58,7 @@ await env.Builds.InsertAsync( await env.PlatformService.BuildCompletedAsync("b0", 0, 0.0); Assert.That(env.Pretranslations.Count, Is.EqualTo(1)); Assert.That( - (await env.Builds.GetAsync(b => b.Id == "b0"))?.ExecutionData.AveragePretranslationConfidence, + (await env.Builds.GetAsync(b => b.Id == "b0"))?.ExecutionData.AverageVersePretranslationConfidence, Is.Zero.Within(0.001) ); @@ -71,13 +71,13 @@ await env.PlatformService.InsertPretranslationsAsync( await env.PlatformService.BuildCompletedAsync("b0", 0, 0.0); Assert.That(env.Pretranslations.Count, Is.EqualTo(0)); Assert.That( - (await env.Builds.GetAsync(b => b.Id == "b0"))?.ExecutionData.AveragePretranslationConfidence, + (await env.Builds.GetAsync(b => b.Id == "b0"))?.ExecutionData.AverageVersePretranslationConfidence, Is.Zero.Within(0.001) ); } [Test] - public async Task BuildCompletedAsync_AveragePretranslationConfidence() + public async Task BuildCompletedAsync_AverageVersePretranslationConfidence() { var env = new TestEnvironment(); await env.Engines.InsertAsync( @@ -109,7 +109,7 @@ await env.PlatformService.InsertPretranslationsAsync( await env.PlatformService.BuildCompletedAsync("b0", 0, 0.0); Assert.That( - (await env.Builds.GetAsync(b => b.Id == "b0"))?.ExecutionData.AveragePretranslationConfidence, + (await env.Builds.GetAsync(b => b.Id == "b0"))?.ExecutionData.AverageVersePretranslationConfidence, Is.EqualTo(0.5).Within(0.0001) ); } From 7e3bc666151ff4a6674c7211a0caa3ad39dd1979 Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Tue, 28 Jul 2026 15:52:23 -0400 Subject: [PATCH 3/9] Adjust message wording --- .../src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs index e9906a1e..9baec0da 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs @@ -193,7 +193,7 @@ MissingParentProjectErrorContract error if (versifications.Count > 1) { warnings.Add( - $"Multiple versifications represented among Paratext projects selected for training or inferencing: {string.Join(", ", versifications)}" + $"There are multiple versifications represented among Paratext projects selected for training or inferencing: {string.Join(", ", versifications)}." ); } @@ -412,7 +412,7 @@ MissingParentProjectErrorContract error Category = "CONFIG", Severity = BuildDiagnosticSeverity.Info, Message = - $"Multiple versifications represented among Paratext projects selected for training or inferencing: {string.Join(", ", projectVersifications)}", + $"There are multiple versifications represented among Paratext projects selected for training or inferencing: {string.Join(", ", projectVersifications)}.", Data = new Dictionary { { "projectVersifications", projectVersifications } }, } ); From 1c9b919b7ab1249262eef667189d9e2b56b563f5 Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Tue, 28 Jul 2026 15:54:22 -0400 Subject: [PATCH 4/9] Add comment --- src/Serval/src/Serval.Translation/Services/PlatformService.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Serval/src/Serval.Translation/Services/PlatformService.cs b/src/Serval/src/Serval.Translation/Services/PlatformService.cs index 6684354d..57c99707 100644 --- a/src/Serval/src/Serval.Translation/Services/PlatformService.cs +++ b/src/Serval/src/Serval.Translation/Services/PlatformService.cs @@ -13,7 +13,7 @@ IEventRouter eventRouter private const int PretranslationInsertBatchSize = 128; private const double BadBookConfidenceThreshold = 0.35; - private const string ModelName = "NLLB"; + private const string ModelName = "NLLB"; //TODO where can we store this since it's used in multiple classes? In a constants.cs? On the build object itself? private readonly IRepository _builds = builds; private readonly IRepository _engines = engines; From b227868dc247be4b39d41b605bb32bf975667db3 Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Tue, 28 Jul 2026 15:56:41 -0400 Subject: [PATCH 5/9] Switch back to general pretranslation average confidence --- src/Serval/src/Serval.Client/Client.g.cs | 154 +++++++++--------- .../Dtos/ExecutionDataDto.cs | 2 +- .../Models/ExecutionData.cs | 2 +- .../Serval.Translation/Services/DtoMapper.cs | 2 +- .../Services/PlatformService.cs | 21 ++- .../test/Serval.E2ETests/ServalApiTests.cs | 2 +- .../Services/PlatformServiceTests.cs | 8 +- 7 files changed, 97 insertions(+), 94 deletions(-) diff --git a/src/Serval/src/Serval.Client/Client.g.cs b/src/Serval/src/Serval.Client/Client.g.cs index 829ae5d2..7a5b4669 100644 --- a/src/Serval/src/Serval.Client/Client.g.cs +++ b/src/Serval/src/Serval.Client/Client.g.cs @@ -448,7 +448,7 @@ private string ConvertToString(object? value, System.Globalization.CultureInfo c var field_ = System.Reflection.IntrospectionExtensions.GetTypeInfo(value.GetType()).GetDeclaredField(name); if (field_ != null) { - var attribute = System.Reflection.CustomAttributeExtensions.GetCustomAttribute(field_, typeof(System.Runtime.Serialization.EnumMemberAttribute)) + var attribute = System.Reflection.CustomAttributeExtensions.GetCustomAttribute(field_, typeof(System.Runtime.Serialization.EnumMemberAttribute)) as System.Runtime.Serialization.EnumMemberAttribute; if (attribute != null) { @@ -460,7 +460,7 @@ private string ConvertToString(object? value, System.Globalization.CultureInfo c return converted == null ? string.Empty : converted; } } - else if (value is bool) + else if (value is bool) { return System.Convert.ToString((bool)value, cultureInfo).ToLowerInvariant(); } @@ -498,7 +498,7 @@ public partial interface IDataFilesClient /// /// /// Sample request: - ///
+ ///
///
POST /files ///
{ ///
"format": "text", @@ -629,7 +629,7 @@ public string BaseUrl /// /// /// Sample request: - ///
+ ///
///
POST /files ///
{ ///
"format": "text", @@ -1394,7 +1394,7 @@ private string ConvertToString(object? value, System.Globalization.CultureInfo c var field_ = System.Reflection.IntrospectionExtensions.GetTypeInfo(value.GetType()).GetDeclaredField(name); if (field_ != null) { - var attribute = System.Reflection.CustomAttributeExtensions.GetCustomAttribute(field_, typeof(System.Runtime.Serialization.EnumMemberAttribute)) + var attribute = System.Reflection.CustomAttributeExtensions.GetCustomAttribute(field_, typeof(System.Runtime.Serialization.EnumMemberAttribute)) as System.Runtime.Serialization.EnumMemberAttribute; if (attribute != null) { @@ -1406,7 +1406,7 @@ private string ConvertToString(object? value, System.Globalization.CultureInfo c return converted == null ? string.Empty : converted; } } - else if (value is bool) + else if (value is bool) { return System.Convert.ToString((bool)value, cultureInfo).ToLowerInvariant(); } @@ -2140,7 +2140,7 @@ private string ConvertToString(object? value, System.Globalization.CultureInfo c var field_ = System.Reflection.IntrospectionExtensions.GetTypeInfo(value.GetType()).GetDeclaredField(name); if (field_ != null) { - var attribute = System.Reflection.CustomAttributeExtensions.GetCustomAttribute(field_, typeof(System.Runtime.Serialization.EnumMemberAttribute)) + var attribute = System.Reflection.CustomAttributeExtensions.GetCustomAttribute(field_, typeof(System.Runtime.Serialization.EnumMemberAttribute)) as System.Runtime.Serialization.EnumMemberAttribute; if (attribute != null) { @@ -2152,7 +2152,7 @@ private string ConvertToString(object? value, System.Globalization.CultureInfo c return converted == null ? string.Empty : converted; } } - else if (value is bool) + else if (value is bool) { return System.Convert.ToString((bool)value, cultureInfo).ToLowerInvariant(); } @@ -2580,7 +2580,7 @@ private string ConvertToString(object? value, System.Globalization.CultureInfo c var field_ = System.Reflection.IntrospectionExtensions.GetTypeInfo(value.GetType()).GetDeclaredField(name); if (field_ != null) { - var attribute = System.Reflection.CustomAttributeExtensions.GetCustomAttribute(field_, typeof(System.Runtime.Serialization.EnumMemberAttribute)) + var attribute = System.Reflection.CustomAttributeExtensions.GetCustomAttribute(field_, typeof(System.Runtime.Serialization.EnumMemberAttribute)) as System.Runtime.Serialization.EnumMemberAttribute; if (attribute != null) { @@ -2592,7 +2592,7 @@ private string ConvertToString(object? value, System.Globalization.CultureInfo c return converted == null ? string.Empty : converted; } } - else if (value is bool) + else if (value is bool) { return System.Convert.ToString((bool)value, cultureInfo).ToLowerInvariant(); } @@ -2715,14 +2715,14 @@ public partial interface ITranslationEnginesClient ///
### nmt ///
The Neural Machine Translation engine is primarily used for pretranslations. It is fine-tuned from Meta's NLLB-200. Valid IETF language tags provided to Serval will be converted to [NLLB-200 codes](https://github.com/facebookresearch/flores/tree/main/flores200#languages-in-flores-200). See more about language tag resolution [here](https://github.com/sillsdev/serval/wiki/FLORES%E2%80%90200-Language-Code-Resolution-for-NMT-Engine). ///
* **`isModelPersisted`**: (default to `false`) Whether the model can be downloaded by the client after it has been successfully built. - ///
+ ///
///
If you use a language among NLLB's supported languages, Serval will utilize everything the NLLB-200 model already knows about that language when translating. If the language you are working with is not among NLLB's supported languages, the language code will have no effect. - ///
+ ///
///
Typical endpoints: pretranslate ///
### echo ///
The echo engine has full coverage of all nmt and smt-transfer endpoints. Endpoints like create and build return empty responses. Endpoints like translate and get-word-graph echo the sent content back to the user in a format that mocks nmt or smt-transfer. For example, translating a segment "test" with the echo engine would yield a translation response with translation "test". This engine is useful for debugging and testing purposes. ///
## Sample request: - ///
+ ///
///
{ ///
"name": "myTeam:myProject:myEngine", ///
"sourceLanguage": "el", @@ -2810,7 +2810,7 @@ public partial interface ITranslationEnginesClient /// /// /// ## Sample request: - ///
+ ///
///
{ ///
"sourceLanguage": "en", ///
"targetLanguage": "en" @@ -2876,30 +2876,30 @@ public partial interface ITranslationEnginesClient ///
Specifying a corpus: ///
* A (legacy) corpus is selected by specifying `corpusId` and a parallel corpus is selected by specifying `parallelCorpusId`. ///
* A parallel corpus can be further filtered by specifying particular corpusIds in `sourceFilters` or `targetFilters`. - ///
+ ///
///
Filtering by text id or chapter: ///
* Paratext projects can be filtered by [book using the `textIds`](https://github.com/sillsdev/libpalaso/blob/master/SIL.Scripture/Canon.cs). ///
* Filters can also be supplied via the `scriptureRange` parameter as ranges of biblical text. See [here](https://github.com/sillsdev/serval/wiki/Filtering-Paratext-Project-Data-with-a-Scripture-Range). ///
* All Paratext project filtering follows original versification. See [here](https://github.com/sillsdev/serval/wiki/Versification-in-Serval) for more information. - ///
+ ///
///
Filter - train on all or none ///
* If `trainOn` or `pretranslate` is not provided, all corpora will be used for training or pretranslation respectively ///
* If a corpus is selected for training or pretranslation and neither `scriptureRange` nor `textIds` is defined, all of the selected corpus will be used. ///
* If a corpus is selected for training or pretranslation and an empty `scriptureRange` or `textIds` is defined, none of the selected corpus will be used. ///
* If a corpus is selected for training or pretranslation but no further filters are provided, all selected corpora will be used for training or pretranslation respectively. - ///
+ ///
///
Specify the corpora and text ids/scripture ranges within those corpora to pretranslate. When a corpus is selected for pretranslation, ///
the following text will be pretranslated: ///
* Text segments that are in the source but do not exist in the target. ///
* Text segments that are in the source and the target, but because of `trainOn` filtering, have not been trained on. ///
If the engine does not support pretranslation, these fields have no effect. ///
Pretranslating uses the same filtering as training. - ///
+ ///
///
The `options` parameter of the build config provides the ability to pass build configuration parameters as a JSON object. ///
See [nmt job settings documentation](https://github.com/sillsdev/serval/wiki/NMT-Build-Options) about configuring job parameters. ///
See [smt-transfer job settings documentation](https://github.com/sillsdev/serval/wiki/SMT-Transfer-Build-Options) about configuring job parameters. ///
See [keyterms parsing documentation](https://github.com/sillsdev/serval/wiki/Paratext-Key-Terms-Parsing) on how to use keyterms for training. - ///
+ ///
///
Note that when using a parallel corpus: ///
* If, within a single parallel corpus, multiple source corpora have data for the same text ids (for text files or Paratext Projects) or books (for Paratext Projects only using the scripture range), those sources will be mixed where they overlap by randomly choosing from each source per line/verse. ///
* If, within a single parallel corpus, multiple target corpora have data for the same text ids (for text files or Paratext Projects) or books (for Paratext Projects only using the scripture range), only the first of the targets that includes that text id/book will be used for that text id/book. @@ -2921,7 +2921,7 @@ public partial interface ITranslationEnginesClient ///
* The references defined in the source file per line, if any. ///
* An auto-generated reference of `[textId]:[lineNumber]`, 1 indexed. ///
* **`translation`**: the text of the pretranslation - ///
+ ///
///
Pretranslations can be filtered by text id if provided. ///
Only pretranslations for the most recent successful build of the engine are returned. /// @@ -2954,7 +2954,7 @@ public partial interface ITranslationEnginesClient ///
* The references defined in the source file per line, if any. ///
* An auto-generated reference of `[textId]:[lineNumber]`, 1 indexed. ///
* **`translation`**: the text of the pretranslation - ///
+ ///
///
Pretranslations can be filtered by text id if provided. ///
Only pretranslations for the most recent successful build of the engine are returned. /// @@ -2996,25 +2996,25 @@ public partial interface ITranslationEnginesClient ///
* `PreferPretranslated`: The existing and pretranslated texts are merged into the USFM, preferring pretranslated text. ///
* `OnlyExisting`: Return the existing target USFM file with no modifications (except updating the USFM id if needed). ///
* `OnlyPretranslated`: Only the pretranslated text is returned; all existing text in the target USFM is removed. - ///
+ ///
///
The source or target book can be used as the USFM template for the pretranslated text. The template can be controlled by the `template` parameter: ///
* `Auto`: The target book is used as the template if it exists; otherwise, the source book is used. **This is the default**. ///
* `Source`: The source book is used as the template. ///
* `Target`: The target book is used as the template. - ///
+ ///
///
The intra-segment USFM markers are handled in the following way: ///
* Each verse and non-verse text segment is stripped of all intra-segment USFM. ///
* Reference (\r) and remark (\rem) markers are not translated but carried through from the source to the target. - ///
+ ///
///
Preserving or stripping different types of USFM markers can be controlled by the `paragraph-marker-behavior`, `embed-behavior`, and `style-marker-behavior` parameters. ///
* `Preserve`: The USFM markers (or the entire embed) are preserved and placed at the end of the verse. **This is the default for paragraph markers and embeds**. ///
* `PreservePosition`: The USFM markers (or the entire embed) are placed in approximately the right location within the verse. **This option is only available for paragraph markers. Quality of placement may differ from language to language. Only works when `template` is set to `Source`**. ///
* `Strip`: The USFM markers (or the entire embed) are removed. **This is the default for style markers**. - ///
+ ///
///
Quote normalization behavior is controlled by the `quote-normalization-behavior` parameter options: ///
* `Normalized`: The quotes in the pretranslated USFM are normalized quotes (typically straight quotes: ', ") in the style of the source data. **This is the default**. ///
* `Denormalized`: The quotes in the pretranslated USFM are denormalized into the style of the target data. Quote denormalization may not be successful in all contexts. A remark will be added to the USFM listing the chapters that were successfully denormalized. - ///
+ ///
///
Only pretranslations for the most recent successful build of the engine are returned. ///
The USFM parsing and marker types used are defined here: [this wiki](https://github.com/sillsdev/serval/wiki/USFM-Parsing-and-Translation). /// @@ -3043,7 +3043,7 @@ public partial interface ITranslationEnginesClient ///
* The references defined in the source file per line, if any. ///
* An auto-generated reference of `[textId]:[lineNumber]`, 1 indexed. ///
* **`translation`**: the text of the pretranslation - ///
+ ///
///
Only pretranslations for the most recent successful build of the engine are returned. /// /// The translation engine id @@ -3074,10 +3074,10 @@ public partial interface ITranslationEnginesClient /// /// If an nmt build was successful and `isModelPersisted` is `true` for the engine, ///
then the model from the most recent successful build can be downloaded. - ///
+ ///
///
The endpoint will return a URL that can be used to download the model for up to 1 hour ///
after the request is made. If the URL is not used within that time, a new request will need to be made. - ///
+ ///
///
The download itself is created by g-zipping together the folder containing the fine tuned model ///
with all necessary supporting files. This zipped folder is then named by the pattern: ///
* <engine_id>_<model_revision>.tar.gz @@ -3097,25 +3097,25 @@ public partial interface ITranslationEnginesClient ///
* `PreferPretranslated`: The existing and pretranslated texts are merged into the USFM, preferring pretranslated text. ///
* `OnlyExisting`: Return the existing target USFM file with no modifications (except updating the USFM id if needed). ///
* `OnlyPretranslated`: Only the pretranslated text is returned; all existing text in the target USFM is removed. - ///
+ ///
///
The source or target book can be used as the USFM template for the pretranslated text. The template can be controlled by the `template` parameter: ///
* `Auto`: The target book is used as the template if it exists; otherwise, the source book is used. **This is the default**. ///
* `Source`: The source book is used as the template. ///
* `Target`: The target book is used as the template. - ///
+ ///
///
The intra-segment USFM markers are handled in the following way: ///
* Each verse and non-verse text segment is stripped of all intra-segment USFM. ///
* Reference (\r) and remark (\rem) markers are not translated but carried through from the source to the target. - ///
+ ///
///
Preserving or stripping different types of USFM markers can be controlled by the `paragraph-marker-behavior`, `embed-behavior`, and `style-marker-behavior` parameters. ///
* `Preserve`: The USFM markers (or the entire embed) are preserved and placed at the end of the verse. **This is the default for paragraph markers and embeds**. ///
* `PreservePosition`: The USFM markers (or the entire embed) are placed in approximately the right location within the verse. **This option is only available for paragraph markers. Quality of placement may differ from language to language. Only works when `template` is set to `Source`**. ///
* `Strip`: The USFM markers (or the entire embed) are removed. **This is the default for style markers**. - ///
+ ///
///
Quote normalization behavior is controlled by the `quote-normalization-behavior` parameter options: ///
* `Normalized`: The quotes in the pretranslated USFM are normalized quotes (typically straight quotes: ', ") in the style of the source data. **This is the default**. ///
* `Denormalized`: The quotes in the pretranslated USFM are denormalized into the style of the target data. Quote denormalization may not be successful in all contexts. A remark will be added to the USFM listing the chapters that were successfully denormalized. - ///
+ ///
///
Only pretranslations for the most recent successful build of the engine are returned. ///
The USFM parsing and marker types used are defined here: [this wiki](https://github.com/sillsdev/serval/wiki/USFM-Parsing-and-Translation). /// @@ -3143,7 +3143,7 @@ public partial interface ITranslationEnginesClient ///
* The references defined in the source file per line, if any. ///
* An auto-generated reference of `[textId]:[lineNumber]`, 1 indexed. ///
* **`translation`**: the text of the pretranslation - ///
+ ///
///
Only pretranslations for the most recent successful build of the engine are returned. /// /// The translation engine id @@ -3854,14 +3854,14 @@ public string BaseUrl ///
### nmt ///
The Neural Machine Translation engine is primarily used for pretranslations. It is fine-tuned from Meta's NLLB-200. Valid IETF language tags provided to Serval will be converted to [NLLB-200 codes](https://github.com/facebookresearch/flores/tree/main/flores200#languages-in-flores-200). See more about language tag resolution [here](https://github.com/sillsdev/serval/wiki/FLORES%E2%80%90200-Language-Code-Resolution-for-NMT-Engine). ///
* **`isModelPersisted`**: (default to `false`) Whether the model can be downloaded by the client after it has been successfully built. - ///
+ ///
///
If you use a language among NLLB's supported languages, Serval will utilize everything the NLLB-200 model already knows about that language when translating. If the language you are working with is not among NLLB's supported languages, the language code will have no effect. - ///
+ ///
///
Typical endpoints: pretranslate ///
### echo ///
The echo engine has full coverage of all nmt and smt-transfer endpoints. Endpoints like create and build return empty responses. Endpoints like translate and get-word-graph echo the sent content back to the user in a format that mocks nmt or smt-transfer. For example, translating a segment "test" with the echo engine would yield a translation response with translation "test". This engine is useful for debugging and testing purposes. ///
## Sample request: - ///
+ ///
///
{ ///
"name": "myTeam:myProject:myEngine", ///
"sourceLanguage": "el", @@ -4622,7 +4622,7 @@ public string BaseUrl /// /// /// ## Sample request: - ///
+ ///
///
{ ///
"sourceLanguage": "en", ///
"targetLanguage": "en" @@ -5182,30 +5182,30 @@ public string BaseUrl ///
Specifying a corpus: ///
* A (legacy) corpus is selected by specifying `corpusId` and a parallel corpus is selected by specifying `parallelCorpusId`. ///
* A parallel corpus can be further filtered by specifying particular corpusIds in `sourceFilters` or `targetFilters`. - ///
+ ///
///
Filtering by text id or chapter: ///
* Paratext projects can be filtered by [book using the `textIds`](https://github.com/sillsdev/libpalaso/blob/master/SIL.Scripture/Canon.cs). ///
* Filters can also be supplied via the `scriptureRange` parameter as ranges of biblical text. See [here](https://github.com/sillsdev/serval/wiki/Filtering-Paratext-Project-Data-with-a-Scripture-Range). ///
* All Paratext project filtering follows original versification. See [here](https://github.com/sillsdev/serval/wiki/Versification-in-Serval) for more information. - ///
+ ///
///
Filter - train on all or none ///
* If `trainOn` or `pretranslate` is not provided, all corpora will be used for training or pretranslation respectively ///
* If a corpus is selected for training or pretranslation and neither `scriptureRange` nor `textIds` is defined, all of the selected corpus will be used. ///
* If a corpus is selected for training or pretranslation and an empty `scriptureRange` or `textIds` is defined, none of the selected corpus will be used. ///
* If a corpus is selected for training or pretranslation but no further filters are provided, all selected corpora will be used for training or pretranslation respectively. - ///
+ ///
///
Specify the corpora and text ids/scripture ranges within those corpora to pretranslate. When a corpus is selected for pretranslation, ///
the following text will be pretranslated: ///
* Text segments that are in the source but do not exist in the target. ///
* Text segments that are in the source and the target, but because of `trainOn` filtering, have not been trained on. ///
If the engine does not support pretranslation, these fields have no effect. ///
Pretranslating uses the same filtering as training. - ///
+ ///
///
The `options` parameter of the build config provides the ability to pass build configuration parameters as a JSON object. ///
See [nmt job settings documentation](https://github.com/sillsdev/serval/wiki/NMT-Build-Options) about configuring job parameters. ///
See [smt-transfer job settings documentation](https://github.com/sillsdev/serval/wiki/SMT-Transfer-Build-Options) about configuring job parameters. ///
See [keyterms parsing documentation](https://github.com/sillsdev/serval/wiki/Paratext-Key-Terms-Parsing) on how to use keyterms for training. - ///
+ ///
///
Note that when using a parallel corpus: ///
* If, within a single parallel corpus, multiple source corpora have data for the same text ids (for text files or Paratext Projects) or books (for Paratext Projects only using the scripture range), those sources will be mixed where they overlap by randomly choosing from each source per line/verse. ///
* If, within a single parallel corpus, multiple target corpora have data for the same text ids (for text files or Paratext Projects) or books (for Paratext Projects only using the scripture range), only the first of the targets that includes that text id/book will be used for that text id/book. @@ -5341,7 +5341,7 @@ public string BaseUrl ///
* The references defined in the source file per line, if any. ///
* An auto-generated reference of `[textId]:[lineNumber]`, 1 indexed. ///
* **`translation`**: the text of the pretranslation - ///
+ ///
///
Pretranslations can be filtered by text id if provided. ///
Only pretranslations for the most recent successful build of the engine are returned. /// @@ -5592,7 +5592,7 @@ public string BaseUrl ///
* The references defined in the source file per line, if any. ///
* An auto-generated reference of `[textId]:[lineNumber]`, 1 indexed. ///
* **`translation`**: the text of the pretranslation - ///
+ ///
///
Pretranslations can be filtered by text id if provided. ///
Only pretranslations for the most recent successful build of the engine are returned. /// @@ -5857,25 +5857,25 @@ public string BaseUrl ///
* `PreferPretranslated`: The existing and pretranslated texts are merged into the USFM, preferring pretranslated text. ///
* `OnlyExisting`: Return the existing target USFM file with no modifications (except updating the USFM id if needed). ///
* `OnlyPretranslated`: Only the pretranslated text is returned; all existing text in the target USFM is removed. - ///
+ ///
///
The source or target book can be used as the USFM template for the pretranslated text. The template can be controlled by the `template` parameter: ///
* `Auto`: The target book is used as the template if it exists; otherwise, the source book is used. **This is the default**. ///
* `Source`: The source book is used as the template. ///
* `Target`: The target book is used as the template. - ///
+ ///
///
The intra-segment USFM markers are handled in the following way: ///
* Each verse and non-verse text segment is stripped of all intra-segment USFM. ///
* Reference (\r) and remark (\rem) markers are not translated but carried through from the source to the target. - ///
+ ///
///
Preserving or stripping different types of USFM markers can be controlled by the `paragraph-marker-behavior`, `embed-behavior`, and `style-marker-behavior` parameters. ///
* `Preserve`: The USFM markers (or the entire embed) are preserved and placed at the end of the verse. **This is the default for paragraph markers and embeds**. ///
* `PreservePosition`: The USFM markers (or the entire embed) are placed in approximately the right location within the verse. **This option is only available for paragraph markers. Quality of placement may differ from language to language. Only works when `template` is set to `Source`**. ///
* `Strip`: The USFM markers (or the entire embed) are removed. **This is the default for style markers**. - ///
+ ///
///
Quote normalization behavior is controlled by the `quote-normalization-behavior` parameter options: ///
* `Normalized`: The quotes in the pretranslated USFM are normalized quotes (typically straight quotes: ', ") in the style of the source data. **This is the default**. ///
* `Denormalized`: The quotes in the pretranslated USFM are denormalized into the style of the target data. Quote denormalization may not be successful in all contexts. A remark will be added to the USFM listing the chapters that were successfully denormalized. - ///
+ ///
///
Only pretranslations for the most recent successful build of the engine are returned. ///
The USFM parsing and marker types used are defined here: [this wiki](https://github.com/sillsdev/serval/wiki/USFM-Parsing-and-Translation). /// @@ -6050,7 +6050,7 @@ public string BaseUrl ///
* The references defined in the source file per line, if any. ///
* An auto-generated reference of `[textId]:[lineNumber]`, 1 indexed. ///
* **`translation`**: the text of the pretranslation - ///
+ ///
///
Only pretranslations for the most recent successful build of the engine are returned. /// /// The translation engine id @@ -6310,10 +6310,10 @@ public string BaseUrl /// /// If an nmt build was successful and `isModelPersisted` is `true` for the engine, ///
then the model from the most recent successful build can be downloaded. - ///
+ ///
///
The endpoint will return a URL that can be used to download the model for up to 1 hour ///
after the request is made. If the URL is not used within that time, a new request will need to be made. - ///
+ ///
///
The download itself is created by g-zipping together the folder containing the fine tuned model ///
with all necessary supporting files. This zipped folder is then named by the pattern: ///
* <engine_id>_<model_revision>.tar.gz @@ -6434,25 +6434,25 @@ public string BaseUrl ///
* `PreferPretranslated`: The existing and pretranslated texts are merged into the USFM, preferring pretranslated text. ///
* `OnlyExisting`: Return the existing target USFM file with no modifications (except updating the USFM id if needed). ///
* `OnlyPretranslated`: Only the pretranslated text is returned; all existing text in the target USFM is removed. - ///
+ ///
///
The source or target book can be used as the USFM template for the pretranslated text. The template can be controlled by the `template` parameter: ///
* `Auto`: The target book is used as the template if it exists; otherwise, the source book is used. **This is the default**. ///
* `Source`: The source book is used as the template. ///
* `Target`: The target book is used as the template. - ///
+ ///
///
The intra-segment USFM markers are handled in the following way: ///
* Each verse and non-verse text segment is stripped of all intra-segment USFM. ///
* Reference (\r) and remark (\rem) markers are not translated but carried through from the source to the target. - ///
+ ///
///
Preserving or stripping different types of USFM markers can be controlled by the `paragraph-marker-behavior`, `embed-behavior`, and `style-marker-behavior` parameters. ///
* `Preserve`: The USFM markers (or the entire embed) are preserved and placed at the end of the verse. **This is the default for paragraph markers and embeds**. ///
* `PreservePosition`: The USFM markers (or the entire embed) are placed in approximately the right location within the verse. **This option is only available for paragraph markers. Quality of placement may differ from language to language. Only works when `template` is set to `Source`**. ///
* `Strip`: The USFM markers (or the entire embed) are removed. **This is the default for style markers**. - ///
+ ///
///
Quote normalization behavior is controlled by the `quote-normalization-behavior` parameter options: ///
* `Normalized`: The quotes in the pretranslated USFM are normalized quotes (typically straight quotes: ', ") in the style of the source data. **This is the default**. ///
* `Denormalized`: The quotes in the pretranslated USFM are denormalized into the style of the target data. Quote denormalization may not be successful in all contexts. A remark will be added to the USFM listing the chapters that were successfully denormalized. - ///
+ ///
///
Only pretranslations for the most recent successful build of the engine are returned. ///
The USFM parsing and marker types used are defined here: [this wiki](https://github.com/sillsdev/serval/wiki/USFM-Parsing-and-Translation). /// @@ -6626,7 +6626,7 @@ public string BaseUrl ///
* The references defined in the source file per line, if any. ///
* An auto-generated reference of `[textId]:[lineNumber]`, 1 indexed. ///
* **`translation`**: the text of the pretranslation - ///
+ ///
///
Only pretranslations for the most recent successful build of the engine are returned. /// /// The translation engine id @@ -7362,7 +7362,7 @@ private string ConvertToString(object? value, System.Globalization.CultureInfo c var field_ = System.Reflection.IntrospectionExtensions.GetTypeInfo(value.GetType()).GetDeclaredField(name); if (field_ != null) { - var attribute = System.Reflection.CustomAttributeExtensions.GetCustomAttribute(field_, typeof(System.Runtime.Serialization.EnumMemberAttribute)) + var attribute = System.Reflection.CustomAttributeExtensions.GetCustomAttribute(field_, typeof(System.Runtime.Serialization.EnumMemberAttribute)) as System.Runtime.Serialization.EnumMemberAttribute; if (attribute != null) { @@ -7374,7 +7374,7 @@ private string ConvertToString(object? value, System.Globalization.CultureInfo c return converted == null ? string.Empty : converted; } } - else if (value is bool) + else if (value is bool) { return System.Convert.ToString((bool)value, cultureInfo).ToLowerInvariant(); } @@ -7774,7 +7774,7 @@ private string ConvertToString(object? value, System.Globalization.CultureInfo c var field_ = System.Reflection.IntrospectionExtensions.GetTypeInfo(value.GetType()).GetDeclaredField(name); if (field_ != null) { - var attribute = System.Reflection.CustomAttributeExtensions.GetCustomAttribute(field_, typeof(System.Runtime.Serialization.EnumMemberAttribute)) + var attribute = System.Reflection.CustomAttributeExtensions.GetCustomAttribute(field_, typeof(System.Runtime.Serialization.EnumMemberAttribute)) as System.Runtime.Serialization.EnumMemberAttribute; if (attribute != null) { @@ -7786,7 +7786,7 @@ private string ConvertToString(object? value, System.Globalization.CultureInfo c return converted == null ? string.Empty : converted; } } - else if (value is bool) + else if (value is bool) { return System.Convert.ToString((bool)value, cultureInfo).ToLowerInvariant(); } @@ -8067,7 +8067,7 @@ private string ConvertToString(object? value, System.Globalization.CultureInfo c var field_ = System.Reflection.IntrospectionExtensions.GetTypeInfo(value.GetType()).GetDeclaredField(name); if (field_ != null) { - var attribute = System.Reflection.CustomAttributeExtensions.GetCustomAttribute(field_, typeof(System.Runtime.Serialization.EnumMemberAttribute)) + var attribute = System.Reflection.CustomAttributeExtensions.GetCustomAttribute(field_, typeof(System.Runtime.Serialization.EnumMemberAttribute)) as System.Runtime.Serialization.EnumMemberAttribute; if (attribute != null) { @@ -8079,7 +8079,7 @@ private string ConvertToString(object? value, System.Globalization.CultureInfo c return converted == null ? string.Empty : converted; } } - else if (value is bool) + else if (value is bool) { return System.Convert.ToString((bool)value, cultureInfo).ToLowerInvariant(); } @@ -8172,7 +8172,7 @@ public partial interface IWordAlignmentEnginesClient ///
The echo-word-alignment engine has full coverage of all endpoints. Endpoints like create and build return empty responses. ///
Endpoints like align echo the sent content back to the user in the proper format. This engine is useful for debugging and testing purposes. ///
## Sample request: - ///
+ ///
///
{ ///
"name": "myTeam:myProject:myEngine", ///
"sourceLanguage": "el", @@ -8266,10 +8266,10 @@ public partial interface IWordAlignmentEnginesClient ///
Paratext projects can be filtered by [book using the `textIds`](https://github.com/sillsdev/libpalaso/blob/master/SIL.Scripture/Canon.cs). ///
Filters can also be supplied via `scriptureRange` parameter as ranges of biblical text. See [here](https://github.com/sillsdev/serval/wiki/Filtering-Paratext-Project-Data-with-a-Scripture-Range) ///
All Paratext project filtering follows original versification. See [here](https://github.com/sillsdev/serval/wiki/Versification-in-Serval) for more information. - ///
+ ///
///
Specify the corpora or text ids to word align on. ///
When a corpus or text id is selected for word align on, only text segments that are in both the source and the target will be aligned. - ///
+ ///
///
The `options` parameter of the build config provides the ability to pass build configuration parameters as a JSON object. ///
See [statistical alignment job settings documentation](https://github.com/sillsdev/serval/wiki/Statistical-Alignment-Build-Options) about configuring job parameters. /// @@ -8292,7 +8292,7 @@ public partial interface IWordAlignmentEnginesClient ///
* **`sourceTokens`**: the tokenized source segment ///
* **`targetTokens`**: the tokenized target segment ///
* **`alignment`**: a list of aligned word pairs with associated scores - ///
+ ///
///
Word alignments can be filtered by text id if provided. ///
Only word alignments for the most recent successful build of the engine are returned. /// @@ -8879,7 +8879,7 @@ public string BaseUrl ///
The echo-word-alignment engine has full coverage of all endpoints. Endpoints like create and build return empty responses. ///
Endpoints like align echo the sent content back to the user in the proper format. This engine is useful for debugging and testing purposes. ///
## Sample request: - ///
+ ///
///
{ ///
"name": "myTeam:myProject:myEngine", ///
"sourceLanguage": "el", @@ -9735,10 +9735,10 @@ public string BaseUrl ///
Paratext projects can be filtered by [book using the `textIds`](https://github.com/sillsdev/libpalaso/blob/master/SIL.Scripture/Canon.cs). ///
Filters can also be supplied via `scriptureRange` parameter as ranges of biblical text. See [here](https://github.com/sillsdev/serval/wiki/Filtering-Paratext-Project-Data-with-a-Scripture-Range) ///
All Paratext project filtering follows original versification. See [here](https://github.com/sillsdev/serval/wiki/Versification-in-Serval) for more information. - ///
+ ///
///
Specify the corpora or text ids to word align on. ///
When a corpus or text id is selected for word align on, only text segments that are in both the source and the target will be aligned. - ///
+ ///
///
The `options` parameter of the build config provides the ability to pass build configuration parameters as a JSON object. ///
See [statistical alignment job settings documentation](https://github.com/sillsdev/serval/wiki/Statistical-Alignment-Build-Options) about configuring job parameters. /// @@ -9875,7 +9875,7 @@ public string BaseUrl ///
* **`sourceTokens`**: the tokenized source segment ///
* **`targetTokens`**: the tokenized target segment ///
* **`alignment`**: a list of aligned word pairs with associated scores - ///
+ ///
///
Word alignments can be filtered by text id if provided. ///
Only word alignments for the most recent successful build of the engine are returned. /// @@ -10358,7 +10358,7 @@ private string ConvertToString(object? value, System.Globalization.CultureInfo c var field_ = System.Reflection.IntrospectionExtensions.GetTypeInfo(value.GetType()).GetDeclaredField(name); if (field_ != null) { - var attribute = System.Reflection.CustomAttributeExtensions.GetCustomAttribute(field_, typeof(System.Runtime.Serialization.EnumMemberAttribute)) + var attribute = System.Reflection.CustomAttributeExtensions.GetCustomAttribute(field_, typeof(System.Runtime.Serialization.EnumMemberAttribute)) as System.Runtime.Serialization.EnumMemberAttribute; if (attribute != null) { @@ -10370,7 +10370,7 @@ private string ConvertToString(object? value, System.Globalization.CultureInfo c return converted == null ? string.Empty : converted; } } - else if (value is bool) + else if (value is bool) { return System.Convert.ToString((bool)value, cultureInfo).ToLowerInvariant(); } @@ -10961,7 +10961,7 @@ public partial class ExecutionData public string? ResolvedTargetLanguage { get; set; } = default!; [Newtonsoft.Json.JsonProperty("averageVersePretranslationConfidence", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public double? AverageVersePretranslationConfidence { get; set; } = default!; + public double? AveragePretranslationConfidence { get; set; } = default!; } diff --git a/src/Serval/src/Serval.Translation/Dtos/ExecutionDataDto.cs b/src/Serval/src/Serval.Translation/Dtos/ExecutionDataDto.cs index 92b89461..af16fdb0 100644 --- a/src/Serval/src/Serval.Translation/Dtos/ExecutionDataDto.cs +++ b/src/Serval/src/Serval.Translation/Dtos/ExecutionDataDto.cs @@ -14,5 +14,5 @@ public record ExecutionDataDto public string? EngineTargetLanguageTag { get; init; } public string? ResolvedSourceLanguage { get; init; } public string? ResolvedTargetLanguage { get; init; } - public double? AverageVersePretranslationConfidence { get; init; } + public double? AveragePretranslationConfidence { get; init; } } diff --git a/src/Serval/src/Serval.Translation/Models/ExecutionData.cs b/src/Serval/src/Serval.Translation/Models/ExecutionData.cs index e12496f7..a7fe4c19 100644 --- a/src/Serval/src/Serval.Translation/Models/ExecutionData.cs +++ b/src/Serval/src/Serval.Translation/Models/ExecutionData.cs @@ -14,5 +14,5 @@ public record ExecutionData public string? EngineTargetLanguageTag { get; init; } public string? ResolvedSourceLanguage { get; init; } public string? ResolvedTargetLanguage { get; init; } - public double? AverageVersePretranslationConfidence { get; init; } + public double? AveragePretranslationConfidence { get; init; } } diff --git a/src/Serval/src/Serval.Translation/Services/DtoMapper.cs b/src/Serval/src/Serval.Translation/Services/DtoMapper.cs index fe87f818..be5db5fd 100644 --- a/src/Serval/src/Serval.Translation/Services/DtoMapper.cs +++ b/src/Serval/src/Serval.Translation/Services/DtoMapper.cs @@ -241,7 +241,7 @@ private static ExecutionDataDto Map(ExecutionData source) => EngineTargetLanguageTag = source.EngineTargetLanguageTag, ResolvedSourceLanguage = source.ResolvedSourceLanguage, ResolvedTargetLanguage = source.ResolvedTargetLanguage, - AverageVersePretranslationConfidence = source.AverageVersePretranslationConfidence, + AveragePretranslationConfidence = source.AveragePretranslationConfidence, }; private static DiagnosticDto Map(Diagnostic source) diff --git a/src/Serval/src/Serval.Translation/Services/PlatformService.cs b/src/Serval/src/Serval.Translation/Services/PlatformService.cs index 57c99707..76af2106 100644 --- a/src/Serval/src/Serval.Translation/Services/PlatformService.cs +++ b/src/Serval/src/Serval.Translation/Services/PlatformService.cs @@ -426,16 +426,19 @@ public async Task InsertPretranslationsAsync( Confidence = item.Confidence, } ); - - if (item.TargetRefs.Count > 0 && ScriptureRef.TryParse(item.TargetRefs[0], out ScriptureRef scriptureRef)) + double? confidence = item.Confidence; + if (confidence != null && confidence > 0.0) { - string bookId = scriptureRef.Book; - double? confidence = item.Confidence; - if (confidence != null && confidence > 0.0) + double logConfidence = Math.Log((double)confidence); + logConfidenceTotal += logConfidence; + confidenceCount++; + + if ( + item.TargetRefs.Count > 0 + && ScriptureRef.TryParse(item.TargetRefs[0], out ScriptureRef scriptureRef) + ) { - double logConfidence = Math.Log((double)confidence); - logConfidenceTotal += logConfidence; - confidenceCount++; + string bookId = scriptureRef.Book; if (!logConfidenceTotalPerBook.ContainsKey(bookId)) logConfidenceTotalPerBook[bookId] = 0.0; @@ -503,7 +506,7 @@ await _builds.UpdateAsync( b => b.Id == buildId, u => u.Set( - b => b.ExecutionData.AverageVersePretranslationConfidence, + b => b.ExecutionData.AveragePretranslationConfidence, // Calculate the geometric mean of the pretranslation confidences confidenceCount > 0 ? Math.Exp(logConfidenceTotal / confidenceCount) diff --git a/src/Serval/test/Serval.E2ETests/ServalApiTests.cs b/src/Serval/test/Serval.E2ETests/ServalApiTests.cs index 4d42818c..1e80410a 100644 --- a/src/Serval/test/Serval.E2ETests/ServalApiTests.cs +++ b/src/Serval/test/Serval.E2ETests/ServalApiTests.cs @@ -292,7 +292,7 @@ public async Task Nmt_Paratext(bool withAdditionalFiles) string buildId = await _helperClient.BuildEngineAsync(engineId); TranslationBuild build = await _helperClient.TranslationEnginesClient.GetBuildAsync(engineId, buildId); Assert.That(build.State, Is.EqualTo(JobState.Completed), JsonSerializer.Serialize(build)); - Assert.That(build.ExecutionData.AverageVersePretranslationConfidence, Is.GreaterThan(0.2)); + Assert.That(build.ExecutionData.AveragePretranslationConfidence, Is.GreaterThan(0.2)); IList translations = await _helperClient.TranslationEnginesClient.GetAllPretranslationsAsync( engineId, diff --git a/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs b/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs index 8dfc19e7..23c032f4 100644 --- a/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs +++ b/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs @@ -58,7 +58,7 @@ await env.Builds.InsertAsync( await env.PlatformService.BuildCompletedAsync("b0", 0, 0.0); Assert.That(env.Pretranslations.Count, Is.EqualTo(1)); Assert.That( - (await env.Builds.GetAsync(b => b.Id == "b0"))?.ExecutionData.AverageVersePretranslationConfidence, + (await env.Builds.GetAsync(b => b.Id == "b0"))?.ExecutionData.AveragePretranslationConfidence, Is.Zero.Within(0.001) ); @@ -71,13 +71,13 @@ await env.PlatformService.InsertPretranslationsAsync( await env.PlatformService.BuildCompletedAsync("b0", 0, 0.0); Assert.That(env.Pretranslations.Count, Is.EqualTo(0)); Assert.That( - (await env.Builds.GetAsync(b => b.Id == "b0"))?.ExecutionData.AverageVersePretranslationConfidence, + (await env.Builds.GetAsync(b => b.Id == "b0"))?.ExecutionData.AveragePretranslationConfidence, Is.Zero.Within(0.001) ); } [Test] - public async Task BuildCompletedAsync_AverageVersePretranslationConfidence() + public async Task BuildCompletedAsync_AveragePretranslationConfidence() { var env = new TestEnvironment(); await env.Engines.InsertAsync( @@ -109,7 +109,7 @@ await env.PlatformService.InsertPretranslationsAsync( await env.PlatformService.BuildCompletedAsync("b0", 0, 0.0); Assert.That( - (await env.Builds.GetAsync(b => b.Id == "b0"))?.ExecutionData.AverageVersePretranslationConfidence, + (await env.Builds.GetAsync(b => b.Id == "b0"))?.ExecutionData.AveragePretranslationConfidence, Is.EqualTo(0.5).Within(0.0001) ); } From 3c78a26e113c12712af6162639fed8f7b1ebf2cb Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Tue, 28 Jul 2026 15:58:47 -0400 Subject: [PATCH 6/9] Rebuild client --- src/Serval/src/Serval.Client/Client.g.cs | 154 +++++++++++------------ 1 file changed, 77 insertions(+), 77 deletions(-) diff --git a/src/Serval/src/Serval.Client/Client.g.cs b/src/Serval/src/Serval.Client/Client.g.cs index 7a5b4669..bc21b82b 100644 --- a/src/Serval/src/Serval.Client/Client.g.cs +++ b/src/Serval/src/Serval.Client/Client.g.cs @@ -448,7 +448,7 @@ private string ConvertToString(object? value, System.Globalization.CultureInfo c var field_ = System.Reflection.IntrospectionExtensions.GetTypeInfo(value.GetType()).GetDeclaredField(name); if (field_ != null) { - var attribute = System.Reflection.CustomAttributeExtensions.GetCustomAttribute(field_, typeof(System.Runtime.Serialization.EnumMemberAttribute)) + var attribute = System.Reflection.CustomAttributeExtensions.GetCustomAttribute(field_, typeof(System.Runtime.Serialization.EnumMemberAttribute)) as System.Runtime.Serialization.EnumMemberAttribute; if (attribute != null) { @@ -460,7 +460,7 @@ private string ConvertToString(object? value, System.Globalization.CultureInfo c return converted == null ? string.Empty : converted; } } - else if (value is bool) + else if (value is bool) { return System.Convert.ToString((bool)value, cultureInfo).ToLowerInvariant(); } @@ -498,7 +498,7 @@ public partial interface IDataFilesClient /// /// /// Sample request: - ///
+ ///
///
POST /files ///
{ ///
"format": "text", @@ -629,7 +629,7 @@ public string BaseUrl /// /// /// Sample request: - ///
+ ///
///
POST /files ///
{ ///
"format": "text", @@ -1394,7 +1394,7 @@ private string ConvertToString(object? value, System.Globalization.CultureInfo c var field_ = System.Reflection.IntrospectionExtensions.GetTypeInfo(value.GetType()).GetDeclaredField(name); if (field_ != null) { - var attribute = System.Reflection.CustomAttributeExtensions.GetCustomAttribute(field_, typeof(System.Runtime.Serialization.EnumMemberAttribute)) + var attribute = System.Reflection.CustomAttributeExtensions.GetCustomAttribute(field_, typeof(System.Runtime.Serialization.EnumMemberAttribute)) as System.Runtime.Serialization.EnumMemberAttribute; if (attribute != null) { @@ -1406,7 +1406,7 @@ private string ConvertToString(object? value, System.Globalization.CultureInfo c return converted == null ? string.Empty : converted; } } - else if (value is bool) + else if (value is bool) { return System.Convert.ToString((bool)value, cultureInfo).ToLowerInvariant(); } @@ -2140,7 +2140,7 @@ private string ConvertToString(object? value, System.Globalization.CultureInfo c var field_ = System.Reflection.IntrospectionExtensions.GetTypeInfo(value.GetType()).GetDeclaredField(name); if (field_ != null) { - var attribute = System.Reflection.CustomAttributeExtensions.GetCustomAttribute(field_, typeof(System.Runtime.Serialization.EnumMemberAttribute)) + var attribute = System.Reflection.CustomAttributeExtensions.GetCustomAttribute(field_, typeof(System.Runtime.Serialization.EnumMemberAttribute)) as System.Runtime.Serialization.EnumMemberAttribute; if (attribute != null) { @@ -2152,7 +2152,7 @@ private string ConvertToString(object? value, System.Globalization.CultureInfo c return converted == null ? string.Empty : converted; } } - else if (value is bool) + else if (value is bool) { return System.Convert.ToString((bool)value, cultureInfo).ToLowerInvariant(); } @@ -2580,7 +2580,7 @@ private string ConvertToString(object? value, System.Globalization.CultureInfo c var field_ = System.Reflection.IntrospectionExtensions.GetTypeInfo(value.GetType()).GetDeclaredField(name); if (field_ != null) { - var attribute = System.Reflection.CustomAttributeExtensions.GetCustomAttribute(field_, typeof(System.Runtime.Serialization.EnumMemberAttribute)) + var attribute = System.Reflection.CustomAttributeExtensions.GetCustomAttribute(field_, typeof(System.Runtime.Serialization.EnumMemberAttribute)) as System.Runtime.Serialization.EnumMemberAttribute; if (attribute != null) { @@ -2592,7 +2592,7 @@ private string ConvertToString(object? value, System.Globalization.CultureInfo c return converted == null ? string.Empty : converted; } } - else if (value is bool) + else if (value is bool) { return System.Convert.ToString((bool)value, cultureInfo).ToLowerInvariant(); } @@ -2715,14 +2715,14 @@ public partial interface ITranslationEnginesClient ///
### nmt ///
The Neural Machine Translation engine is primarily used for pretranslations. It is fine-tuned from Meta's NLLB-200. Valid IETF language tags provided to Serval will be converted to [NLLB-200 codes](https://github.com/facebookresearch/flores/tree/main/flores200#languages-in-flores-200). See more about language tag resolution [here](https://github.com/sillsdev/serval/wiki/FLORES%E2%80%90200-Language-Code-Resolution-for-NMT-Engine). ///
* **`isModelPersisted`**: (default to `false`) Whether the model can be downloaded by the client after it has been successfully built. - ///
+ ///
///
If you use a language among NLLB's supported languages, Serval will utilize everything the NLLB-200 model already knows about that language when translating. If the language you are working with is not among NLLB's supported languages, the language code will have no effect. - ///
+ ///
///
Typical endpoints: pretranslate ///
### echo ///
The echo engine has full coverage of all nmt and smt-transfer endpoints. Endpoints like create and build return empty responses. Endpoints like translate and get-word-graph echo the sent content back to the user in a format that mocks nmt or smt-transfer. For example, translating a segment "test" with the echo engine would yield a translation response with translation "test". This engine is useful for debugging and testing purposes. ///
## Sample request: - ///
+ ///
///
{ ///
"name": "myTeam:myProject:myEngine", ///
"sourceLanguage": "el", @@ -2810,7 +2810,7 @@ public partial interface ITranslationEnginesClient /// /// /// ## Sample request: - ///
+ ///
///
{ ///
"sourceLanguage": "en", ///
"targetLanguage": "en" @@ -2876,30 +2876,30 @@ public partial interface ITranslationEnginesClient ///
Specifying a corpus: ///
* A (legacy) corpus is selected by specifying `corpusId` and a parallel corpus is selected by specifying `parallelCorpusId`. ///
* A parallel corpus can be further filtered by specifying particular corpusIds in `sourceFilters` or `targetFilters`. - ///
+ ///
///
Filtering by text id or chapter: ///
* Paratext projects can be filtered by [book using the `textIds`](https://github.com/sillsdev/libpalaso/blob/master/SIL.Scripture/Canon.cs). ///
* Filters can also be supplied via the `scriptureRange` parameter as ranges of biblical text. See [here](https://github.com/sillsdev/serval/wiki/Filtering-Paratext-Project-Data-with-a-Scripture-Range). ///
* All Paratext project filtering follows original versification. See [here](https://github.com/sillsdev/serval/wiki/Versification-in-Serval) for more information. - ///
+ ///
///
Filter - train on all or none ///
* If `trainOn` or `pretranslate` is not provided, all corpora will be used for training or pretranslation respectively ///
* If a corpus is selected for training or pretranslation and neither `scriptureRange` nor `textIds` is defined, all of the selected corpus will be used. ///
* If a corpus is selected for training or pretranslation and an empty `scriptureRange` or `textIds` is defined, none of the selected corpus will be used. ///
* If a corpus is selected for training or pretranslation but no further filters are provided, all selected corpora will be used for training or pretranslation respectively. - ///
+ ///
///
Specify the corpora and text ids/scripture ranges within those corpora to pretranslate. When a corpus is selected for pretranslation, ///
the following text will be pretranslated: ///
* Text segments that are in the source but do not exist in the target. ///
* Text segments that are in the source and the target, but because of `trainOn` filtering, have not been trained on. ///
If the engine does not support pretranslation, these fields have no effect. ///
Pretranslating uses the same filtering as training. - ///
+ ///
///
The `options` parameter of the build config provides the ability to pass build configuration parameters as a JSON object. ///
See [nmt job settings documentation](https://github.com/sillsdev/serval/wiki/NMT-Build-Options) about configuring job parameters. ///
See [smt-transfer job settings documentation](https://github.com/sillsdev/serval/wiki/SMT-Transfer-Build-Options) about configuring job parameters. ///
See [keyterms parsing documentation](https://github.com/sillsdev/serval/wiki/Paratext-Key-Terms-Parsing) on how to use keyterms for training. - ///
+ ///
///
Note that when using a parallel corpus: ///
* If, within a single parallel corpus, multiple source corpora have data for the same text ids (for text files or Paratext Projects) or books (for Paratext Projects only using the scripture range), those sources will be mixed where they overlap by randomly choosing from each source per line/verse. ///
* If, within a single parallel corpus, multiple target corpora have data for the same text ids (for text files or Paratext Projects) or books (for Paratext Projects only using the scripture range), only the first of the targets that includes that text id/book will be used for that text id/book. @@ -2921,7 +2921,7 @@ public partial interface ITranslationEnginesClient ///
* The references defined in the source file per line, if any. ///
* An auto-generated reference of `[textId]:[lineNumber]`, 1 indexed. ///
* **`translation`**: the text of the pretranslation - ///
+ ///
///
Pretranslations can be filtered by text id if provided. ///
Only pretranslations for the most recent successful build of the engine are returned. /// @@ -2954,7 +2954,7 @@ public partial interface ITranslationEnginesClient ///
* The references defined in the source file per line, if any. ///
* An auto-generated reference of `[textId]:[lineNumber]`, 1 indexed. ///
* **`translation`**: the text of the pretranslation - ///
+ ///
///
Pretranslations can be filtered by text id if provided. ///
Only pretranslations for the most recent successful build of the engine are returned. /// @@ -2996,25 +2996,25 @@ public partial interface ITranslationEnginesClient ///
* `PreferPretranslated`: The existing and pretranslated texts are merged into the USFM, preferring pretranslated text. ///
* `OnlyExisting`: Return the existing target USFM file with no modifications (except updating the USFM id if needed). ///
* `OnlyPretranslated`: Only the pretranslated text is returned; all existing text in the target USFM is removed. - ///
+ ///
///
The source or target book can be used as the USFM template for the pretranslated text. The template can be controlled by the `template` parameter: ///
* `Auto`: The target book is used as the template if it exists; otherwise, the source book is used. **This is the default**. ///
* `Source`: The source book is used as the template. ///
* `Target`: The target book is used as the template. - ///
+ ///
///
The intra-segment USFM markers are handled in the following way: ///
* Each verse and non-verse text segment is stripped of all intra-segment USFM. ///
* Reference (\r) and remark (\rem) markers are not translated but carried through from the source to the target. - ///
+ ///
///
Preserving or stripping different types of USFM markers can be controlled by the `paragraph-marker-behavior`, `embed-behavior`, and `style-marker-behavior` parameters. ///
* `Preserve`: The USFM markers (or the entire embed) are preserved and placed at the end of the verse. **This is the default for paragraph markers and embeds**. ///
* `PreservePosition`: The USFM markers (or the entire embed) are placed in approximately the right location within the verse. **This option is only available for paragraph markers. Quality of placement may differ from language to language. Only works when `template` is set to `Source`**. ///
* `Strip`: The USFM markers (or the entire embed) are removed. **This is the default for style markers**. - ///
+ ///
///
Quote normalization behavior is controlled by the `quote-normalization-behavior` parameter options: ///
* `Normalized`: The quotes in the pretranslated USFM are normalized quotes (typically straight quotes: ', ") in the style of the source data. **This is the default**. ///
* `Denormalized`: The quotes in the pretranslated USFM are denormalized into the style of the target data. Quote denormalization may not be successful in all contexts. A remark will be added to the USFM listing the chapters that were successfully denormalized. - ///
+ ///
///
Only pretranslations for the most recent successful build of the engine are returned. ///
The USFM parsing and marker types used are defined here: [this wiki](https://github.com/sillsdev/serval/wiki/USFM-Parsing-and-Translation). /// @@ -3043,7 +3043,7 @@ public partial interface ITranslationEnginesClient ///
* The references defined in the source file per line, if any. ///
* An auto-generated reference of `[textId]:[lineNumber]`, 1 indexed. ///
* **`translation`**: the text of the pretranslation - ///
+ ///
///
Only pretranslations for the most recent successful build of the engine are returned. /// /// The translation engine id @@ -3074,10 +3074,10 @@ public partial interface ITranslationEnginesClient /// /// If an nmt build was successful and `isModelPersisted` is `true` for the engine, ///
then the model from the most recent successful build can be downloaded. - ///
+ ///
///
The endpoint will return a URL that can be used to download the model for up to 1 hour ///
after the request is made. If the URL is not used within that time, a new request will need to be made. - ///
+ ///
///
The download itself is created by g-zipping together the folder containing the fine tuned model ///
with all necessary supporting files. This zipped folder is then named by the pattern: ///
* <engine_id>_<model_revision>.tar.gz @@ -3097,25 +3097,25 @@ public partial interface ITranslationEnginesClient ///
* `PreferPretranslated`: The existing and pretranslated texts are merged into the USFM, preferring pretranslated text. ///
* `OnlyExisting`: Return the existing target USFM file with no modifications (except updating the USFM id if needed). ///
* `OnlyPretranslated`: Only the pretranslated text is returned; all existing text in the target USFM is removed. - ///
+ ///
///
The source or target book can be used as the USFM template for the pretranslated text. The template can be controlled by the `template` parameter: ///
* `Auto`: The target book is used as the template if it exists; otherwise, the source book is used. **This is the default**. ///
* `Source`: The source book is used as the template. ///
* `Target`: The target book is used as the template. - ///
+ ///
///
The intra-segment USFM markers are handled in the following way: ///
* Each verse and non-verse text segment is stripped of all intra-segment USFM. ///
* Reference (\r) and remark (\rem) markers are not translated but carried through from the source to the target. - ///
+ ///
///
Preserving or stripping different types of USFM markers can be controlled by the `paragraph-marker-behavior`, `embed-behavior`, and `style-marker-behavior` parameters. ///
* `Preserve`: The USFM markers (or the entire embed) are preserved and placed at the end of the verse. **This is the default for paragraph markers and embeds**. ///
* `PreservePosition`: The USFM markers (or the entire embed) are placed in approximately the right location within the verse. **This option is only available for paragraph markers. Quality of placement may differ from language to language. Only works when `template` is set to `Source`**. ///
* `Strip`: The USFM markers (or the entire embed) are removed. **This is the default for style markers**. - ///
+ ///
///
Quote normalization behavior is controlled by the `quote-normalization-behavior` parameter options: ///
* `Normalized`: The quotes in the pretranslated USFM are normalized quotes (typically straight quotes: ', ") in the style of the source data. **This is the default**. ///
* `Denormalized`: The quotes in the pretranslated USFM are denormalized into the style of the target data. Quote denormalization may not be successful in all contexts. A remark will be added to the USFM listing the chapters that were successfully denormalized. - ///
+ ///
///
Only pretranslations for the most recent successful build of the engine are returned. ///
The USFM parsing and marker types used are defined here: [this wiki](https://github.com/sillsdev/serval/wiki/USFM-Parsing-and-Translation). /// @@ -3143,7 +3143,7 @@ public partial interface ITranslationEnginesClient ///
* The references defined in the source file per line, if any. ///
* An auto-generated reference of `[textId]:[lineNumber]`, 1 indexed. ///
* **`translation`**: the text of the pretranslation - ///
+ ///
///
Only pretranslations for the most recent successful build of the engine are returned. /// /// The translation engine id @@ -3854,14 +3854,14 @@ public string BaseUrl ///
### nmt ///
The Neural Machine Translation engine is primarily used for pretranslations. It is fine-tuned from Meta's NLLB-200. Valid IETF language tags provided to Serval will be converted to [NLLB-200 codes](https://github.com/facebookresearch/flores/tree/main/flores200#languages-in-flores-200). See more about language tag resolution [here](https://github.com/sillsdev/serval/wiki/FLORES%E2%80%90200-Language-Code-Resolution-for-NMT-Engine). ///
* **`isModelPersisted`**: (default to `false`) Whether the model can be downloaded by the client after it has been successfully built. - ///
+ ///
///
If you use a language among NLLB's supported languages, Serval will utilize everything the NLLB-200 model already knows about that language when translating. If the language you are working with is not among NLLB's supported languages, the language code will have no effect. - ///
+ ///
///
Typical endpoints: pretranslate ///
### echo ///
The echo engine has full coverage of all nmt and smt-transfer endpoints. Endpoints like create and build return empty responses. Endpoints like translate and get-word-graph echo the sent content back to the user in a format that mocks nmt or smt-transfer. For example, translating a segment "test" with the echo engine would yield a translation response with translation "test". This engine is useful for debugging and testing purposes. ///
## Sample request: - ///
+ ///
///
{ ///
"name": "myTeam:myProject:myEngine", ///
"sourceLanguage": "el", @@ -4622,7 +4622,7 @@ public string BaseUrl /// /// /// ## Sample request: - ///
+ ///
///
{ ///
"sourceLanguage": "en", ///
"targetLanguage": "en" @@ -5182,30 +5182,30 @@ public string BaseUrl ///
Specifying a corpus: ///
* A (legacy) corpus is selected by specifying `corpusId` and a parallel corpus is selected by specifying `parallelCorpusId`. ///
* A parallel corpus can be further filtered by specifying particular corpusIds in `sourceFilters` or `targetFilters`. - ///
+ ///
///
Filtering by text id or chapter: ///
* Paratext projects can be filtered by [book using the `textIds`](https://github.com/sillsdev/libpalaso/blob/master/SIL.Scripture/Canon.cs). ///
* Filters can also be supplied via the `scriptureRange` parameter as ranges of biblical text. See [here](https://github.com/sillsdev/serval/wiki/Filtering-Paratext-Project-Data-with-a-Scripture-Range). ///
* All Paratext project filtering follows original versification. See [here](https://github.com/sillsdev/serval/wiki/Versification-in-Serval) for more information. - ///
+ ///
///
Filter - train on all or none ///
* If `trainOn` or `pretranslate` is not provided, all corpora will be used for training or pretranslation respectively ///
* If a corpus is selected for training or pretranslation and neither `scriptureRange` nor `textIds` is defined, all of the selected corpus will be used. ///
* If a corpus is selected for training or pretranslation and an empty `scriptureRange` or `textIds` is defined, none of the selected corpus will be used. ///
* If a corpus is selected for training or pretranslation but no further filters are provided, all selected corpora will be used for training or pretranslation respectively. - ///
+ ///
///
Specify the corpora and text ids/scripture ranges within those corpora to pretranslate. When a corpus is selected for pretranslation, ///
the following text will be pretranslated: ///
* Text segments that are in the source but do not exist in the target. ///
* Text segments that are in the source and the target, but because of `trainOn` filtering, have not been trained on. ///
If the engine does not support pretranslation, these fields have no effect. ///
Pretranslating uses the same filtering as training. - ///
+ ///
///
The `options` parameter of the build config provides the ability to pass build configuration parameters as a JSON object. ///
See [nmt job settings documentation](https://github.com/sillsdev/serval/wiki/NMT-Build-Options) about configuring job parameters. ///
See [smt-transfer job settings documentation](https://github.com/sillsdev/serval/wiki/SMT-Transfer-Build-Options) about configuring job parameters. ///
See [keyterms parsing documentation](https://github.com/sillsdev/serval/wiki/Paratext-Key-Terms-Parsing) on how to use keyterms for training. - ///
+ ///
///
Note that when using a parallel corpus: ///
* If, within a single parallel corpus, multiple source corpora have data for the same text ids (for text files or Paratext Projects) or books (for Paratext Projects only using the scripture range), those sources will be mixed where they overlap by randomly choosing from each source per line/verse. ///
* If, within a single parallel corpus, multiple target corpora have data for the same text ids (for text files or Paratext Projects) or books (for Paratext Projects only using the scripture range), only the first of the targets that includes that text id/book will be used for that text id/book. @@ -5341,7 +5341,7 @@ public string BaseUrl ///
* The references defined in the source file per line, if any. ///
* An auto-generated reference of `[textId]:[lineNumber]`, 1 indexed. ///
* **`translation`**: the text of the pretranslation - ///
+ ///
///
Pretranslations can be filtered by text id if provided. ///
Only pretranslations for the most recent successful build of the engine are returned. /// @@ -5592,7 +5592,7 @@ public string BaseUrl ///
* The references defined in the source file per line, if any. ///
* An auto-generated reference of `[textId]:[lineNumber]`, 1 indexed. ///
* **`translation`**: the text of the pretranslation - ///
+ ///
///
Pretranslations can be filtered by text id if provided. ///
Only pretranslations for the most recent successful build of the engine are returned. /// @@ -5857,25 +5857,25 @@ public string BaseUrl ///
* `PreferPretranslated`: The existing and pretranslated texts are merged into the USFM, preferring pretranslated text. ///
* `OnlyExisting`: Return the existing target USFM file with no modifications (except updating the USFM id if needed). ///
* `OnlyPretranslated`: Only the pretranslated text is returned; all existing text in the target USFM is removed. - ///
+ ///
///
The source or target book can be used as the USFM template for the pretranslated text. The template can be controlled by the `template` parameter: ///
* `Auto`: The target book is used as the template if it exists; otherwise, the source book is used. **This is the default**. ///
* `Source`: The source book is used as the template. ///
* `Target`: The target book is used as the template. - ///
+ ///
///
The intra-segment USFM markers are handled in the following way: ///
* Each verse and non-verse text segment is stripped of all intra-segment USFM. ///
* Reference (\r) and remark (\rem) markers are not translated but carried through from the source to the target. - ///
+ ///
///
Preserving or stripping different types of USFM markers can be controlled by the `paragraph-marker-behavior`, `embed-behavior`, and `style-marker-behavior` parameters. ///
* `Preserve`: The USFM markers (or the entire embed) are preserved and placed at the end of the verse. **This is the default for paragraph markers and embeds**. ///
* `PreservePosition`: The USFM markers (or the entire embed) are placed in approximately the right location within the verse. **This option is only available for paragraph markers. Quality of placement may differ from language to language. Only works when `template` is set to `Source`**. ///
* `Strip`: The USFM markers (or the entire embed) are removed. **This is the default for style markers**. - ///
+ ///
///
Quote normalization behavior is controlled by the `quote-normalization-behavior` parameter options: ///
* `Normalized`: The quotes in the pretranslated USFM are normalized quotes (typically straight quotes: ', ") in the style of the source data. **This is the default**. ///
* `Denormalized`: The quotes in the pretranslated USFM are denormalized into the style of the target data. Quote denormalization may not be successful in all contexts. A remark will be added to the USFM listing the chapters that were successfully denormalized. - ///
+ ///
///
Only pretranslations for the most recent successful build of the engine are returned. ///
The USFM parsing and marker types used are defined here: [this wiki](https://github.com/sillsdev/serval/wiki/USFM-Parsing-and-Translation). /// @@ -6050,7 +6050,7 @@ public string BaseUrl ///
* The references defined in the source file per line, if any. ///
* An auto-generated reference of `[textId]:[lineNumber]`, 1 indexed. ///
* **`translation`**: the text of the pretranslation - ///
+ ///
///
Only pretranslations for the most recent successful build of the engine are returned. /// /// The translation engine id @@ -6310,10 +6310,10 @@ public string BaseUrl /// /// If an nmt build was successful and `isModelPersisted` is `true` for the engine, ///
then the model from the most recent successful build can be downloaded. - ///
+ ///
///
The endpoint will return a URL that can be used to download the model for up to 1 hour ///
after the request is made. If the URL is not used within that time, a new request will need to be made. - ///
+ ///
///
The download itself is created by g-zipping together the folder containing the fine tuned model ///
with all necessary supporting files. This zipped folder is then named by the pattern: ///
* <engine_id>_<model_revision>.tar.gz @@ -6434,25 +6434,25 @@ public string BaseUrl ///
* `PreferPretranslated`: The existing and pretranslated texts are merged into the USFM, preferring pretranslated text. ///
* `OnlyExisting`: Return the existing target USFM file with no modifications (except updating the USFM id if needed). ///
* `OnlyPretranslated`: Only the pretranslated text is returned; all existing text in the target USFM is removed. - ///
+ ///
///
The source or target book can be used as the USFM template for the pretranslated text. The template can be controlled by the `template` parameter: ///
* `Auto`: The target book is used as the template if it exists; otherwise, the source book is used. **This is the default**. ///
* `Source`: The source book is used as the template. ///
* `Target`: The target book is used as the template. - ///
+ ///
///
The intra-segment USFM markers are handled in the following way: ///
* Each verse and non-verse text segment is stripped of all intra-segment USFM. ///
* Reference (\r) and remark (\rem) markers are not translated but carried through from the source to the target. - ///
+ ///
///
Preserving or stripping different types of USFM markers can be controlled by the `paragraph-marker-behavior`, `embed-behavior`, and `style-marker-behavior` parameters. ///
* `Preserve`: The USFM markers (or the entire embed) are preserved and placed at the end of the verse. **This is the default for paragraph markers and embeds**. ///
* `PreservePosition`: The USFM markers (or the entire embed) are placed in approximately the right location within the verse. **This option is only available for paragraph markers. Quality of placement may differ from language to language. Only works when `template` is set to `Source`**. ///
* `Strip`: The USFM markers (or the entire embed) are removed. **This is the default for style markers**. - ///
+ ///
///
Quote normalization behavior is controlled by the `quote-normalization-behavior` parameter options: ///
* `Normalized`: The quotes in the pretranslated USFM are normalized quotes (typically straight quotes: ', ") in the style of the source data. **This is the default**. ///
* `Denormalized`: The quotes in the pretranslated USFM are denormalized into the style of the target data. Quote denormalization may not be successful in all contexts. A remark will be added to the USFM listing the chapters that were successfully denormalized. - ///
+ ///
///
Only pretranslations for the most recent successful build of the engine are returned. ///
The USFM parsing and marker types used are defined here: [this wiki](https://github.com/sillsdev/serval/wiki/USFM-Parsing-and-Translation). /// @@ -6626,7 +6626,7 @@ public string BaseUrl ///
* The references defined in the source file per line, if any. ///
* An auto-generated reference of `[textId]:[lineNumber]`, 1 indexed. ///
* **`translation`**: the text of the pretranslation - ///
+ ///
///
Only pretranslations for the most recent successful build of the engine are returned. /// /// The translation engine id @@ -7362,7 +7362,7 @@ private string ConvertToString(object? value, System.Globalization.CultureInfo c var field_ = System.Reflection.IntrospectionExtensions.GetTypeInfo(value.GetType()).GetDeclaredField(name); if (field_ != null) { - var attribute = System.Reflection.CustomAttributeExtensions.GetCustomAttribute(field_, typeof(System.Runtime.Serialization.EnumMemberAttribute)) + var attribute = System.Reflection.CustomAttributeExtensions.GetCustomAttribute(field_, typeof(System.Runtime.Serialization.EnumMemberAttribute)) as System.Runtime.Serialization.EnumMemberAttribute; if (attribute != null) { @@ -7374,7 +7374,7 @@ private string ConvertToString(object? value, System.Globalization.CultureInfo c return converted == null ? string.Empty : converted; } } - else if (value is bool) + else if (value is bool) { return System.Convert.ToString((bool)value, cultureInfo).ToLowerInvariant(); } @@ -7774,7 +7774,7 @@ private string ConvertToString(object? value, System.Globalization.CultureInfo c var field_ = System.Reflection.IntrospectionExtensions.GetTypeInfo(value.GetType()).GetDeclaredField(name); if (field_ != null) { - var attribute = System.Reflection.CustomAttributeExtensions.GetCustomAttribute(field_, typeof(System.Runtime.Serialization.EnumMemberAttribute)) + var attribute = System.Reflection.CustomAttributeExtensions.GetCustomAttribute(field_, typeof(System.Runtime.Serialization.EnumMemberAttribute)) as System.Runtime.Serialization.EnumMemberAttribute; if (attribute != null) { @@ -7786,7 +7786,7 @@ private string ConvertToString(object? value, System.Globalization.CultureInfo c return converted == null ? string.Empty : converted; } } - else if (value is bool) + else if (value is bool) { return System.Convert.ToString((bool)value, cultureInfo).ToLowerInvariant(); } @@ -8067,7 +8067,7 @@ private string ConvertToString(object? value, System.Globalization.CultureInfo c var field_ = System.Reflection.IntrospectionExtensions.GetTypeInfo(value.GetType()).GetDeclaredField(name); if (field_ != null) { - var attribute = System.Reflection.CustomAttributeExtensions.GetCustomAttribute(field_, typeof(System.Runtime.Serialization.EnumMemberAttribute)) + var attribute = System.Reflection.CustomAttributeExtensions.GetCustomAttribute(field_, typeof(System.Runtime.Serialization.EnumMemberAttribute)) as System.Runtime.Serialization.EnumMemberAttribute; if (attribute != null) { @@ -8079,7 +8079,7 @@ private string ConvertToString(object? value, System.Globalization.CultureInfo c return converted == null ? string.Empty : converted; } } - else if (value is bool) + else if (value is bool) { return System.Convert.ToString((bool)value, cultureInfo).ToLowerInvariant(); } @@ -8172,7 +8172,7 @@ public partial interface IWordAlignmentEnginesClient ///
The echo-word-alignment engine has full coverage of all endpoints. Endpoints like create and build return empty responses. ///
Endpoints like align echo the sent content back to the user in the proper format. This engine is useful for debugging and testing purposes. ///
## Sample request: - ///
+ ///
///
{ ///
"name": "myTeam:myProject:myEngine", ///
"sourceLanguage": "el", @@ -8266,10 +8266,10 @@ public partial interface IWordAlignmentEnginesClient ///
Paratext projects can be filtered by [book using the `textIds`](https://github.com/sillsdev/libpalaso/blob/master/SIL.Scripture/Canon.cs). ///
Filters can also be supplied via `scriptureRange` parameter as ranges of biblical text. See [here](https://github.com/sillsdev/serval/wiki/Filtering-Paratext-Project-Data-with-a-Scripture-Range) ///
All Paratext project filtering follows original versification. See [here](https://github.com/sillsdev/serval/wiki/Versification-in-Serval) for more information. - ///
+ ///
///
Specify the corpora or text ids to word align on. ///
When a corpus or text id is selected for word align on, only text segments that are in both the source and the target will be aligned. - ///
+ ///
///
The `options` parameter of the build config provides the ability to pass build configuration parameters as a JSON object. ///
See [statistical alignment job settings documentation](https://github.com/sillsdev/serval/wiki/Statistical-Alignment-Build-Options) about configuring job parameters. /// @@ -8292,7 +8292,7 @@ public partial interface IWordAlignmentEnginesClient ///
* **`sourceTokens`**: the tokenized source segment ///
* **`targetTokens`**: the tokenized target segment ///
* **`alignment`**: a list of aligned word pairs with associated scores - ///
+ ///
///
Word alignments can be filtered by text id if provided. ///
Only word alignments for the most recent successful build of the engine are returned. /// @@ -8879,7 +8879,7 @@ public string BaseUrl ///
The echo-word-alignment engine has full coverage of all endpoints. Endpoints like create and build return empty responses. ///
Endpoints like align echo the sent content back to the user in the proper format. This engine is useful for debugging and testing purposes. ///
## Sample request: - ///
+ ///
///
{ ///
"name": "myTeam:myProject:myEngine", ///
"sourceLanguage": "el", @@ -9735,10 +9735,10 @@ public string BaseUrl ///
Paratext projects can be filtered by [book using the `textIds`](https://github.com/sillsdev/libpalaso/blob/master/SIL.Scripture/Canon.cs). ///
Filters can also be supplied via `scriptureRange` parameter as ranges of biblical text. See [here](https://github.com/sillsdev/serval/wiki/Filtering-Paratext-Project-Data-with-a-Scripture-Range) ///
All Paratext project filtering follows original versification. See [here](https://github.com/sillsdev/serval/wiki/Versification-in-Serval) for more information. - ///
+ ///
///
Specify the corpora or text ids to word align on. ///
When a corpus or text id is selected for word align on, only text segments that are in both the source and the target will be aligned. - ///
+ ///
///
The `options` parameter of the build config provides the ability to pass build configuration parameters as a JSON object. ///
See [statistical alignment job settings documentation](https://github.com/sillsdev/serval/wiki/Statistical-Alignment-Build-Options) about configuring job parameters. /// @@ -9875,7 +9875,7 @@ public string BaseUrl ///
* **`sourceTokens`**: the tokenized source segment ///
* **`targetTokens`**: the tokenized target segment ///
* **`alignment`**: a list of aligned word pairs with associated scores - ///
+ ///
///
Word alignments can be filtered by text id if provided. ///
Only word alignments for the most recent successful build of the engine are returned. /// @@ -10358,7 +10358,7 @@ private string ConvertToString(object? value, System.Globalization.CultureInfo c var field_ = System.Reflection.IntrospectionExtensions.GetTypeInfo(value.GetType()).GetDeclaredField(name); if (field_ != null) { - var attribute = System.Reflection.CustomAttributeExtensions.GetCustomAttribute(field_, typeof(System.Runtime.Serialization.EnumMemberAttribute)) + var attribute = System.Reflection.CustomAttributeExtensions.GetCustomAttribute(field_, typeof(System.Runtime.Serialization.EnumMemberAttribute)) as System.Runtime.Serialization.EnumMemberAttribute; if (attribute != null) { @@ -10370,7 +10370,7 @@ private string ConvertToString(object? value, System.Globalization.CultureInfo c return converted == null ? string.Empty : converted; } } - else if (value is bool) + else if (value is bool) { return System.Convert.ToString((bool)value, cultureInfo).ToLowerInvariant(); } @@ -10960,7 +10960,7 @@ public partial class ExecutionData [Newtonsoft.Json.JsonProperty("resolvedTargetLanguage", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] public string? ResolvedTargetLanguage { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("averageVersePretranslationConfidence", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + [Newtonsoft.Json.JsonProperty("averagePretranslationConfidence", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] public double? AveragePretranslationConfidence { get; set; } = default!; } From 9ee00307afde996f117f876bd1a974b874bf86a5 Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Tue, 28 Jul 2026 16:28:40 -0400 Subject: [PATCH 7/9] Add diagnostics to word alignment as well; make warnings obsolete --- .../EchoWordAlignmentPreprocessBuildJob.cs | 20 ++++++++ .../ServalTranslationPlatformService.cs | 4 +- .../ServalWordAlignmentPlatformService.cs | 12 +++++ src/Serval/src/Serval.Client/Client.g.cs | 46 +++++++++++++++++++ .../DiagnosticContract.cs | 17 +++++++ .../ExecutionDataContract.cs | 1 + .../Dtos/DiagnosticDto.cs | 17 +++++++ .../Dtos/WordAlignmentExecutionDataDto.cs | 1 + .../Serval.WordAlignment/Models/Diagnostic.cs | 17 +++++++ .../Models/ExecutionData.cs | 1 + .../Services/DtoMapper.cs | 13 ++++++ 11 files changed, 147 insertions(+), 2 deletions(-) create mode 100644 src/Serval/src/Serval.WordAlignment.Contracts/DiagnosticContract.cs create mode 100644 src/Serval/src/Serval.WordAlignment/Dtos/DiagnosticDto.cs create mode 100644 src/Serval/src/Serval.WordAlignment/Models/Diagnostic.cs diff --git a/src/Machine/src/Serval.Machine.Shared/Services/EchoWordAlignmentPreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/EchoWordAlignmentPreprocessBuildJob.cs index 6f4e64ef..66e26b7a 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/EchoWordAlignmentPreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/EchoWordAlignmentPreprocessBuildJob.cs @@ -103,6 +103,25 @@ CancellationToken cancellationToken parallelCorpora ); + int maxWarnings = BuildJobOptions.MaxWarnings; + if (warnings.Count > maxWarnings) + { + string tooManyWarningsWarning = + $"There were {warnings.Count} warnings. Only the first {maxWarnings} are shown."; + warnings = [tooManyWarningsWarning, .. warnings.Take(maxWarnings)]; + } + + IReadOnlyList diagnostics = GetDiagnostics( + stats.TrainCount, + stats.InferenceCount, + sourceLanguageTag, + targetLanguageTag, + true, + true, + isNonPersistedTranslationEngine, + parallelCorpora + ); + // Log summary of build data var buildPreprocessSummary = new JsonObject { @@ -125,6 +144,7 @@ CancellationToken cancellationToken IsInferenceFilteredByChapter = stats.IsInferenceFilteredByChapter, IsTrainFilteredByChapter = stats.IsTrainFilteredByChapter, Warnings = warnings, + Diagnostics = diagnostics, EngineSourceLanguageTag = sourceLanguageTag, EngineTargetLanguageTag = targetLanguageTag, }; diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationPlatformService.cs b/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationPlatformService.cs index f134e1ee..f27999b9 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationPlatformService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationPlatformService.cs @@ -106,12 +106,12 @@ public Task UpdateBuildExecutionDataAsync( IsPretranslateFilteredByChapter = executionData.IsInferenceFilteredByChapter, Warnings = executionData.Warnings, Diagnostics = executionData - .Diagnostics?.Select(d => new DiagnosticContract + .Diagnostics?.Select(d => new Translation.Contracts.DiagnosticContract { Code = d.Code, Category = d.Category, Message = d.Message, - Severity = (DiagnosticSeverity)d.Severity, + Severity = (Translation.Contracts.DiagnosticSeverity)d.Severity, Data = d.Data, }) .ToList(), diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ServalWordAlignmentPlatformService.cs b/src/Machine/src/Serval.Machine.Shared/Services/ServalWordAlignmentPlatformService.cs index 45eb75b1..45f27268 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/ServalWordAlignmentPlatformService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/ServalWordAlignmentPlatformService.cs @@ -104,6 +104,18 @@ public Task UpdateBuildExecutionDataAsync( TrainVerseCount = executionData.TrainVerseCount, WordAlignVerseCount = executionData.InferenceVerseCount, Warnings = executionData.Warnings, + Diagnostics = + executionData + .Diagnostics?.Select(d => new WordAlignment.Contracts.DiagnosticContract + { + Code = d.Code, + Category = d.Category, + Message = d.Message, + Severity = (WordAlignment.Contracts.DiagnosticSeverity)d.Severity, + Data = d.Data, + }) + .ToList() + ?? [], EngineSourceLanguageTag = executionData.EngineSourceLanguageTag, EngineTargetLanguageTag = executionData.EngineTargetLanguageTag, }, diff --git a/src/Serval/src/Serval.Client/Client.g.cs b/src/Serval/src/Serval.Client/Client.g.cs index bc21b82b..3b19c43b 100644 --- a/src/Serval/src/Serval.Client/Client.g.cs +++ b/src/Serval/src/Serval.Client/Client.g.cs @@ -11742,6 +11742,10 @@ public partial class WordAlignmentExecutionData [System.ComponentModel.DataAnnotations.Required] public System.Collections.Generic.IList Warnings { get; set; } = new System.Collections.ObjectModel.Collection(); + [Newtonsoft.Json.JsonProperty("diagnostics", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public System.Collections.Generic.IList Diagnostics { get; set; } = new System.Collections.ObjectModel.Collection(); + [Newtonsoft.Json.JsonProperty("engineSourceLanguageTag", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] public string? EngineSourceLanguageTag { get; set; } = default!; @@ -11750,6 +11754,48 @@ public partial class WordAlignmentExecutionData } + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.7.1.0 (NJsonSchema v11.6.1.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class Diagnostic2 + { + + [Newtonsoft.Json.JsonProperty("code", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Code { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("category", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Category { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("message", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Message { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("severity", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + [Newtonsoft.Json.JsonConverter(typeof(Newtonsoft.Json.Converters.StringEnumConverter))] + public DiagnosticSeverity2 Severity { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("data", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public System.Collections.Generic.IDictionary Data { get; set; } = new System.Collections.Generic.Dictionary(); + + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.7.1.0 (NJsonSchema v11.6.1.0 (Newtonsoft.Json v13.0.0.0))")] + public enum DiagnosticSeverity2 + { + + [System.Runtime.Serialization.EnumMember(Value = @"Info")] + Info = 0, + + [System.Runtime.Serialization.EnumMember(Value = @"Warn")] + Warn = 1, + + [System.Runtime.Serialization.EnumMember(Value = @"Error")] + Error = 2, + + } + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.7.1.0 (NJsonSchema v11.6.1.0 (Newtonsoft.Json v13.0.0.0))")] public partial class WordAlignmentEngine { diff --git a/src/Serval/src/Serval.WordAlignment.Contracts/DiagnosticContract.cs b/src/Serval/src/Serval.WordAlignment.Contracts/DiagnosticContract.cs new file mode 100644 index 00000000..1c6a751a --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment.Contracts/DiagnosticContract.cs @@ -0,0 +1,17 @@ +namespace Serval.WordAlignment.Contracts; + +public record DiagnosticContract +{ + public required string Code { get; init; } + public required string Category { get; init; } + public required string Message { get; init; } + public required DiagnosticSeverity Severity { get; init; } + public required Dictionary Data { get; init; } +} + +public enum DiagnosticSeverity +{ + Info, + Warn, + Error, +} diff --git a/src/Serval/src/Serval.WordAlignment.Contracts/ExecutionDataContract.cs b/src/Serval/src/Serval.WordAlignment.Contracts/ExecutionDataContract.cs index ac0d1dc0..fd89367e 100644 --- a/src/Serval/src/Serval.WordAlignment.Contracts/ExecutionDataContract.cs +++ b/src/Serval/src/Serval.WordAlignment.Contracts/ExecutionDataContract.cs @@ -9,6 +9,7 @@ public record ExecutionDataContract public IReadOnlyDictionary>? TrainVerseCount { get; init; } public IReadOnlyDictionary>? WordAlignVerseCount { get; init; } public IReadOnlyList? Warnings { get; init; } + public IReadOnlyList? Diagnostics { get; init; } public string? EngineSourceLanguageTag { get; init; } public string? EngineTargetLanguageTag { get; init; } } diff --git a/src/Serval/src/Serval.WordAlignment/Dtos/DiagnosticDto.cs b/src/Serval/src/Serval.WordAlignment/Dtos/DiagnosticDto.cs new file mode 100644 index 00000000..adeba861 --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Dtos/DiagnosticDto.cs @@ -0,0 +1,17 @@ +namespace Serval.WordAlignment.Dtos; + +public record DiagnosticDto +{ + public required string Code { get; init; } + public required string Category { get; init; } + public required string Message { get; init; } + public required DiagnosticSeverity Severity { get; init; } + public required Dictionary Data { get; init; } +} + +public enum DiagnosticSeverity +{ + Info, + Warn, + Error, +} diff --git a/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentExecutionDataDto.cs b/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentExecutionDataDto.cs index 9ed2a182..dd9bcc39 100644 --- a/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentExecutionDataDto.cs +++ b/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentExecutionDataDto.cs @@ -9,6 +9,7 @@ public record WordAlignmentExecutionDataDto public IReadOnlyDictionary>? TrainVerseCount { get; init; } public IReadOnlyDictionary>? WordAlignVerseCount { get; init; } public IReadOnlyList Warnings { get; init; } = []; + public IReadOnlyList Diagnostics { get; init; } = []; public string? EngineSourceLanguageTag { get; init; } public string? EngineTargetLanguageTag { get; init; } } diff --git a/src/Serval/src/Serval.WordAlignment/Models/Diagnostic.cs b/src/Serval/src/Serval.WordAlignment/Models/Diagnostic.cs new file mode 100644 index 00000000..f8657779 --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Models/Diagnostic.cs @@ -0,0 +1,17 @@ +namespace Serval.WordAlignment.Models; + +public record Diagnostic +{ + public required string Code { get; init; } + public required string Category { get; init; } + public required string Message { get; init; } + public required DiagnosticSeverity Severity { get; init; } + public required Dictionary Data { get; init; } +} + +public enum DiagnosticSeverity +{ + Info, + Warn, + Error, +} diff --git a/src/Serval/src/Serval.WordAlignment/Models/ExecutionData.cs b/src/Serval/src/Serval.WordAlignment/Models/ExecutionData.cs index 824e80b2..3bd6cc28 100644 --- a/src/Serval/src/Serval.WordAlignment/Models/ExecutionData.cs +++ b/src/Serval/src/Serval.WordAlignment/Models/ExecutionData.cs @@ -9,6 +9,7 @@ public record ExecutionData public IReadOnlyDictionary>? TrainVerseCount { get; init; } public IReadOnlyDictionary>? WordAlignVerseCount { get; init; } public IReadOnlyList? Warnings { get; init; } + public IReadOnlyList? Diagnostics { get; init; } public string? EngineSourceLanguageTag { get; init; } public string? EngineTargetLanguageTag { get; init; } } diff --git a/src/Serval/src/Serval.WordAlignment/Services/DtoMapper.cs b/src/Serval/src/Serval.WordAlignment/Services/DtoMapper.cs index 1b368ac2..5d489dd2 100644 --- a/src/Serval/src/Serval.WordAlignment/Services/DtoMapper.cs +++ b/src/Serval/src/Serval.WordAlignment/Services/DtoMapper.cs @@ -195,8 +195,21 @@ private static WordAlignmentExecutionDataDto Map(ExecutionData source) IsTrainFilteredByChapter = source.IsTrainFilteredByChapter, IsWordAlignFilteredByChapter = source.IsWordAlignFilteredByChapter, Warnings = source.Warnings ?? [], + Diagnostics = source.Diagnostics?.Select(Map).ToList() ?? [], EngineSourceLanguageTag = source.EngineSourceLanguageTag, EngineTargetLanguageTag = source.EngineTargetLanguageTag, }; } + + private static DiagnosticDto Map(Diagnostic source) + { + return new DiagnosticDto + { + Code = source.Code, + Category = source.Category, + Message = source.Message, + Severity = (Dtos.DiagnosticSeverity)source.Severity, + Data = source.Data, + }; + } } From a3a15842977af79e97d13948a2bec2fc4d39e203 Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Tue, 28 Jul 2026 16:28:51 -0400 Subject: [PATCH 8/9] Make warnings obsolete --- src/Serval/src/Serval.Client/Client.g.cs | 2 ++ src/Serval/src/Serval.Translation/Dtos/ExecutionDataDto.cs | 2 ++ .../Serval.WordAlignment/Dtos/WordAlignmentExecutionDataDto.cs | 2 ++ 3 files changed, 6 insertions(+) diff --git a/src/Serval/src/Serval.Client/Client.g.cs b/src/Serval/src/Serval.Client/Client.g.cs index 3b19c43b..107f3cff 100644 --- a/src/Serval/src/Serval.Client/Client.g.cs +++ b/src/Serval/src/Serval.Client/Client.g.cs @@ -10942,6 +10942,7 @@ public partial class ExecutionData [Newtonsoft.Json.JsonProperty("warnings", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required] + [System.Obsolete] public System.Collections.Generic.IList Warnings { get; set; } = new System.Collections.ObjectModel.Collection(); [Newtonsoft.Json.JsonProperty("diagnostics", Required = Newtonsoft.Json.Required.Always)] @@ -11740,6 +11741,7 @@ public partial class WordAlignmentExecutionData [Newtonsoft.Json.JsonProperty("warnings", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required] + [System.Obsolete] public System.Collections.Generic.IList Warnings { get; set; } = new System.Collections.ObjectModel.Collection(); [Newtonsoft.Json.JsonProperty("diagnostics", Required = Newtonsoft.Json.Required.Always)] diff --git a/src/Serval/src/Serval.Translation/Dtos/ExecutionDataDto.cs b/src/Serval/src/Serval.Translation/Dtos/ExecutionDataDto.cs index af16fdb0..e9b5252a 100644 --- a/src/Serval/src/Serval.Translation/Dtos/ExecutionDataDto.cs +++ b/src/Serval/src/Serval.Translation/Dtos/ExecutionDataDto.cs @@ -8,6 +8,8 @@ public record ExecutionDataDto public bool? IsPretranslateFilteredByChapter { get; init; } public IReadOnlyDictionary>? TrainVerseCount { get; init; } public IReadOnlyDictionary>? PretranslateVerseCount { get; init; } + + [Obsolete] public IReadOnlyList Warnings { get; init; } = []; public IReadOnlyList Diagnostics { get; init; } = []; public string? EngineSourceLanguageTag { get; init; } diff --git a/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentExecutionDataDto.cs b/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentExecutionDataDto.cs index dd9bcc39..843afe28 100644 --- a/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentExecutionDataDto.cs +++ b/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentExecutionDataDto.cs @@ -8,6 +8,8 @@ public record WordAlignmentExecutionDataDto public bool? IsWordAlignFilteredByChapter { get; init; } public IReadOnlyDictionary>? TrainVerseCount { get; init; } public IReadOnlyDictionary>? WordAlignVerseCount { get; init; } + + [Obsolete] public IReadOnlyList Warnings { get; init; } = []; public IReadOnlyList Diagnostics { get; init; } = []; public string? EngineSourceLanguageTag { get; init; } From 28753aa5aaa90a1c5ff15e1c79fb7c8b17fffda1 Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Wed, 29 Jul 2026 15:40:35 -0400 Subject: [PATCH 9/9] Address reviewer comments --- .../Configuration/BuildJobOptions.cs | 1 + .../Services/EchoPreprocessBuildJob.cs | 24 ++--- .../EchoWordAlignmentPreprocessBuildJob.cs | 24 ++--- .../Services/NmtPreprocessBuildJob.cs | 77 ++++----------- .../Services/PreprocessBuildJob.cs | 96 ++----------------- .../Services/TranslationPreprocessBuildJob.cs | 24 ++--- .../WordAlignmentPreprocessBuildJob.cs | 24 ++--- src/Serval/src/Serval.Client/Client.g.cs | 22 ++--- .../Dtos/ExecutionDataDto.cs | 2 +- ...sticDto.cs => TranslationDiagnosticDto.cs} | 6 +- .../Serval.Translation/Services/DtoMapper.cs | 6 +- .../Services/PlatformService.cs | 24 +++-- ...icDto.cs => WordAlignmentDiagnosticDto.cs} | 6 +- .../Dtos/WordAlignmentExecutionDataDto.cs | 2 +- .../Services/DtoMapper.cs | 6 +- 15 files changed, 119 insertions(+), 225 deletions(-) rename src/Serval/src/Serval.Translation/Dtos/{DiagnosticDto.cs => TranslationDiagnosticDto.cs} (65%) rename src/Serval/src/Serval.WordAlignment/Dtos/{DiagnosticDto.cs => WordAlignmentDiagnosticDto.cs} (64%) diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/BuildJobOptions.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/BuildJobOptions.cs index b6b9da57..efc982f2 100644 --- a/src/Machine/src/Serval.Machine.Shared/Configuration/BuildJobOptions.cs +++ b/src/Machine/src/Serval.Machine.Shared/Configuration/BuildJobOptions.cs @@ -7,4 +7,5 @@ public class BuildJobOptions public IList ClearML { get; set; } = new List(); public bool PreserveBuildFiles { get; set; } = false; public int MaxWarnings { get; set; } = 1000; + public int MaxDiagnostics { get; set; } = 1000; } diff --git a/src/Machine/src/Serval.Machine.Shared/Services/EchoPreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/EchoPreprocessBuildJob.cs index 7c0b70db..aabc1e11 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/EchoPreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/EchoPreprocessBuildJob.cs @@ -35,14 +35,25 @@ protected override async Task UpdateBuildExecutionData( CancellationToken cancellationToken ) { - IReadOnlyList warnings = GetWarnings( + IReadOnlyList diagnostics = GetDiagnostics( stats.TrainCount, stats.InferenceCount, sourceLanguageTag, targetLanguageTag, + sourceLanguageHasNativeSupport: true, + targetLanguageHasNativeSupport: true, + isNonPersistedTranslationEngine, parallelCorpora ); + IReadOnlyList warnings = diagnostics.Select(d => d.Message).ToList(); + + int maxDiagnostics = BuildJobOptions.MaxDiagnostics; + if (diagnostics.Count > maxDiagnostics) + { + diagnostics = diagnostics.OrderByDescending(d => d.Severity).Take(maxDiagnostics).ToList(); + } + int maxWarnings = BuildJobOptions.MaxWarnings; if (warnings.Count > maxWarnings) { @@ -51,17 +62,6 @@ CancellationToken cancellationToken warnings = [tooManyWarningsWarning, .. warnings.Take(maxWarnings)]; } - IReadOnlyList diagnostics = GetDiagnostics( - stats.TrainCount, - stats.InferenceCount, - sourceLanguageTag, - targetLanguageTag, - true, - true, - isNonPersistedTranslationEngine, - parallelCorpora - ); - // Log summary of build data var buildPreprocessSummary = new JsonObject { diff --git a/src/Machine/src/Serval.Machine.Shared/Services/EchoWordAlignmentPreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/EchoWordAlignmentPreprocessBuildJob.cs index 66e26b7a..0be276a2 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/EchoWordAlignmentPreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/EchoWordAlignmentPreprocessBuildJob.cs @@ -95,14 +95,25 @@ protected override async Task UpdateBuildExecutionData( CancellationToken cancellationToken ) { - IReadOnlyList warnings = GetWarnings( + IReadOnlyList diagnostics = GetDiagnostics( stats.TrainCount, stats.InferenceCount, sourceLanguageTag, targetLanguageTag, + sourceLanguageHasNativeSupport: true, + targetLanguageHasNativeSupport: true, + isNonPersistedTranslationEngine, parallelCorpora ); + IReadOnlyList warnings = diagnostics.Select(d => d.Message).ToList(); + + int maxDiagnostics = BuildJobOptions.MaxDiagnostics; + if (diagnostics.Count > maxDiagnostics) + { + diagnostics = diagnostics.OrderByDescending(d => d.Severity).Take(maxDiagnostics).ToList(); + } + int maxWarnings = BuildJobOptions.MaxWarnings; if (warnings.Count > maxWarnings) { @@ -111,17 +122,6 @@ CancellationToken cancellationToken warnings = [tooManyWarningsWarning, .. warnings.Take(maxWarnings)]; } - IReadOnlyList diagnostics = GetDiagnostics( - stats.TrainCount, - stats.InferenceCount, - sourceLanguageTag, - targetLanguageTag, - true, - true, - isNonPersistedTranslationEngine, - parallelCorpora - ); - // Log summary of build data var buildPreprocessSummary = new JsonObject { diff --git a/src/Machine/src/Serval.Machine.Shared/Services/NmtPreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/NmtPreprocessBuildJob.cs index ac70f612..bd0691bd 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/NmtPreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/NmtPreprocessBuildJob.cs @@ -24,7 +24,7 @@ IOptionsMonitor options { private readonly ILanguageTagService _languageTagService = languageTagService; private const string ModelName = "NLLB"; - private const string MinimumTrainCount = "600"; //TODO move to options? + private const int MinimumTrainCount = 600; //TODO move to options? private bool ResolveLanguageCode(string languageCode, out string resolvedCode) { @@ -65,14 +65,25 @@ CancellationToken cancellationToken bool sourceLanguageHasNativeSupport = ResolveLanguageCode(sourceLanguageTag, out string resolvedSourceLanguage); bool targetLanguageHasNativeSupport = ResolveLanguageCode(targetLanguageTag, out string resolvedTargetLanguage); - IReadOnlyList warnings = GetWarnings( + IReadOnlyList diagnostics = GetDiagnostics( stats.TrainCount, stats.InferenceCount, sourceLanguageTag, targetLanguageTag, + sourceLanguageHasNativeSupport, + targetLanguageHasNativeSupport, + isNonPersistedTranslationEngine, parallelCorpora ); + IReadOnlyList warnings = diagnostics.Select(d => d.Message).ToList(); + + int maxDiagnostics = BuildJobOptions.MaxDiagnostics; + if (diagnostics.Count > maxDiagnostics) + { + diagnostics = diagnostics.OrderByDescending(d => d.Severity).Take(maxDiagnostics).ToList(); + } + int maxWarnings = BuildJobOptions.MaxWarnings; if (warnings.Count > maxWarnings) { @@ -81,17 +92,6 @@ CancellationToken cancellationToken warnings = [tooManyWarningsWarning, .. warnings.Take(maxWarnings)]; } - IReadOnlyList diagnostics = GetDiagnostics( - stats.TrainCount, - stats.InferenceCount, - sourceLanguageTag, - targetLanguageTag, - sourceLanguageHasNativeSupport, - targetLanguageHasNativeSupport, - isNonPersistedTranslationEngine, - parallelCorpora - ); - // Log summary of build data JsonObject buildPreprocessSummary = new() { @@ -132,45 +132,6 @@ CancellationToken cancellationToken } } - protected override IReadOnlyList GetWarnings( - int trainCount, - int inferenceCount, - string sourceLanguageTag, - string targetLanguageTag, - IReadOnlyList parallelCorpora - ) - { - List warnings = - [ - .. base.GetWarnings(trainCount, inferenceCount, sourceLanguageTag, targetLanguageTag, parallelCorpora), - ]; - - // Has at least a Gospel of Mark amount of data and not the special case of no data which will be caught elsewhere - if (trainCount < 600 && trainCount != 0) - { - warnings.Add($"Only {trainCount} segments were selected for training."); - } - - if ( - _languageTagService.ConvertToFlores200Code(sourceLanguageTag, out string resolvedCode) - == Flores200Support.None - ) - { - warnings.Add( - $"The script for the source language '{resolvedCode}' is not known to the base model {ModelName}" - ); - } - - if (_languageTagService.ConvertToFlores200Code(targetLanguageTag, out resolvedCode) == Flores200Support.None) - { - warnings.Add( - $"The script for the target language '{resolvedCode}' is not known to the base model {ModelName}" - ); - } - - return warnings; - } - protected override IReadOnlyList GetDiagnostics( int trainCount, int inferenceCount, @@ -202,7 +163,7 @@ .. base.GetDiagnostics( diagnostics.Add( new BuildDiagnostic { - Code = "CONFIG-003", + Code = "CONFIG-0003", Category = "CONFIG", Severity = BuildDiagnosticSeverity.Warn, Message = @@ -224,14 +185,14 @@ .. base.GetDiagnostics( diagnostics.Add( new BuildDiagnostic { - Code = "MODEL-001", + Code = "MODEL-0001", Category = "MODEL", Severity = BuildDiagnosticSeverity.Warn, Message = $"The script for the source language '{resolvedCode}' is not known to the base model {ModelName}", Data = new Dictionary { - { "resolvedCode", sourceLanguageTag }, + { "resolvedCode", resolvedCode }, { "modelName", ModelName }, }, } @@ -243,14 +204,14 @@ .. base.GetDiagnostics( diagnostics.Add( new BuildDiagnostic { - Code = "MODEL-002", + Code = "MODEL-0002", Category = "MODEL", Severity = BuildDiagnosticSeverity.Warn, Message = $"The script for the target language '{resolvedCode}' is not known to the base model {ModelName}", Data = new Dictionary { - { "resolvedCode", targetLanguageTag }, + { "resolvedCode", resolvedCode }, { "modelName", ModelName }, }, } @@ -269,7 +230,7 @@ .. base.GetDiagnostics( diagnostics.Add( new BuildDiagnostic { - Code = "MODEL-004", + Code = "MODEL-0004", Category = "MODEL", Severity = BuildDiagnosticSeverity.Error, Message = diff --git a/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs index 9baec0da..ce550252 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs @@ -124,82 +124,6 @@ protected override async Task CleanupAsync(string engineId, string buildId, JobC } } - protected virtual IReadOnlyList GetWarnings( - int trainCount, - int inferenceCount, - string sourceLanguageTag, - string targetLanguageTag, - IReadOnlyList parallelCorpora - ) - { - List warnings = []; - HashSet versifications = []; - - foreach ( - ( - string parallelCorpusId, - string monolingualCorpusId, - string projectName, - string projectGuid, - string versificationName, - IReadOnlyList diagnostics - ) in ParallelCorpusService.AnalyzeUsfmVersification(parallelCorpora) - ) - { - versifications.Add(versificationName); - foreach (UsfmVersificationDiagnosticContract diagnostic in diagnostics) - { - string diagnosticDetails = - $"in project {projectName} {projectGuid} at {diagnostic.Filename} " - + (diagnostic.LineNumbers.Count == 1 ? "line " : "lines ") - + $"{string.Join(", ", diagnostic.LineNumbers)}, " - + (diagnostic.NumAffectedVerses == 1 ? "verse " : "verses ") - + $"{string.Join(", ", diagnostic.References)} " - + $"(parallel corpus {parallelCorpusId}, monolingual corpus {monolingualCorpusId})."; - warnings.Add( - diagnostic.Type switch - { - Serval.Shared.Contracts.UsfmVersificationDiagnosticType.InvalidChapter => - $"Invalid chapter number {diagnosticDetails}", - Serval.Shared.Contracts.UsfmVersificationDiagnosticType.InvalidVerse => - $"Invalid verse number {diagnosticDetails}", - Serval.Shared.Contracts.UsfmVersificationDiagnosticType.Extra => - $"{diagnostic.NumAffectedVerses} extra verses {diagnosticDetails}", - Serval.Shared.Contracts.UsfmVersificationDiagnosticType.Missing => - $"Missing {diagnostic.NumAffectedVerses} verses {diagnosticDetails}", - Serval.Shared.Contracts.UsfmVersificationDiagnosticType.IncorrectVerseSegment => - $"Incorrect verse segment {diagnosticDetails}", - Serval.Shared.Contracts.UsfmVersificationDiagnosticType.UnsupportedVerseRange => - $"Unsupported verse range {diagnosticDetails}", - _ => $"USFM versification issue {diagnosticDetails}", - } - ); - } - } - - foreach ( - ( - string parallelCorpusId, - string monolingualCorpusId, - MissingParentProjectErrorContract error - ) in ParallelCorpusService.FindMissingParentProjects(parallelCorpora) - ) - { - warnings.Add( - $"Unable to locate parent project {error.ParentProjectName} {error.ParentProjectGuid} of daughter project {error.ProjectName} {error.ProjectGuid} (parallel corpus {parallelCorpusId}, monolingual corpus {monolingualCorpusId})" - ); - } - - if (versifications.Count > 1) - { - warnings.Add( - $"There are multiple versifications represented among Paratext projects selected for training or inferencing: {string.Join(", ", versifications)}." - ); - } - - return warnings; - } - protected virtual IReadOnlyList GetDiagnostics( int trainCount, int inferenceCount, @@ -240,7 +164,7 @@ IReadOnlyList usfmDiagnostics { Serval.Shared.Contracts.UsfmVersificationDiagnosticType.InvalidChapter => new BuildDiagnostic { - Code = "USFM-001", + Code = "USFM-0001", Category = "USFM", Severity = BuildDiagnosticSeverity.Warn, Message = $"Invalid chapter number {diagnosticDetails}", @@ -263,7 +187,7 @@ IReadOnlyList usfmDiagnostics }, Serval.Shared.Contracts.UsfmVersificationDiagnosticType.InvalidVerse => new BuildDiagnostic { - Code = "USFM-002", + Code = "USFM-0002", Category = "USFM", Severity = BuildDiagnosticSeverity.Warn, Message = $"Invalid verse number {diagnosticDetails}", @@ -286,7 +210,7 @@ IReadOnlyList usfmDiagnostics }, Serval.Shared.Contracts.UsfmVersificationDiagnosticType.Extra => new BuildDiagnostic { - Code = "USFM-003", + Code = "USFM-0003", Category = "USFM", Severity = BuildDiagnosticSeverity.Info, Message = $"{usfmDiagnostic.NumAffectedVerses} extra verses {diagnosticDetails}", @@ -304,7 +228,7 @@ IReadOnlyList usfmDiagnostics }, Serval.Shared.Contracts.UsfmVersificationDiagnosticType.Missing => new BuildDiagnostic { - Code = "USFM-004", + Code = "USFM-0004", Category = "USFM", Severity = BuildDiagnosticSeverity.Warn, Message = $"Missing {usfmDiagnostic.NumAffectedVerses} verses {diagnosticDetails}", @@ -323,7 +247,7 @@ IReadOnlyList usfmDiagnostics Serval.Shared.Contracts.UsfmVersificationDiagnosticType.IncorrectVerseSegment => new BuildDiagnostic { - Code = "USFM-005", + Code = "USFM-0005", Category = "USFM", Severity = BuildDiagnosticSeverity.Info, Message = $"Incorrect verse segment {diagnosticDetails}", @@ -347,7 +271,7 @@ IReadOnlyList usfmDiagnostics Serval.Shared.Contracts.UsfmVersificationDiagnosticType.UnsupportedVerseRange => new BuildDiagnostic { - Code = "USFM-006", + Code = "USFM-0006", Category = "USFM", Severity = BuildDiagnosticSeverity.Info, Message = $"Unsupported verse range {diagnosticDetails}", @@ -385,7 +309,7 @@ MissingParentProjectErrorContract error diagnostics.Add( new BuildDiagnostic { - Code = "CONFIG-001", + Code = "CONFIG-0001", Category = "CONFIG", Severity = BuildDiagnosticSeverity.Warn, Message = @@ -403,12 +327,12 @@ MissingParentProjectErrorContract error ); } - if (projectVersifications.Count > 1) + if (projectVersifications.Values.Distinct().Count() > 1) { diagnostics.Add( new BuildDiagnostic { - Code = "CONFIG-002", + Code = "CONFIG-0002", Category = "CONFIG", Severity = BuildDiagnosticSeverity.Info, Message = @@ -423,7 +347,7 @@ MissingParentProjectErrorContract error diagnostics.Add( new BuildDiagnostic { - Code = "CONFIG-004", + Code = "CONFIG-0004", Category = "CONFIG", Severity = BuildDiagnosticSeverity.Error, Message = "There was no data specified for inferencing and the model is not persisted.", diff --git a/src/Machine/src/Serval.Machine.Shared/Services/TranslationPreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/TranslationPreprocessBuildJob.cs index c03ba7f9..22d0142f 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/TranslationPreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/TranslationPreprocessBuildJob.cs @@ -94,14 +94,25 @@ protected override async Task UpdateBuildExecutionData( CancellationToken cancellationToken ) { - IReadOnlyList warnings = GetWarnings( + IReadOnlyList diagnostics = GetDiagnostics( stats.TrainCount, stats.InferenceCount, sourceLanguageTag, targetLanguageTag, + true, + true, + isNonPersistedTranslationEngine, parallelCorpora ); + IReadOnlyList warnings = diagnostics.Select(d => d.Message).ToList(); + + int maxDiagnostics = BuildJobOptions.MaxDiagnostics; + if (diagnostics.Count > maxDiagnostics) + { + diagnostics = diagnostics.OrderByDescending(d => d.Severity).Take(maxDiagnostics).ToList(); + } + int maxWarnings = BuildJobOptions.MaxWarnings; if (warnings.Count > maxWarnings) { @@ -110,17 +121,6 @@ CancellationToken cancellationToken warnings = [tooManyWarningsWarning, .. warnings.Take(maxWarnings)]; } - IReadOnlyList diagnostics = GetDiagnostics( - stats.TrainCount, - stats.InferenceCount, - sourceLanguageTag, - targetLanguageTag, - true, - true, - isNonPersistedTranslationEngine, - parallelCorpora - ); - // Log summary of build data JsonObject buildPreprocessSummary = new() { diff --git a/src/Machine/src/Serval.Machine.Shared/Services/WordAlignmentPreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/WordAlignmentPreprocessBuildJob.cs index 069fdba1..ee8bafb0 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/WordAlignmentPreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/WordAlignmentPreprocessBuildJob.cs @@ -93,14 +93,25 @@ protected override async Task UpdateBuildExecutionData( CancellationToken cancellationToken ) { - IReadOnlyList warnings = GetWarnings( + IReadOnlyList diagnostics = GetDiagnostics( stats.TrainCount, stats.InferenceCount, sourceLanguageTag, targetLanguageTag, + true, + true, + isNonPersistedTranslationEngine, parallelCorpora ); + IReadOnlyList warnings = diagnostics.Select(d => d.Message).ToList(); + + int maxDiagnostics = BuildJobOptions.MaxDiagnostics; + if (diagnostics.Count > maxDiagnostics) + { + diagnostics = diagnostics.OrderByDescending(d => d.Severity).Take(maxDiagnostics).ToList(); + } + int maxWarnings = BuildJobOptions.MaxWarnings; if (warnings.Count > maxWarnings) { @@ -109,17 +120,6 @@ CancellationToken cancellationToken warnings = [tooManyWarningsWarning, .. warnings.Take(maxWarnings)]; } - IReadOnlyList diagnostics = GetDiagnostics( - stats.TrainCount, - stats.InferenceCount, - sourceLanguageTag, - targetLanguageTag, - true, - true, - isNonPersistedTranslationEngine, - parallelCorpora - ); - // Log summary of build data JsonObject buildPreprocessSummary = new() { diff --git a/src/Serval/src/Serval.Client/Client.g.cs b/src/Serval/src/Serval.Client/Client.g.cs index 107f3cff..2b96eb61 100644 --- a/src/Serval/src/Serval.Client/Client.g.cs +++ b/src/Serval/src/Serval.Client/Client.g.cs @@ -10945,9 +10945,8 @@ public partial class ExecutionData [System.Obsolete] public System.Collections.Generic.IList Warnings { get; set; } = new System.Collections.ObjectModel.Collection(); - [Newtonsoft.Json.JsonProperty("diagnostics", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required] - public System.Collections.Generic.IList Diagnostics { get; set; } = new System.Collections.ObjectModel.Collection(); + [Newtonsoft.Json.JsonProperty("diagnostics", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IList? Diagnostics { get; set; } = default!; [Newtonsoft.Json.JsonProperty("engineSourceLanguageTag", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] public string? EngineSourceLanguageTag { get; set; } = default!; @@ -10967,7 +10966,7 @@ public partial class ExecutionData } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.7.1.0 (NJsonSchema v11.6.1.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class Diagnostic + public partial class TranslationDiagnostic { [Newtonsoft.Json.JsonProperty("code", Required = Newtonsoft.Json.Required.Always)] @@ -10985,7 +10984,7 @@ public partial class Diagnostic [Newtonsoft.Json.JsonProperty("severity", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] [Newtonsoft.Json.JsonConverter(typeof(Newtonsoft.Json.Converters.StringEnumConverter))] - public DiagnosticSeverity Severity { get; set; } = default!; + public TranslationDiagnosticSeverity Severity { get; set; } = default!; [Newtonsoft.Json.JsonProperty("data", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required] @@ -10994,7 +10993,7 @@ public partial class Diagnostic } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.7.1.0 (NJsonSchema v11.6.1.0 (Newtonsoft.Json v13.0.0.0))")] - public enum DiagnosticSeverity + public enum TranslationDiagnosticSeverity { [System.Runtime.Serialization.EnumMember(Value = @"Info")] @@ -11744,9 +11743,8 @@ public partial class WordAlignmentExecutionData [System.Obsolete] public System.Collections.Generic.IList Warnings { get; set; } = new System.Collections.ObjectModel.Collection(); - [Newtonsoft.Json.JsonProperty("diagnostics", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required] - public System.Collections.Generic.IList Diagnostics { get; set; } = new System.Collections.ObjectModel.Collection(); + [Newtonsoft.Json.JsonProperty("diagnostics", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IList? Diagnostics { get; set; } = default!; [Newtonsoft.Json.JsonProperty("engineSourceLanguageTag", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] public string? EngineSourceLanguageTag { get; set; } = default!; @@ -11757,7 +11755,7 @@ public partial class WordAlignmentExecutionData } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.7.1.0 (NJsonSchema v11.6.1.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class Diagnostic2 + public partial class WordAlignmentDiagnostic { [Newtonsoft.Json.JsonProperty("code", Required = Newtonsoft.Json.Required.Always)] @@ -11775,7 +11773,7 @@ public partial class Diagnostic2 [Newtonsoft.Json.JsonProperty("severity", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] [Newtonsoft.Json.JsonConverter(typeof(Newtonsoft.Json.Converters.StringEnumConverter))] - public DiagnosticSeverity2 Severity { get; set; } = default!; + public WordAlignmentDiagnosticSeverity Severity { get; set; } = default!; [Newtonsoft.Json.JsonProperty("data", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required] @@ -11784,7 +11782,7 @@ public partial class Diagnostic2 } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.7.1.0 (NJsonSchema v11.6.1.0 (Newtonsoft.Json v13.0.0.0))")] - public enum DiagnosticSeverity2 + public enum WordAlignmentDiagnosticSeverity { [System.Runtime.Serialization.EnumMember(Value = @"Info")] diff --git a/src/Serval/src/Serval.Translation/Dtos/ExecutionDataDto.cs b/src/Serval/src/Serval.Translation/Dtos/ExecutionDataDto.cs index e9b5252a..c9ce06b5 100644 --- a/src/Serval/src/Serval.Translation/Dtos/ExecutionDataDto.cs +++ b/src/Serval/src/Serval.Translation/Dtos/ExecutionDataDto.cs @@ -11,7 +11,7 @@ public record ExecutionDataDto [Obsolete] public IReadOnlyList Warnings { get; init; } = []; - public IReadOnlyList Diagnostics { get; init; } = []; + public IReadOnlyList? Diagnostics { get; init; } public string? EngineSourceLanguageTag { get; init; } public string? EngineTargetLanguageTag { get; init; } public string? ResolvedSourceLanguage { get; init; } diff --git a/src/Serval/src/Serval.Translation/Dtos/DiagnosticDto.cs b/src/Serval/src/Serval.Translation/Dtos/TranslationDiagnosticDto.cs similarity index 65% rename from src/Serval/src/Serval.Translation/Dtos/DiagnosticDto.cs rename to src/Serval/src/Serval.Translation/Dtos/TranslationDiagnosticDto.cs index a45b1d88..21111c2d 100644 --- a/src/Serval/src/Serval.Translation/Dtos/DiagnosticDto.cs +++ b/src/Serval/src/Serval.Translation/Dtos/TranslationDiagnosticDto.cs @@ -1,15 +1,15 @@ namespace Serval.Translation.Dtos; -public record DiagnosticDto +public record TranslationDiagnosticDto { public required string Code { get; init; } public required string Category { get; init; } public required string Message { get; init; } - public required DiagnosticSeverity Severity { get; init; } + public required TranslationDiagnosticSeverity Severity { get; init; } public required Dictionary Data { get; init; } } -public enum DiagnosticSeverity +public enum TranslationDiagnosticSeverity { Info, Warn, diff --git a/src/Serval/src/Serval.Translation/Services/DtoMapper.cs b/src/Serval/src/Serval.Translation/Services/DtoMapper.cs index be5db5fd..721a08fe 100644 --- a/src/Serval/src/Serval.Translation/Services/DtoMapper.cs +++ b/src/Serval/src/Serval.Translation/Services/DtoMapper.cs @@ -244,14 +244,14 @@ private static ExecutionDataDto Map(ExecutionData source) => AveragePretranslationConfidence = source.AveragePretranslationConfidence, }; - private static DiagnosticDto Map(Diagnostic source) + private static TranslationDiagnosticDto Map(Diagnostic source) { - return new DiagnosticDto + return new TranslationDiagnosticDto { Code = source.Code, Category = source.Category, Message = source.Message, - Severity = (Dtos.DiagnosticSeverity)source.Severity, + Severity = (Dtos.TranslationDiagnosticSeverity)source.Severity, Data = source.Data, }; } diff --git a/src/Serval/src/Serval.Translation/Services/PlatformService.cs b/src/Serval/src/Serval.Translation/Services/PlatformService.cs index 76af2106..4475cc98 100644 --- a/src/Serval/src/Serval.Translation/Services/PlatformService.cs +++ b/src/Serval/src/Serval.Translation/Services/PlatformService.cs @@ -460,7 +460,7 @@ public async Task InsertPretranslationsAsync( if (batch.Count > 0) await _pretranslations.InsertAllAsync(batch, CancellationToken.None); - IEnumerable badBookConfidences = logConfidenceTotalPerBook + List badBookConfidences = logConfidenceTotalPerBook .Select(kvp => { string bookId = kvp.Key; @@ -472,22 +472,24 @@ public async Task InsertPretranslationsAsync( .Where(b => b.averageConfidence < BadBookConfidenceThreshold) .Select(b => new Diagnostic { - Code = "MODEL-004", + Code = "MODEL-0003", Category = "MODEL", Severity = Models.DiagnosticSeverity.Warn, Message = - $"The average pretranslation model confidence {b.averageConfidence} in book {b.bookId} is unusually low for the base model {ModelName}", + $"The average pretranslation model confidence {b.averageConfidence:f4} in book {b.bookId} is unusually low for the base model {ModelName}", Data = new Dictionary { { "bookId", b.bookId }, { "averagePretranslationConfidence", b.averageConfidence }, { "modelName", ModelName }, }, - }); + }) + .ToList(); - if (badBookConfidences.Any()) + Build? currentBuild = null; + if (badBookConfidences.Count > 0) { - Build? currentBuild = await _builds.GetAsync(b => b.Id == buildId, cancellationToken); + currentBuild = await _builds.GetAsync(b => b.Id == buildId, cancellationToken); await _builds.UpdateAsync( b => b.Id == buildId, @@ -505,13 +507,21 @@ await _builds.UpdateAsync( await _builds.UpdateAsync( b => b.Id == buildId, u => + { u.Set( b => b.ExecutionData.AveragePretranslationConfidence, // Calculate the geometric mean of the pretranslation confidences confidenceCount > 0 ? Math.Exp(logConfidenceTotal / confidenceCount) : 0.0 - ), + ); + u.Set( + b => b.ExecutionData.Diagnostics, + currentBuild?.ExecutionData.Diagnostics is null + ? [.. badBookConfidences] + : [.. currentBuild.ExecutionData.Diagnostics, .. badBookConfidences] + ); + }, cancellationToken: cancellationToken ); } diff --git a/src/Serval/src/Serval.WordAlignment/Dtos/DiagnosticDto.cs b/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentDiagnosticDto.cs similarity index 64% rename from src/Serval/src/Serval.WordAlignment/Dtos/DiagnosticDto.cs rename to src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentDiagnosticDto.cs index adeba861..7b602d51 100644 --- a/src/Serval/src/Serval.WordAlignment/Dtos/DiagnosticDto.cs +++ b/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentDiagnosticDto.cs @@ -1,15 +1,15 @@ namespace Serval.WordAlignment.Dtos; -public record DiagnosticDto +public record WordAlignmentDiagnosticDto { public required string Code { get; init; } public required string Category { get; init; } public required string Message { get; init; } - public required DiagnosticSeverity Severity { get; init; } + public required WordAlignmentDiagnosticSeverity Severity { get; init; } public required Dictionary Data { get; init; } } -public enum DiagnosticSeverity +public enum WordAlignmentDiagnosticSeverity { Info, Warn, diff --git a/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentExecutionDataDto.cs b/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentExecutionDataDto.cs index 843afe28..75670eb0 100644 --- a/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentExecutionDataDto.cs +++ b/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentExecutionDataDto.cs @@ -11,7 +11,7 @@ public record WordAlignmentExecutionDataDto [Obsolete] public IReadOnlyList Warnings { get; init; } = []; - public IReadOnlyList Diagnostics { get; init; } = []; + public IReadOnlyList? Diagnostics { get; init; } public string? EngineSourceLanguageTag { get; init; } public string? EngineTargetLanguageTag { get; init; } } diff --git a/src/Serval/src/Serval.WordAlignment/Services/DtoMapper.cs b/src/Serval/src/Serval.WordAlignment/Services/DtoMapper.cs index 5d489dd2..60a38a3b 100644 --- a/src/Serval/src/Serval.WordAlignment/Services/DtoMapper.cs +++ b/src/Serval/src/Serval.WordAlignment/Services/DtoMapper.cs @@ -201,14 +201,14 @@ private static WordAlignmentExecutionDataDto Map(ExecutionData source) }; } - private static DiagnosticDto Map(Diagnostic source) + private static WordAlignmentDiagnosticDto Map(Diagnostic source) { - return new DiagnosticDto + return new WordAlignmentDiagnosticDto { Code = source.Code, Category = source.Category, Message = source.Message, - Severity = (Dtos.DiagnosticSeverity)source.Severity, + Severity = (Dtos.WordAlignmentDiagnosticSeverity)source.Severity, Data = source.Data, }; }