diff --git a/src/SIL.Machine.Translation.Thot/SIL.Machine.Translation.Thot.csproj b/src/SIL.Machine.Translation.Thot/SIL.Machine.Translation.Thot.csproj index 88bb2fa0d..73a0adf16 100644 --- a/src/SIL.Machine.Translation.Thot/SIL.Machine.Translation.Thot.csproj +++ b/src/SIL.Machine.Translation.Thot/SIL.Machine.Translation.Thot.csproj @@ -12,7 +12,7 @@ - + diff --git a/src/SIL.Machine.Translation.Thot/Thot.cs b/src/SIL.Machine.Translation.Thot/Thot.cs index a012f8a5b..a7bf4de90 100644 --- a/src/SIL.Machine.Translation.Thot/Thot.cs +++ b/src/SIL.Machine.Translation.Thot/Thot.cs @@ -172,6 +172,21 @@ uint capacity [DllImport("thot", CallingConvention = CallingConvention.Cdecl)] public static extern uint swAlignModel_getMaxSentenceLength(IntPtr swAlignModelHandle); + [DllImport("thot", CallingConvention = CallingConvention.Cdecl)] + public static extern uint swAlignModel_getNumSentencePairs(IntPtr swAlignModelHandle); + + [DllImport("thot", CallingConvention = CallingConvention.Cdecl)] + public static extern double swAlignModel_getTrainingAlignment( + IntPtr swAlignModelHandle, + uint n, + IntPtr matrix, + ref uint iLen, + ref uint jLen + ); + + [DllImport("thot", CallingConvention = CallingConvention.Cdecl)] + public static extern void swAlignModel_setEmitTrainingAlignments(IntPtr swAlignModelHandle, bool value); + [DllImport("thot", CallingConvention = CallingConvention.Cdecl)] public static extern void swAlignModel_setVariationalBayes(IntPtr swAlignModelHandle, bool variationalBayes); diff --git a/src/SIL.Machine.Translation.Thot/ThotSymmetrizedWordAlignmentModel.cs b/src/SIL.Machine.Translation.Thot/ThotSymmetrizedWordAlignmentModel.cs new file mode 100644 index 000000000..1c386f3d7 --- /dev/null +++ b/src/SIL.Machine.Translation.Thot/ThotSymmetrizedWordAlignmentModel.cs @@ -0,0 +1,56 @@ +namespace SIL.Machine.Translation.Thot +{ + public class ThotSymmetrizedWordAlignmentModel : SymmetrizedWordAlignmentModel, ITransductiveWordAlignmentModel + { + private readonly ThotWordAlignmentModel _directWordAlignmentModel; + private readonly ThotWordAlignmentModel _inverseWordAlignmentModel; + + public ThotSymmetrizedWordAlignmentModel( + ThotWordAlignmentModel directWordAlignmentModel, + ThotWordAlignmentModel inverseWordAlignmentModel + ) + : base(directWordAlignmentModel, inverseWordAlignmentModel) + { + _directWordAlignmentModel = directWordAlignmentModel; + _inverseWordAlignmentModel = inverseWordAlignmentModel; + } + + public bool EmitTrainingAlignments + { + get => _directWordAlignmentModel.EmitTrainingAlignments; + set + { + _directWordAlignmentModel.EmitTrainingAlignments = value; + _inverseWordAlignmentModel.EmitTrainingAlignments = value; + } + } + + public int TrainingAlignmentCount => _directWordAlignmentModel.TrainingAlignmentCount; + + public WordAlignmentMatrix GetTrainingAlignment(int n) + { + WordAlignmentMatrix bestMatrix = _directWordAlignmentModel.GetTrainingAlignment(n); + if (Heuristic == SymmetrizationHeuristic.None) + return bestMatrix; + + WordAlignmentMatrix invMatrix = _inverseWordAlignmentModel.GetTrainingAlignment(n); + invMatrix.Transpose(); + + // Skip the combine when the matrices are degenerate or their dimensions don't + // line up (e.g. an out-of-range n, or a pair filtered out of training in only + // one direction): the heuristic operations require matching dimensions. + if ( + bestMatrix.RowCount == 0 + || bestMatrix.ColumnCount == 0 + || invMatrix.RowCount != bestMatrix.RowCount + || invMatrix.ColumnCount != bestMatrix.ColumnCount + ) + { + return bestMatrix; + } + + bestMatrix.SymmetrizeWith(invMatrix, Heuristic); + return bestMatrix; + } + } +} diff --git a/src/SIL.Machine.Translation.Thot/ThotWordAlignmentModel.cs b/src/SIL.Machine.Translation.Thot/ThotWordAlignmentModel.cs index a4b335eb6..a61096dec 100644 --- a/src/SIL.Machine.Translation.Thot/ThotWordAlignmentModel.cs +++ b/src/SIL.Machine.Translation.Thot/ThotWordAlignmentModel.cs @@ -13,7 +13,10 @@ namespace SIL.Machine.Translation.Thot { - public abstract class ThotWordAlignmentModel : DisposableBase, IIbm1WordAlignmentModel + public abstract class ThotWordAlignmentModel + : DisposableBase, + ITransductiveWordAlignmentModel, + IIbm1WordAlignmentModel { public static ThotWordAlignmentModel Create(ThotWordAlignmentModelType type) { @@ -156,6 +159,28 @@ public void Save() Thot.swAlignModel_save(Handle, _prefFileName); } + public bool EmitTrainingAlignments { get; set; } + + public int TrainingAlignmentCount => (int)Thot.swAlignModel_getNumSentencePairs(Handle); + + public WordAlignmentMatrix GetTrainingAlignment(int n) + { + CheckDisposed(); + IntPtr nativeMatrix = Thot.AllocNativeMatrix(_sourceWords.Count, _targetWords.Count); + + uint iLen = (uint)_sourceWords.Count; + uint jLen = (uint)_targetWords.Count; + try + { + Thot.swAlignModel_getTrainingAlignment(Handle, (uint)n, nativeMatrix, ref iLen, ref jLen); + return Thot.ConvertNativeMatrixToWordAlignmentMatrix(nativeMatrix, iLen, jLen); + } + finally + { + Thot.FreeNativeMatrix(nativeMatrix, iLen); + } + } + public double GetTranslationScore(string sourceWord, string targetWord) { return GetTranslationProbability(sourceWord, targetWord); @@ -316,7 +341,7 @@ private class Trainer : ThotWordAlignmentModelTrainer private readonly ThotWordAlignmentModel _model; public Trainer(ThotWordAlignmentModel model, IParallelTextCorpus corpus) - : base(model.Type, corpus, model._prefFileName, model.Parameters) + : base(model.Type, corpus, model._prefFileName, model.Parameters, model.EmitTrainingAlignments) { _model = model; CloseOnDispose = false; diff --git a/src/SIL.Machine.Translation.Thot/ThotWordAlignmentModelTrainer.cs b/src/SIL.Machine.Translation.Thot/ThotWordAlignmentModelTrainer.cs index c1624d4c9..c5af861f1 100644 --- a/src/SIL.Machine.Translation.Thot/ThotWordAlignmentModelTrainer.cs +++ b/src/SIL.Machine.Translation.Thot/ThotWordAlignmentModelTrainer.cs @@ -38,7 +38,8 @@ public ThotWordAlignmentModelTrainer( ThotWordAlignmentModelType modelType, IParallelTextCorpus corpus, string prefFileName, - ThotWordAlignmentParameters parameters = null + ThotWordAlignmentParameters parameters = null, + bool emitTrainingAlignments = false ) { _prefFileName = prefFileName; @@ -47,6 +48,8 @@ public ThotWordAlignmentModelTrainer( if (parameters == null) parameters = new ThotWordAlignmentParameters(); + EmitTrainingAlignments = emitTrainingAlignments; + _models = new List<(IntPtr, int)>(); if (modelType == ThotWordAlignmentModelType.FastAlign) { @@ -197,6 +200,8 @@ public ThotWordAlignmentModelTrainer( public TrainStats Stats { get; } = new TrainStats(); + public bool EmitTrainingAlignments { get; } + public int MaxCorpusCount { get; set; } = int.MaxValue; public Task TrainAsync(IProgress progress = null, CancellationToken cancellationToken = default) @@ -243,6 +248,14 @@ void Report() => Report(); cancellationToken.ThrowIfCancellationRequested(); + if (EmitTrainingAlignments) + { + // Retain the alignments computed during training so that they can be returned without a + // separate inference pass. Only the final (most refined) model's alignments are needed, + // since that is the model used for inference. + Thot.swAlignModel_setEmitTrainingAlignments(Handle, true); + } + int trainedSegmentCount = 0; foreach ((IntPtr handle, int storedIterationCount) in _models) { diff --git a/src/SIL.Machine/Corpora/CorporaExtensions.cs b/src/SIL.Machine/Corpora/CorporaExtensions.cs index 6b0000893..3d1614323 100644 --- a/src/SIL.Machine/Corpora/CorporaExtensions.cs +++ b/src/SIL.Machine/Corpora/CorporaExtensions.cs @@ -1132,6 +1132,14 @@ public static IParallelTextCorpus Translate( return new TranslateParallelTextCorpus(corpus, translationEngine, batchSize); } + public static IParallelTextCorpus WordAlign( + this IParallelTextCorpus corpus, + ITransductiveWordAlignmentModel model + ) + { + return new TransductiveWordAlignParallelTextCorpus(corpus, model); + } + public static IParallelTextCorpus WordAlign( this IParallelTextCorpus corpus, IWordAligner aligner, @@ -1244,6 +1252,61 @@ public override IEnumerable GetRows(IEnumerable textIds } } + private class TransductiveWordAlignParallelTextCorpus : ParallelTextCorpusBase + { + private readonly IParallelTextCorpus _corpus; + private readonly ITransductiveWordAlignmentModel _model; + + public TransductiveWordAlignParallelTextCorpus( + IParallelTextCorpus corpus, + ITransductiveWordAlignmentModel model + ) + { + _corpus = corpus; + _model = model; + } + + public override bool IsSourceTokenized => _corpus.IsSourceTokenized; + + public override bool IsTargetTokenized => _corpus.IsTargetTokenized; + + public override IEnumerable GetRows(IEnumerable textIds) + { + // The training alignments are keyed by the order in which the sentence pairs were added + // during training, so the full corpus must be iterated to keep the index in sync; rows that + // are not in the requested texts are skipped rather than filtered out of the enumeration. + var textIdList = textIds?.ToList(); + List rows = _corpus.GetRows().ToList(); + for (int i = 0; i < rows.Count; i++) + { + ParallelTextRow row = rows[i]; + if (textIdList != null && !textIdList.Contains(row.TextId)) + continue; + + WordAlignmentMatrix alignment = _model.GetTrainingAlignment(i); + WordAlignmentMatrix knownAlignment = row.CreateAlignmentMatrix(); + if (knownAlignment != null) + { + knownAlignment.PrioritySymmetrizeWith(alignment); + alignment = knownAlignment; + } + + IReadOnlyCollection wordPairs = alignment.ToAlignedWordPairs(); + if (_model is IWordAlignmentModel wordAlignmentModel) + { + wordAlignmentModel.ComputeAlignedWordPairScores( + row.SourceSegment, + row.TargetSegment, + wordPairs + ); + } + + row.AlignedWordPairs = wordPairs; + yield return row; + } + } + } + private class TranslateParallelTextCorpus : ParallelTextCorpusBase { private readonly IParallelTextCorpus _corpus; diff --git a/src/SIL.Machine/Translation/ITransductiveWordAlignmentModel.cs b/src/SIL.Machine/Translation/ITransductiveWordAlignmentModel.cs new file mode 100644 index 000000000..a7aaf8cbb --- /dev/null +++ b/src/SIL.Machine/Translation/ITransductiveWordAlignmentModel.cs @@ -0,0 +1,8 @@ +namespace SIL.Machine.Translation +{ + public interface ITransductiveWordAlignmentModel + { + int TrainingAlignmentCount { get; } + WordAlignmentMatrix GetTrainingAlignment(int n); + } +} diff --git a/tests/SIL.Machine.Translation.Thot.Tests/TestHelpers.cs b/tests/SIL.Machine.Translation.Thot.Tests/TestHelpers.cs index 34d1b8815..2d0a93d03 100644 --- a/tests/SIL.Machine.Translation.Thot.Tests/TestHelpers.cs +++ b/tests/SIL.Machine.Translation.Thot.Tests/TestHelpers.cs @@ -12,9 +12,19 @@ public static class TestHelpers Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "data", "toy_corpus_fa"); public static string ToyCorpusFastAlignConfigFileName => Path.Combine(ToyCorpusFastAlignFolderName, "smt.cfg"); - public static IEnumerable Split(this string segment) + public static IReadOnlyList AlignmentStrings( + IParallelTextCorpus corpus, + IEnumerable? textIds = null + ) { - return segment.Split(' '); + return + [ + .. corpus + .GetRows(textIds) + .SelectMany(row => + row.AlignedWordPairs.Select(wp => new AlignedWordPair(wp.SourceIndex, wp.TargetIndex).ToString()) + ), + ]; } public static ParallelTextCorpus CreateTestParallelCorpus() @@ -22,8 +32,7 @@ public static ParallelTextCorpus CreateTestParallelCorpus() var srcCorpus = new DictionaryTextCorpus( new MemoryText( "text1", - new[] - { + [ Row(1, "isthay isyay ayay esttay-N ."), Row(2, "ouyay ouldshay esttay-V oftenyay ."), Row(3, "isyay isthay orkingway ?"), @@ -32,15 +41,14 @@ public static ParallelTextCorpus CreateTestParallelCorpus() Row(6, "orkway-N ancay ebay ardhay !"), Row(7, "ayay esttay-N ancay ebay ardhay ."), Row(8, "isthay isyay ayay ordway !"), - } + ] ) ); var trgCorpus = new DictionaryTextCorpus( new MemoryText( "text1", - new[] - { + [ Row(1, "this is a test N ."), Row(2, "you should test V often ."), Row(3, "is this working ?"), @@ -49,13 +57,68 @@ public static ParallelTextCorpus CreateTestParallelCorpus() Row(6, "work N can be hard !"), Row(7, "a test N can be hard ."), Row(8, "this is a word !"), - } + ] ) ); return new ParallelTextCorpus(srcCorpus, trgCorpus); } + public static ParallelTextCorpus CreateTwoTextParallelCorpus() + { + var src = new DictionaryTextCorpus( + new MemoryText( + "text1", + [ + new TextRow("text1", 1) { Segment = "el gato".Split(' ') }, + new TextRow("text1", 2) { Segment = "la casa".Split(' ') }, + ] + ), + new MemoryText( + "text2", + [ + new TextRow("text2", 1) { Segment = "el perro corre".Split(' ') }, + new TextRow("text2", 2) { Segment = "la mesa".Split(' ') }, + ] + ) + ); + + var trg = new DictionaryTextCorpus( + new MemoryText( + "text1", + [ + new TextRow("text1", 1) { Segment = "the cat".Split(' ') }, + new TextRow("text1", 2) { Segment = "the house".Split(' ') }, + ] + ), + new MemoryText( + "text2", + [ + new TextRow("text2", 1) { Segment = "the dog runs".Split(' ') }, + new TextRow("text2", 2) { Segment = "the table".Split(' ') }, + ] + ) + ); + + return new ParallelTextCorpus(src, trg); + } + + public static async Task CreateWordAligner(IParallelTextCorpus corpus) + where T : ThotWordAlignmentModel, new() + { + var aligner = new ThotSymmetrizedWordAlignmentModel(new T(), new T()) + { + Heuristic = SymmetrizationHeuristic.GrowDiagFinalAnd, + // Retain the alignments computed during training so that the corpus can be aligned + // without a separate, potentially expensive, inference pass. + EmitTrainingAlignments = true, + }; + ITrainer trainer = aligner.CreateTrainer(corpus); + await trainer.TrainAsync(); + await trainer.SaveAsync(); + return aligner; + } + private static TextRow Row(int rowRef, string segment) { return new TextRow("text1", rowRef) { Segment = segment.Split() }; diff --git a/tests/SIL.Machine.Translation.Thot.Tests/ThotCorpusTests.cs b/tests/SIL.Machine.Translation.Thot.Tests/ThotCorpusTests.cs new file mode 100644 index 000000000..a1eb91a6b --- /dev/null +++ b/tests/SIL.Machine.Translation.Thot.Tests/ThotCorpusTests.cs @@ -0,0 +1,54 @@ +using NUnit.Framework; +using SIL.Machine.Corpora; + +namespace SIL.Machine.Translation.Thot; + +[TestFixture(typeof(ThotEflomalWordAlignmentModel))] +[TestFixture(typeof(ThotFastAlignWordAlignmentModel))] +[TestFixture(typeof(ThotIbm1WordAlignmentModel))] +public class ThotCorpusTests + where T : ThotWordAlignmentModel, new() +{ + [Test] + public async Task WordAlignCorpus_TransductiveMatchesInference() + { + ParallelTextCorpus corpus = TestHelpers.CreateTestParallelCorpus(); + using ThotSymmetrizedWordAlignmentModel aligner = await TestHelpers.CreateWordAligner(corpus); + + // For deterministic models, the alignments retained during training match those produced by a + // separate inference pass, so the transductive output must equal aligning each row directly. + IReadOnlyList transductive = TestHelpers.AlignmentStrings(corpus.WordAlign(aligner)); + + using var model = new ThotSymmetrizedWordAlignmentModel(new T(), new T()); + model.Heuristic = SymmetrizationHeuristic.GrowDiagFinalAnd; + ITrainer trainer = model.CreateTrainer(TestHelpers.CreateTestParallelCorpus()); + await trainer.TrainAsync(); + await trainer.SaveAsync(); + IReadOnlyList inference = + [ + .. TestHelpers + .CreateTestParallelCorpus() + .GetRows() + .SelectMany(row => + model + .Align(row.SourceSegment, row.TargetSegment) + .ToAlignedWordPairs() + .Select(wp => new AlignedWordPair(wp.SourceIndex, wp.TargetIndex).ToString()) + ), + ]; + Assert.That(transductive, Is.EquivalentTo(inference)); + } + + [Test] + public async Task WordAlignCorpus_DefaultIsTransductive() + { + ParallelTextCorpus corpus = TestHelpers.CreateTestParallelCorpus(); + using ThotSymmetrizedWordAlignmentModel aligner = await TestHelpers.CreateWordAligner(corpus); + List rows = [.. corpus.WordAlign(aligner).GetRows()]; + using (Assert.EnterMultipleScope()) + { + Assert.That(rows, Has.Count.EqualTo(8)); + Assert.That(rows.Any(row => row.AlignedWordPairs.Count > 0), Is.True); + } + } +} diff --git a/tests/SIL.Machine.Translation.Thot.Tests/ThotFastAlignWordAlignmentModelTests.cs b/tests/SIL.Machine.Translation.Thot.Tests/ThotFastAlignWordAlignmentModelTests.cs index e5ae41694..68568f8bd 100644 --- a/tests/SIL.Machine.Translation.Thot.Tests/ThotFastAlignWordAlignmentModelTests.cs +++ b/tests/SIL.Machine.Translation.Thot.Tests/ThotFastAlignWordAlignmentModelTests.cs @@ -1,4 +1,5 @@ using NUnit.Framework; +using SIL.Machine.Corpora; using SIL.Machine.Utils; namespace SIL.Machine.Translation.Thot; @@ -176,4 +177,89 @@ public void Constructor_ModelCorrupted() File.WriteAllText(modelPrefix + ".src", "corrupted"); Assert.Throws(() => new ThotFastAlignWordAlignmentModel(modelPrefix)); } + + [Test] + public async Task EmitTrainingAlignments_SingleDirection() + { + ParallelTextCorpus corpus = TestHelpers.CreateTestParallelCorpus(); + ParallelTextRow row = corpus.GetRows().First(); + using var model = new ThotFastAlignWordAlignmentModel(); + model.EmitTrainingAlignments = true; + ITrainer trainer = model.CreateTrainer(corpus); + await trainer.TrainAsync(); + await trainer.SaveAsync(); + using (Assert.EnterMultipleScope()) + { + Assert.That(model.TrainingAlignmentCount, Is.EqualTo(8)); + // For a deterministic model, the retained training alignment matches the inference alignment, + // and it survives the trainer being closed. + Assert.That( + model.GetTrainingAlignment(0).ValueEquals(model.Align(row.SourceSegment, row.TargetSegment)), + Is.True + ); + } + } + + [Test] + public async Task EmitTrainingAlignments_Symmetrized() + { + ParallelTextCorpus corpus = TestHelpers.CreateTestParallelCorpus(); + ParallelTextRow row = corpus.GetRows().First(); + using var model = new ThotSymmetrizedWordAlignmentModel( + new ThotFastAlignWordAlignmentModel(), + new ThotFastAlignWordAlignmentModel() + ); + model.EmitTrainingAlignments = true; + ITrainer trainer = model.CreateTrainer(corpus); + await trainer.TrainAsync(); + await trainer.SaveAsync(); + using (Assert.EnterMultipleScope()) + { + Assert.That(model.TrainingAlignmentCount, Is.EqualTo(8)); + // The C++ symmetrized transductive alignment matches the C++ symmetrized inference alignment. + Assert.That( + model.GetTrainingAlignment(0).ValueEquals(model.Align(row.SourceSegment, row.TargetSegment)), + Is.True + ); + } + } + + [Test] + public async Task EmitTrainingAlignments_Disabled() + { + ParallelTextCorpus corpus = TestHelpers.CreateTestParallelCorpus(); + using var model = new ThotFastAlignWordAlignmentModel(); + ITrainer trainer = model.CreateTrainer(corpus); + await trainer.TrainAsync(); + await trainer.SaveAsync(); + // When emission is not enabled, retrieval returns a degenerate result rather than raising. + WordAlignmentMatrix alignment = model.GetTrainingAlignment(0); + using (Assert.EnterMultipleScope()) + { + Assert.That(alignment.ColumnCount, Is.Zero); + Assert.That(alignment.RowCount, Is.Zero); + } + } + + [Test] + public async Task WordAlignCorpus_TransductiveTextIdsKeepIndexInSync() + { + // Filtering by text must not desync the training-alignment index: the rows for a requested text + // must get exactly the alignments they got in the unfiltered pass, not those of earlier rows. + IParallelTextCorpus corpus = TestHelpers.CreateTwoTextParallelCorpus(); + using ThotSymmetrizedWordAlignmentModel aligner = + await TestHelpers.CreateWordAligner(corpus); + corpus = corpus.WordAlign(aligner); + List full = [.. corpus.GetRows()]; + IReadOnlyList text2Expected = + [ + .. full.Skip(2) + .SelectMany(row => + row.AlignedWordPairs.Select(wp => new AlignedWordPair(wp.SourceIndex, wp.TargetIndex).ToString()) + ), + ]; + + IReadOnlyList text2Actual = TestHelpers.AlignmentStrings(corpus, ["text2"]); + Assert.That(text2Actual, Is.EqualTo(text2Expected)); + } }