From d3d534818aab235114625415f649c46f4f4449ce Mon Sep 17 00:00:00 2001 From: Peter Chapman Date: Thu, 30 Jul 2026 07:46:38 +1200 Subject: [PATCH] Encode build options as base64 --- .../Services/NmtClearMLBuildJobFactory.cs | 6 +++++- .../Services/SmtTransferClearMLBuildJobFactory.cs | 6 +++++- src/Machine/src/Serval.Machine.Translation/Usings.cs | 1 + .../Services/StatisticalClearMLBuildJobFactory.cs | 6 +++++- .../src/Serval.Machine.WordAlignment/Usings.cs | 1 + .../Services/NmtClearMLBuildJobFactoryTests.cs | 12 +++++++----- .../test/Serval.Machine.Translation.Tests/Usings.cs | 3 ++- 7 files changed, 26 insertions(+), 9 deletions(-) diff --git a/src/Machine/src/Serval.Machine.Translation/Services/NmtClearMLBuildJobFactory.cs b/src/Machine/src/Serval.Machine.Translation/Services/NmtClearMLBuildJobFactory.cs index 0aece6c6..c1c82b26 100644 --- a/src/Machine/src/Serval.Machine.Translation/Services/NmtClearMLBuildJobFactory.cs +++ b/src/Machine/src/Serval.Machine.Translation/Services/NmtClearMLBuildJobFactory.cs @@ -41,7 +41,11 @@ public async Task CreateJobScriptAsync( + $" 'trg_lang': '{trgLang}',\n" + $" 'shared_file_uri': '{baseUri}',\n" + $" 'shared_file_folder': '{folder}',\n" - + (buildOptions is not null ? $" 'build_options': '''{buildOptions}''',\n" : "") + + ( + buildOptions is not null + ? $" 'build_options': '''{Convert.ToBase64String(Encoding.UTF8.GetBytes(buildOptions))}''',\n" + : "" + ) // buildRevision + 1 because the build revision is incremented after the build job // is finished successfully but the file should be saved with the new revision number + (engine.IsModelPersisted ? $" 'save_model': '{engineId}_{engine.BuildRevision + 1}',\n" : $"") diff --git a/src/Machine/src/Serval.Machine.Translation/Services/SmtTransferClearMLBuildJobFactory.cs b/src/Machine/src/Serval.Machine.Translation/Services/SmtTransferClearMLBuildJobFactory.cs index 5cfe488f..7975dd22 100644 --- a/src/Machine/src/Serval.Machine.Translation/Services/SmtTransferClearMLBuildJobFactory.cs +++ b/src/Machine/src/Serval.Machine.Translation/Services/SmtTransferClearMLBuildJobFactory.cs @@ -35,7 +35,11 @@ public async Task CreateJobScriptAsync( + $" 'build_id': '{buildId}',\n" + $" 'shared_file_uri': '{baseUri}',\n" + $" 'shared_file_folder': '{folder}',\n" - + (buildOptions is not null ? $" 'build_options': '''{buildOptions}''',\n" : "") + + ( + buildOptions is not null + ? $" 'build_options': '''{Convert.ToBase64String(Encoding.UTF8.GetBytes(buildOptions))}''',\n" + : "" + ) + $" 'clearml': True\n" + "}\n" + "run(args)\n"; diff --git a/src/Machine/src/Serval.Machine.Translation/Usings.cs b/src/Machine/src/Serval.Machine.Translation/Usings.cs index fb2f912d..e9f2f45f 100644 --- a/src/Machine/src/Serval.Machine.Translation/Usings.cs +++ b/src/Machine/src/Serval.Machine.Translation/Usings.cs @@ -4,6 +4,7 @@ global using System.IO.Compression; global using System.Reflection; global using System.Runtime.CompilerServices; +global using System.Text; global using System.Text.Json; global using System.Text.Json.Nodes; global using System.Text.Json.Serialization; diff --git a/src/Machine/src/Serval.Machine.WordAlignment/Services/StatisticalClearMLBuildJobFactory.cs b/src/Machine/src/Serval.Machine.WordAlignment/Services/StatisticalClearMLBuildJobFactory.cs index 60a7303c..295218d6 100644 --- a/src/Machine/src/Serval.Machine.WordAlignment/Services/StatisticalClearMLBuildJobFactory.cs +++ b/src/Machine/src/Serval.Machine.WordAlignment/Services/StatisticalClearMLBuildJobFactory.cs @@ -35,7 +35,11 @@ public async Task CreateJobScriptAsync( + $" 'build_id': '{buildId}',\n" + $" 'shared_file_uri': '{baseUri}',\n" + $" 'shared_file_folder': '{folder}',\n" - + (buildOptions is not null ? $" 'build_options': '''{buildOptions}''',\n" : "") + + ( + buildOptions is not null + ? $" 'build_options': '''{Convert.ToBase64String(Encoding.UTF8.GetBytes(buildOptions))}''',\n" + : "" + ) + $" 'clearml': True\n" + "}\n" + "run(args)\n"; diff --git a/src/Machine/src/Serval.Machine.WordAlignment/Usings.cs b/src/Machine/src/Serval.Machine.WordAlignment/Usings.cs index 652cc069..5390325a 100644 --- a/src/Machine/src/Serval.Machine.WordAlignment/Usings.cs +++ b/src/Machine/src/Serval.Machine.WordAlignment/Usings.cs @@ -2,6 +2,7 @@ global using System.IO.Compression; global using System.Reflection; global using System.Runtime.CompilerServices; +global using System.Text; global using System.Text.Json; global using System.Text.Json.Nodes; global using System.Text.Json.Serialization; diff --git a/src/Machine/test/Serval.Machine.Translation.Tests/Services/NmtClearMLBuildJobFactoryTests.cs b/src/Machine/test/Serval.Machine.Translation.Tests/Services/NmtClearMLBuildJobFactoryTests.cs index 3c8df41c..d5ed43ff 100644 --- a/src/Machine/test/Serval.Machine.Translation.Tests/Services/NmtClearMLBuildJobFactoryTests.cs +++ b/src/Machine/test/Serval.Machine.Translation.Tests/Services/NmtClearMLBuildJobFactoryTests.cs @@ -7,18 +7,20 @@ public class NmtClearMLBuildJobFactoryTests public async Task CreateJobScriptAsync_BuildOptions() { var env = new TestEnvironment(); + const string BuildOptions = "{ \"max_steps\": \"10\" }"; + string buildOptionsBase64 = Convert.ToBase64String(Encoding.UTF8.GetBytes(BuildOptions)); string script = await env.BuildJobFactory.CreateJobScriptAsync( "engine1", "build1", "test_model", BuildStage.Train, - buildOptions: "{ \"max_steps\": \"10\" }" + buildOptions: BuildOptions ); Assert.That( script, Is.EqualTo( - @"from machine.jobs.build_nmt_engine import run -args = { + $@"from machine.jobs.build_nmt_engine import run +args = {{ 'model_type': 'test_model', 'engine_id': 'engine1', 'build_id': 'build1', @@ -26,9 +28,9 @@ public async Task CreateJobScriptAsync_BuildOptions() 'trg_lang': 'eng_Latn', 'shared_file_uri': 's3://bucket', 'shared_file_folder': 'folder1/folder2', - 'build_options': '''{ ""max_steps"": ""10"" }''', + 'build_options': '''{buildOptionsBase64}''', 'clearml': True -} +}} run(args) ".ReplaceLineEndings("\n") ) diff --git a/src/Machine/test/Serval.Machine.Translation.Tests/Usings.cs b/src/Machine/test/Serval.Machine.Translation.Tests/Usings.cs index fcde2ede..e78738f8 100644 --- a/src/Machine/test/Serval.Machine.Translation.Tests/Usings.cs +++ b/src/Machine/test/Serval.Machine.Translation.Tests/Usings.cs @@ -1,4 +1,5 @@ -global using System.Text.Json; +global using System.Text; +global using System.Text.Json; global using System.Text.Json.Nodes; global using Microsoft.Extensions.DependencyInjection; global using Microsoft.Extensions.Logging;