From e0d41da6c19ec09ae62e8a0ced76d730977271fe Mon Sep 17 00:00:00 2001 From: Adam Korynta Date: Thu, 16 Jul 2026 13:45:50 -0700 Subject: [PATCH 1/4] migrate Java/Jython focused REGI Headless to regi-python via jPype --- build.gradle | 65 +++- regi-headless/build.gradle | 144 +++++++- .../usace/rowcps/headless/CLIOptions.java | 341 ------------------ .../headless/HeadlessRegiDomainFactory.java | 14 +- .../rowcps/headless/PythonEvaluator.java | 72 ---- .../rowcps/headless/PythonJulHandler.java | 47 +++ .../usace/rowcps/headless/PythonLogSink.java | 7 + .../java/usace/rowcps/headless/RegiCLI.java | 152 -------- regi-headless/src/main/python/pyproject.toml | 17 + .../src/main/python/regi_python/__init__.py | 12 + .../main/python/regi_python/regi_python.py | 89 +++++ .../python/regi_python/regi_python_logging.py | 129 +++++++ .../retrieve/xmlmodel/HydroGenData.xsd | 218 ----------- .../java/usace/rowcps/headless/TestFrame.java | 151 -------- .../usace/rowcps/headless/TestHeadless.java | 148 -------- .../rowcps/headless/tests/TestVariables.java | 39 -- .../src/test/python/test_district_scripts.py | 244 +++++++++++++ .../src/test/python/test_regi_python.py | 84 +++++ 18 files changed, 834 insertions(+), 1139 deletions(-) delete mode 100644 regi-headless/src/main/java/usace/rowcps/headless/CLIOptions.java delete mode 100644 regi-headless/src/main/java/usace/rowcps/headless/PythonEvaluator.java create mode 100644 regi-headless/src/main/java/usace/rowcps/headless/PythonJulHandler.java create mode 100644 regi-headless/src/main/java/usace/rowcps/headless/PythonLogSink.java delete mode 100644 regi-headless/src/main/java/usace/rowcps/headless/RegiCLI.java create mode 100644 regi-headless/src/main/python/pyproject.toml create mode 100644 regi-headless/src/main/python/regi_python/__init__.py create mode 100644 regi-headless/src/main/python/regi_python/regi_python.py create mode 100644 regi-headless/src/main/python/regi_python/regi_python_logging.py delete mode 100644 regi-headless/src/main/resources/usace/rowcps/headless/sigstages/retrieve/xmlmodel/HydroGenData.xsd delete mode 100644 regi-headless/src/test/java/usace/rowcps/headless/TestFrame.java delete mode 100644 regi-headless/src/test/java/usace/rowcps/headless/TestHeadless.java delete mode 100644 regi-headless/src/test/java/usace/rowcps/headless/tests/TestVariables.java create mode 100644 regi-headless/src/test/python/test_district_scripts.py create mode 100644 regi-headless/src/test/python/test_regi_python.py diff --git a/build.gradle b/build.gradle index 84613dc..046d611 100644 --- a/build.gradle +++ b/build.gradle @@ -3,14 +3,65 @@ plugins { id "org.sonarqube" version "7.3.1.8318" } -def versionLabel(gitInfo) { - def branch = gitInfo.branchName // all branches are snapshots, only tags get released - def tag = gitInfo.lastTag - // tag is returned as is. Branch may need cleanup - return branch == null ? tag : "99." + branch.replace("/","-") + "-SNAPSHOT" +def gitOutput(String... args) { + return providers.exec { + commandLine(['git'] + args.toList()) + ignoreExitValue = true + }.standardOutput.asText.get().trim() +} + +def normalizeReleaseTag(String tag) { + if (!tag) { + return null + } + + return tag.startsWith('v') ? tag.substring(1) : tag +} + +def pep440VersionFromLatestTag() { + def exactTag = gitOutput('describe', '--tags', '--exact-match') + if (exactTag) { + return normalizeReleaseTag(exactTag) + } + + def tag = gitOutput('describe', '--tags', '--abbrev=0') + if (!tag) { + tag = '0.0.0' + } + + def baseVersion = normalizeReleaseTag(tag) + + def distanceText = tag == '0.0.0' + ? gitOutput('rev-list', 'HEAD', '--count') + : gitOutput('rev-list', "${tag}..HEAD", '--count') + def distance = distanceText?.isInteger() ? distanceText.toInteger() : 0 + + def hash = gitOutput('rev-parse', '--short=7', 'HEAD') + def dirty = gitOutput('status', '--porcelain') ? true : false + + if (distance == 0 && !dirty) { + return baseVersion + } + + def version = "${baseVersion}.post${distance}" + def localParts = [] + + if (hash) { + localParts.add("g${hash}") + } + + if (dirty) { + localParts.add("dirty") + } + + if (!localParts.isEmpty()) { + version += "+" + localParts.join(".") + } + + return version } allprojects { - group = 'mil.army.wmist.regi-headless' - version = versionLabel(versionDetails()) + group = 'mil.army.wmist.regi-python' + version = pep440VersionFromLatestTag() } diff --git a/regi-headless/build.gradle b/regi-headless/build.gradle index c4a6bf1..fe64972 100644 --- a/regi-headless/build.gradle +++ b/regi-headless/build.gradle @@ -1,6 +1,9 @@ +import com.pswidersk.gradle.python.VenvTask + plugins { id 'regi-headless.deps-conventions' id 'regi-headless.java-conventions' + id 'com.pswidersk.python-plugin' version '3.2.16' } dependencies { @@ -45,6 +48,143 @@ jar { } } -test { - useJUnitPlatform() +pythonPlugin { + pythonVersion = "3.11.15" + condaVersion = "26.3.2-2" + condaInstaller = "Miniforge3" + installDir = file(layout.buildDirectory.dir("python")) +} + +tasks.register('installPythonBuildTools', VenvTask) { + group = 'build setup' + description = 'Installs Python packages needed to build and test the wheel.' + + venvExec = 'pip' + args = ['install', '--upgrade', 'pip', 'build', 'pytest'] + + outputs.file(layout.buildDirectory.file("python-build-tools/install.marker")) + + doLast { + def markerFile = layout.buildDirectory.file("python-build-tools/install.marker").get().asFile + markerFile.parentFile.mkdirs() + markerFile.text = "pip build pytest installed\n" + } +} + +tasks.register('bundlePython', Sync) { + description = 'Bundles Python scripts and creates the java_lib directory' + + dependsOn jar + + into layout.buildDirectory.dir("install/regi_python") + + inputs.dir("src/main/python") + inputs.file(jar.archiveFile) + inputs.files(configurations.runtimeClasspath) + inputs.property("version", project.version.toString()) + + from('src/main/python') { + include 'pyproject.toml' + filter { line -> line.replaceAll('@VERSION@', project.version.toString()) } + } + + from('src/main/python') { + exclude 'pyproject.toml' + } + + into('regi_python/lib') { + from configurations.runtimeClasspath + from jar.archiveFile + exclude "**/*.nbm" + } +} + +tasks.register('buildPythonWheel', VenvTask) { + group = "distribution" + description = "Builds a Python .whl file using the Gradle-managed Python environment." + + dependsOn bundlePython + dependsOn installPythonBuildTools + + workingDir = layout.buildDirectory.dir("install/regi_python").get().asFile + venvExec = "python" + args = ["-m", "build", "--wheel"] + + inputs.files(fileTree(layout.buildDirectory.dir("install/regi_python")) { + exclude "dist/**" + exclude "build/**" + exclude "*.egg-info/**" + }) + inputs.property("version", project.version.toString()) + outputs.dir(layout.buildDirectory.dir("install/regi_python/dist")) + + doFirst { + delete layout.buildDirectory.dir("install/regi_python/dist") + } + + doLast { + println "Python Wheel built in: ${workingDir}/dist" + } +} + +tasks.register('installPythonWheelForSmokeTest', VenvTask) { + group = 'verification' + description = 'Installs the built Python wheel into the Gradle-managed Python environment.' + + dependsOn buildPythonWheel + + venvExec = 'pip' + + inputs.files(fileTree(dir: "${buildDir}/install/regi_python/dist", include: '*.whl')) + outputs.file(layout.buildDirectory.file("test-python-wheel/install.marker")) + + doFirst { + def wheelFile = fileTree(dir: "${buildDir}/install/regi_python/dist", include: '*.whl').singleFile + args = ['install', '--force-reinstall', wheelFile.absolutePath] + } + + doLast { + def wheelFile = fileTree(dir: "${buildDir}/install/regi_python/dist", include: '*.whl').singleFile + def markerFile = layout.buildDirectory.file("test-python-wheel/install.marker").get().asFile + markerFile.parentFile.mkdirs() + markerFile.text = "${wheelFile.name}\n${wheelFile.length()} bytes\n" + } +} +tasks.register('testPythonWheel', VenvTask) { + group = 'verification' + description = 'Runs pytest against the installed Python wheel.' + + dependsOn installPythonWheelForSmokeTest + + venvExec = 'python' + args = ['-m', 'pytest', 'src/test/python'] + + inputs.files(fileTree(dir: 'src/test/python', include: '**/*.py')) + outputs.upToDateWhen { false } +} + +tasks.register('smokeTestDistrictScripts', VenvTask) { + group = 'verification' + description = 'Validates migrated district and example scripts against the Python entrypoint shape and Java scriptable APIs.' + + dependsOn installPythonBuildTools + + venvExec = 'python' + args = [ + '-m', 'pytest', + '-o', 'log_cli=true', + '--log-cli-level=INFO', + 'src/test/python/test_district_scripts.py' + ] + + inputs.files(fileTree(dir: '../district-scripts', include: '**/*.py')) + inputs.files(fileTree(dir: 'src/test/resources/usace/rowcps/headless/examples', include: '**/*.py')) + inputs.files(fileTree(dir: 'src/main/java/usace/rowcps/headless', include: '**/*.java')) + inputs.file('src/test/python/test_district_scripts.py') + outputs.upToDateWhen { false } +} + +check { + dependsOn testPythonWheel + dependsOn smokeTestDistrictScripts } diff --git a/regi-headless/src/main/java/usace/rowcps/headless/CLIOptions.java b/regi-headless/src/main/java/usace/rowcps/headless/CLIOptions.java deleted file mode 100644 index bf6a1f6..0000000 --- a/regi-headless/src/main/java/usace/rowcps/headless/CLIOptions.java +++ /dev/null @@ -1,341 +0,0 @@ -package usace.rowcps.headless; - -import hec.lang.PasswordFileEntry; -import java.io.BufferedReader; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileReader; -import java.io.IOException; -import java.util.Map; -import java.util.Properties; -import java.util.Set; -import java.util.TimeZone; -import java.util.logging.Level; -import java.util.logging.Logger; -import org.kohsuke.args4j.CmdLineException; -import org.kohsuke.args4j.Option; -import rma.services.ServiceLookup; -import rma.services.tz.TimeZoneDisplayService; - -/** - * - * @author ryan - */ -public class CLIOptions -{ - private static final Logger logger = Logger.getLogger(CLIOptions.class.getName()); - //public String rowcpsTimezone; - //public String oracleUrl; - //public String oracleUser; - //public String oraclePassword; - //private Map properties = new HashMap(); - Properties props; - - public final static String URL = "oracle.url"; - public final static String USER = "oracle.user"; - public final static String PASSWORD = "oracle.password"; - public final static String OFFICEID = "oracle.officeId"; - - public final static String TIMEZONE = "rowcps.timezone"; - public final static String PROJ_DIR = "rowcps.projectDir"; - public final static String PROJ_NAME = "rowcps.projectName"; - - public final static String HEC_PASSWD_FILE = "hec.passwd"; - - public CLIOptions() - { - this(System.getProperties()); - } - - public CLIOptions(Properties defaultProperties) - { - props = new Properties(defaultProperties); - } - - @Option(name = "-D", metaVar = "=", usage = "use value for given property") - private void setProperty(final String property) throws CmdLineException - { - String[] arr = property.split("="); - setProperty(arr); - } - - public void setProperty(String[] arr) throws CmdLineException - { - if (arr.length != 2) { - throw new CmdLineException("Properties must be specified in the form:" + - "="); - } - props.setProperty(arr[0], arr[1]); - //properties.put(arr[0], arr[1]); - } - - public Object getProperty(String key) - { - return props.get(key); - } - - /** - * @return the rowcpsTimezone - */ - public String getRowcpsTimezone() - { - return props.getProperty(TIMEZONE); - //return rowcpsTimezone; - } - - public File getRowcpsProjectDir() - { - File retval = null; - - String path = props.getProperty(PROJ_DIR); - if(path != null){ - retval = new File(path); - } - - return retval; - } - - @Option(name = "-D" + PROJ_DIR, metaVar = "", usage = "directory containing Regi project") - public void setRowcpsProjectDir(String filepath) - { - props.setProperty(PROJ_DIR, filepath); - } - - @Option(name = "-D" + HEC_PASSWD_FILE, metaVar = "", usage = "directory containing Regi project") - public void setHecPasswordFilepath(String filepath) - { - props.setProperty(HEC_PASSWD_FILE, filepath); - } - - public String getHecPasswordFilepath(){ - return props.getProperty(HEC_PASSWD_FILE); - } - - public PasswordFileEntry getHecPasswordFileEntry() - { - PasswordFileEntry retval = null; - - // String office = System.getProperty("cwms.dbi.OfficeId"); - String dburl = getOracleUrl(); - if (dburl != null && !dburl.isEmpty()) { - String instance = dburl; - - int idx = dburl.indexOf("@"); - if (idx != -1) { - instance = dburl.substring(idx + 1); - } - - hec.io.PasswordFile passwordFile = null; - try { - String filePath = getHecPasswordFilepath(); - passwordFile = new hec.io.PasswordFile(filePath, false); - retval = passwordFile.getEntry(instance); - - if (retval == null) { - /* - * System.out.println( - * "getConnectionInfo: Failed to find Password Entry for instance " - * + instance); - */ - logger.severe("getConnectionInfo: Failed to find Password Entry for instance " + instance); - - } -// // _connectionInfo = new -// // ConnectionInfo(office,dburl,entry.getUserName(),entry.getPassword()); -// _connectionLoginInfo = new ConnectionLoginInfoImpl(dburl, entry.getUserName(), entry.getPassword(), -// getOfficeId()); - } catch (java.io.IOException ioe) { - /* - * System.out.println( - * "getConnectionInfo: Error reading password file " + ioe); - */ - logger.severe("getConnectionInfo: Error reading password file " + ioe); - - } finally { - if (passwordFile != null) { - passwordFile.close(); - } - } - } - - return retval; - } - - - public String getRowcpsProjectName() - { - return props.getProperty(PROJ_NAME); - } - - @Option(name = "-D" + PROJ_NAME, usage = "name of Regi project") - public void setRowcpsProjectName(String name) - { - props.setProperty(PROJ_NAME, name); - } - - public String getOracleOfficeId() - { - return props.getProperty(OFFICEID); - } - - @Option(name = "-D" + OFFICEID, usage = "office id") - public void setOracleOfficeId(String id) - { - props.setProperty(OFFICEID, id); - } - - /** - * @param rowcpsTimezone the rowcpsTimezone to set - */ - @Option(name = "-D" + TIMEZONE) - public void setRowcpsTimezone(String rowcpsTimezone) throws CmdLineException - { - setProperty(new String[]{TIMEZONE, rowcpsTimezone}); - TimeZone timeZone = TimeZone.getTimeZone(rowcpsTimezone); - if(timeZone == null) - { - timeZone = TimeZone.getDefault(); - Logger.getLogger(CLIOptions.class.getName()).log(Level.WARNING, "Attempted to set invalid time zone to Regi Domain: "+rowcpsTimezone); - } - TimeZoneDisplayService timeZoneDisplayService = ServiceLookup.getTimeZoneDisplayService(); - timeZoneDisplayService.setTimeZone(timeZone); - } - - /** - * @return the oracleUrl - */ - public String getOracleUrl() - { - return props.getProperty(URL); - //return oracleUrl; - } - - /** - * @param oracleUrl the oracleUrl to set - */ - @Option(name = "-D" + URL) - public void setOracleUrl(String oracleUrl) throws CmdLineException - { - //this.oracleUrl = oracleUrl; - setProperty(new String[]{URL, oracleUrl}); - } - - /** - * @return the oracleUser - */ - public String getOracleUser() - { - String user = props.getProperty(USER); - - if (user == null) { - - PasswordFileEntry entry = getHecPasswordFileEntry(); - if (entry != null) { - user = entry.getUserName(); - } - - } - return user; - //return oracleUser; - } - - /** - * @param oracleUser the oracleUser to set - */ - @Option(name = "-D" + USER) - public void setOracleUser(String oracleUser) throws CmdLineException - { - //this.oracleUser = oracleUser; - setProperty(new String[]{USER, oracleUser}); - } - - /** - * @return the oraclePassword - */ - public char[] getOraclePassword() - { - char[] pass = null; - //return oraclePassword; - String passStr = props.getProperty(PASSWORD); - if (passStr != null) { - pass = passStr.toCharArray(); - } else { - PasswordFileEntry entry = getHecPasswordFileEntry(); - if(entry != null){ - pass = entry.getPassword().toCharArray(); - } - } - - return pass; - } - - /** - * @param oraclePassword the oraclePassword to set - */ - @Option(name = "-D" + PASSWORD) - public void setOraclePassword(String oraclePassword) throws CmdLineException - { - setProperty(new String[]{PASSWORD, oraclePassword}); - //this.oraclePassword = oraclePassword; - } - - @Option(name = "-p", aliases = {"-properties"}, metaVar = "", - usage = "import properties from given file") - public void importProperties(File file) - { - if (file != null && file.exists()) { - Properties fileProps = new Properties(); - - try (BufferedReader br = new BufferedReader(new FileReader(file))) { - fileProps.load(br); - - Set> entrySet = fileProps.entrySet(); - for (Map.Entry entry : entrySet) { - Object keyObj = entry.getKey(); - Object valueObj = entry.getValue(); - - if (keyObj != null && valueObj != null) { - props.put(keyObj, valueObj); - } - } - } catch (FileNotFoundException ex) { - Logger.getLogger(CLIOptions.class.getName()).log(Level.SEVERE, null, ex); - } catch (IOException ex) { - Logger.getLogger(CLIOptions.class.getName()).log(Level.SEVERE, null, ex); - } - } - else{ - Logger.getLogger(CLIOptions.class.getName()).log(Level.SEVERE, "Unable to find credentials file at: "+(file ==null ? "null" : file)); - } - } - - @Option(name = "-f", aliases = {"-file"}, metaVar = "", - usage = "script file to execute") - public void setScriptFile(File file) - { - props.setProperty("script", file.getAbsolutePath()); - } - - public String getScriptPath() - { - return props.getProperty("script"); - } - - public File getScriptFile() - { - File retval = null; - String scriptPath = getScriptPath(); - if (scriptPath != null && !scriptPath.isEmpty()) { - File afile = new File(scriptPath); - if (afile.exists()) { - retval = afile; - } - } - return retval; - } - - Properties getProperties() { - return new Properties(props); - } - -} diff --git a/regi-headless/src/main/java/usace/rowcps/headless/HeadlessRegiDomainFactory.java b/regi-headless/src/main/java/usace/rowcps/headless/HeadlessRegiDomainFactory.java index b2cff43..1b4f073 100644 --- a/regi-headless/src/main/java/usace/rowcps/headless/HeadlessRegiDomainFactory.java +++ b/regi-headless/src/main/java/usace/rowcps/headless/HeadlessRegiDomainFactory.java @@ -27,10 +27,6 @@ import usace.rowcps.regi.model.ManagerId; import usace.rowcps.regi.model.RegiDomain; -/** - * - * @author ryan - */ public class HeadlessRegiDomainFactory { @@ -40,11 +36,11 @@ public class HeadlessRegiDomainFactory public RegiDomain createDomain() throws DbConnectionException, DbPluginNotFoundException, IOException { - Path projectDir = Paths.get("regi-projects", "regi-cli"); + Path projectDir = Paths.get("regi-projects", "regi-python"); logger.log(Level.INFO, "Creating project dir: "+ projectDir); Files.createDirectories(projectDir); - Path projectFile = projectDir.resolve("regi-cli.prj"); + Path projectFile = projectDir.resolve("regi-python.prj"); if(!Files.exists(projectFile)) { Files.createFile(projectFile); } @@ -66,15 +62,15 @@ public RegiDomain createDomain() throws DbConnectionException, } String cdaUrl = System.getenv("CDA_URL"); - String apiKey = System.getenv("API_KEY"); + String apiKey = System.getenv("CDA_API_KEY"); String officeId = System.getenv("OFFICE_ID"); CdaAuthenticationSource cdaAuthenticationSource = new CdaAuthenticationSource("", cdaUrl, officeId, new CwmsApiKeyAuthExtension(apiKey)); try { - ServerSuite serverSuite = ServerSuiteUtil.login("REGI CLI", cdaAuthenticationSource, false, false, false); + ServerSuite serverSuite = ServerSuiteUtil.login("regi-python", cdaAuthenticationSource, false, false, false); DataAccessFactory dataAccessFactory = serverSuite.getDataAccessFactory(); - try(var key = dataAccessFactory.getDataAccessKey("REGI CLI")) { + try(var key = dataAccessFactory.getDataAccessKey("regi-python")) { String username = dataAccessFactory.getDao(CwmsSecurityDao.class).getCurrentUserId(key); connectionManager.setUsername(username); } diff --git a/regi-headless/src/main/java/usace/rowcps/headless/PythonEvaluator.java b/regi-headless/src/main/java/usace/rowcps/headless/PythonEvaluator.java deleted file mode 100644 index a40a329..0000000 --- a/regi-headless/src/main/java/usace/rowcps/headless/PythonEvaluator.java +++ /dev/null @@ -1,72 +0,0 @@ -package usace.rowcps.headless; - -import usace.rowcps.headless.interfaces.ScriptEvaluator; -import java.io.Reader; -import java.util.Map; -import javax.script.Bindings; -import javax.script.ScriptEngine; -import javax.script.ScriptEngineManager; -import javax.script.ScriptException; -import javax.script.SimpleScriptContext; - -/** - * - * @author ryan - */ -public class PythonEvaluator implements ScriptEvaluator -{ - - public final static String ENGINE_NAME = "python"; - private static final String REFERENCE_ERROR = "ReferenceError"; - private static final String NAME_ERROR = "NameError"; - - @Override - public Object evaluateExpression(Reader reader, Map variables) - { - - // I think this would let us restrict the classes loadable by python. - ClassLoader ctxtLoader = Thread.currentThread().getContextClassLoader(); - - ScriptEngineManager scriptManager = new ScriptEngineManager(ctxtLoader); - ScriptEngine pythonEngine = scriptManager.getEngineByName(ENGINE_NAME); - - SimpleScriptContext context = (SimpleScriptContext) pythonEngine.getContext(); - - Object javaValue = null; - try { - Bindings bindings = populateBingings(pythonEngine, variables); // fyi bindings actually SimpleBindings - javaValue = pythonEngine.eval(reader, bindings); - } catch (ScriptException e) { - handleException(e, variables); - } - return javaValue; - } - - private static Bindings populateBingings(ScriptEngine engine, Map variables) - { - Bindings bindings = engine.createBindings(); - for (Map.Entry entrySet : variables.entrySet()) { - String name = entrySet.getKey(); - Object value = entrySet.getValue(); - bindings.put(name, value); - } - - return bindings; - } - - private static Object handleException(ScriptException exception, Map variables) - { - if (isReferenceError(exception)) { - throw new IllegalArgumentException("Couldn't resolve some variables in expression with vars " + variables. - keySet(), exception); - } - throw new RuntimeException(exception); - } - - private static boolean isReferenceError(ScriptException exception) - { - String message = exception.getMessage(); - return message.startsWith(REFERENCE_ERROR); - } - -} diff --git a/regi-headless/src/main/java/usace/rowcps/headless/PythonJulHandler.java b/regi-headless/src/main/java/usace/rowcps/headless/PythonJulHandler.java new file mode 100644 index 0000000..532f277 --- /dev/null +++ b/regi-headless/src/main/java/usace/rowcps/headless/PythonJulHandler.java @@ -0,0 +1,47 @@ +package usace.rowcps.headless; + +import java.util.logging.ErrorManager; +import java.util.logging.Formatter; +import java.util.logging.Handler; +import java.util.logging.LogRecord; + +public final class PythonJulHandler extends Handler { + + private final PythonLogSink sink; + + private final Formatter messageFormatter = new Formatter() { + @Override + public String format(LogRecord record) { + return formatMessage(record); + } + }; + + public PythonJulHandler(PythonLogSink sink) { + this.sink = sink; + } + + @Override + public void publish(LogRecord record) { + if (record == null || !isLoggable(record)) { + return; + } + + try { + String message = messageFormatter.format(record); + sink.log(record, message); + } catch (RuntimeException ex) { + // Safeguard against failures in Python log handling. + reportError("Failed to publish JUL record to Python logging.", ex, ErrorManager.WRITE_FAILURE); + } + } + + @Override + public void flush() { + // No-op. Python logging handlers manage their own flushing. + } + + @Override + public void close() { + // No-op. + } +} diff --git a/regi-headless/src/main/java/usace/rowcps/headless/PythonLogSink.java b/regi-headless/src/main/java/usace/rowcps/headless/PythonLogSink.java new file mode 100644 index 0000000..70a2ea0 --- /dev/null +++ b/regi-headless/src/main/java/usace/rowcps/headless/PythonLogSink.java @@ -0,0 +1,7 @@ +package usace.rowcps.headless; + +import java.util.logging.LogRecord; + +public interface PythonLogSink { + void log(LogRecord record, String message); +} diff --git a/regi-headless/src/main/java/usace/rowcps/headless/RegiCLI.java b/regi-headless/src/main/java/usace/rowcps/headless/RegiCLI.java deleted file mode 100644 index 17dc332..0000000 --- a/regi-headless/src/main/java/usace/rowcps/headless/RegiCLI.java +++ /dev/null @@ -1,152 +0,0 @@ -package usace.rowcps.headless; - -import java.io.IOException; -import usace.rowcps.headless.interfaces.ScriptEvaluator; -import hec.db.DbConnectionException; -import hec.db.DbIoException; -import hec.db.DbPluginNotFoundException; -import hec.db.InvalidDbConnectionException; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileReader; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.logging.Level; -import java.util.logging.Logger; -import org.kohsuke.args4j.CmdLineException; -import org.kohsuke.args4j.CmdLineParser; -import usace.rowcps.metrics.RegiMetricsService; -import usace.rowcps.regi.factories.RowcpsExecutorService; -import usace.rowcps.regi.model.ManagerId; -import usace.rowcps.regi.model.RegiDomain; -import usace.rowcps.regi.preferences.RegiPreferences; - -/** - * - * @author ryan - */ -public class RegiCLI -{ - - private static final Logger LOGGER = Logger.getLogger(RegiCLI.class.getName()); - - static - { - RegiMetricsService.init(RegiPreferences.getClientNode().node("Metrics"), "REGI Headless"); - } - - public static void main(String[] args) - { - CLIOptions opt = new CLIOptions(System.getProperties()); - CmdLineParser parser = new CmdLineParser(opt); - - try - { - runHeadless(parser, args, opt); - } - catch (DbConnectionException | DbPluginNotFoundException | InvalidDbConnectionException | IOException ex) - { - LOGGER.log(Level.SEVERE, "Headless error connecting to database.", ex); - System.exit(-1); - return; - } - catch (CmdLineException | RuntimeException e) - { - LOGGER.log(Level.SEVERE, "Error running headless", e); - System.err.println("java -jar myprogram.jar [options...] arguments..."); - parser.printUsage(System.err); - System.exit(-1); - return; - } - - LOGGER.info("Exiting."); - System.exit(0); - } - - /** - * Used by TestHeadless unit test class to run headless without calling System.exit(0) - * - * @param args - * @throws DbConnectionException - * @throws InvalidDbConnectionException - * @throws CmdLineException - * @throws DbPluginNotFoundException - */ - static void runHeadlessTest(String[] args) - throws DbConnectionException, InvalidDbConnectionException, CmdLineException, DbPluginNotFoundException, - IOException { - CLIOptions opt = new CLIOptions(System.getProperties()); - CmdLineParser parser = new CmdLineParser(opt); - runHeadless(parser, args, opt); - } - - private static void runHeadless(CmdLineParser parser, String[] args, CLIOptions opt) - throws DbConnectionException, InvalidDbConnectionException, CmdLineException, DbPluginNotFoundException, - IOException { - parser.parseArgument(args); - System.setProperties(opt.getProperties()); - - HeadlessRegiDomainFactory factory = new HeadlessRegiDomainFactory(); - ManagerId managerId = factory.getManagerId(); - RegiDomain regiDomain = factory.createDomain(); - - if (regiDomain != null) - { - ScriptEvaluator pe = new PythonEvaluator(); - Map vars = new HashMap<>(); - - RegiCalcRegistry reg = new RegiCalcRegistry(regiDomain, managerId); - vars.put("registry", reg); - - File scriptFile = opt.getScriptFile(); - - try - { - FileReader fr = new FileReader(scriptFile); - LOGGER.info("Evaluating script file"); - Object retval = pe.evaluateExpression(fr, vars); - - LOGGER.info("Jython script completed normally, commiting data."); - regiDomain.commitData(managerId); - LOGGER.info("RegiDomain committed data."); - } - catch (DbConnectionException | DbIoException | FileNotFoundException ex) - { - LOGGER.log(Level.SEVERE, "Exception occurred while evaluating Jython file:" + System.lineSeparator() + scriptFile, ex); - } - finally - { - LOGGER.info("RegiDomain closing."); - shutdownRowcpsAccessFactory(managerId); - regiDomain.closing(); - } - } - } - - private static void shutdownRowcpsAccessFactory(ManagerId managerId) - { - LOGGER.log(Level.INFO, "Shutting down RowcpsExecutorService for {0}", managerId); - RowcpsExecutorService res = RowcpsExecutorService.getInstance(managerId); - - res.shutdown(); // signal shutdown - this will stop accepting new jobs and allow existing jobs to complete. - boolean exitted = false; - try - { - // We are willing to wait a little to achieve a clean shutdown. - exitted = res.awaitTermination(3000, TimeUnit.MILLISECONDS); - - } - catch (InterruptedException ie) - { - Thread.currentThread().interrupt(); - } - if (!exitted) - { - // Some of the running tasks didn't exit in the time we were willing to wait. - List wereWaiting = res.shutdownNow(); // This will interrupt them if they support interruption. - } - LOGGER.log(Level.INFO, "RowcpsExecutorService for {0} shutdown complete.", managerId); - } -} diff --git a/regi-headless/src/main/python/pyproject.toml b/regi-headless/src/main/python/pyproject.toml new file mode 100644 index 0000000..48db039 --- /dev/null +++ b/regi-headless/src/main/python/pyproject.toml @@ -0,0 +1,17 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "regi-python" +version = "@VERSION@" +description = "USACE REGI Python Bridge" +dependencies = [ + "JPype1>=1.4.1", +] + +[tool.setuptools] +packages = ["regi_python"] + +[tool.setuptools.package-data] +"regi_python" = ["lib/*.jar"] \ No newline at end of file diff --git a/regi-headless/src/main/python/regi_python/__init__.py b/regi-headless/src/main/python/regi_python/__init__.py new file mode 100644 index 0000000..9bc0c71 --- /dev/null +++ b/regi-headless/src/main/python/regi_python/__init__.py @@ -0,0 +1,12 @@ +"""Public package surface for the REGI Python bridge.""" + +from .regi_python import regi_session, run_headless +from importlib.metadata import version, PackageNotFoundError + +__all__ = ["regi_session", "run_headless"] + +try: + __version__ = version("regi_python") +except PackageNotFoundError: + # package is not installed + __version__ = "unknown" diff --git a/regi-headless/src/main/python/regi_python/regi_python.py b/regi-headless/src/main/python/regi_python/regi_python.py new file mode 100644 index 0000000..f0a5ac3 --- /dev/null +++ b/regi-headless/src/main/python/regi_python/regi_python.py @@ -0,0 +1,89 @@ +"""Runtime bridge for executing REGI calculations from Python.""" + +import os +import jpype +import jpype.imports +from contextlib import contextmanager +from .regi_python_logging import configure_logging, configure_jul_to_python_logging + +logger = configure_logging() + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +LIB_PATH = os.path.join(BASE_DIR, "lib", "*") + +@contextmanager +def regi_session(): + """ + Start the JVM on entry and shut it down when the context exits. + """ + started_jvm = False + if not jpype.isJVMStarted(): + _prepend_java_home_to_path() + logger.info("Starting JVM...") + jpype.startJVM( + jpype.getDefaultJVMPath(), + convertStrings=True, + classpath=[LIB_PATH] + ) + configure_jul_to_python_logging(logger) + started_jvm = True + + try: + yield + finally: + if started_jvm and jpype.isJVMStarted(): + logger.info("Shutting down JVM...") + jpype.shutdownJVM() + +def run_headless(calculation_callback): + """Run a callback against a headless REGI domain.""" + _require_environment_variables("CDA_URL", "CDA_API_KEY", "OFFICE_ID") + + # Import these only after the JVM has started. + from usace.rowcps.headless import HeadlessRegiDomainFactory, RegiCalcRegistry + from usace.rowcps.regi.factories import RowcpsExecutorService + from java.util.concurrent import TimeUnit + factory = HeadlessRegiDomainFactory() + logger.info("Attempting to create RegiDomain...") + regi_domain = factory.createDomain() + manager_id = factory.getManagerId() + registry = RegiCalcRegistry(regi_domain, manager_id) + + try: + logger.info("Executing callback...") + calculation_callback(registry) + regi_domain.commitData(manager_id) + except Exception as e: + logger.error("Execution failed.", exc_info=True) + raise + finally: + _shutdown_executor(manager_id) + regi_domain.closing() + + +def _require_environment_variables(*variable_names): + missing = [name for name in variable_names if not os.environ.get(name)] + if missing: + raise RuntimeError( + "Missing required environment variables: " + ", ".join(missing) + ) + + +def _prepend_java_home_to_path(): + java_home = os.environ.get("JAVA_HOME") + if not java_home: + return + + java_bin = os.path.join(java_home, "bin") + path = os.environ.get("PATH", "") + if java_bin not in path: + os.environ["PATH"] = java_bin + os.pathsep + path + + +def _shutdown_executor(manager_id): + from usace.rowcps.regi.factories import RowcpsExecutorService + from java.util.concurrent import TimeUnit + res = RowcpsExecutorService.getInstance(manager_id) + res.shutdown() + if not res.awaitTermination(3000, TimeUnit.MILLISECONDS): + res.shutdownNow() diff --git a/regi-headless/src/main/python/regi_python/regi_python_logging.py b/regi-headless/src/main/python/regi_python/regi_python_logging.py new file mode 100644 index 0000000..e0b6099 --- /dev/null +++ b/regi-headless/src/main/python/regi_python/regi_python_logging.py @@ -0,0 +1,129 @@ +"""Logging helpers for the REGI Python bridge.""" + +import os +import logging +from jpype import JImplements, JOverride + +_java_log_sink = None + + +def _get_log_level(): + log_level_name = os.environ.get("REGI_LOG_LEVEL", "INFO").upper() + log_level = getattr(logging, log_level_name, None) + invalid_log_level = not isinstance(log_level, int) + if invalid_log_level: + log_level = logging.INFO + return log_level, invalid_log_level, log_level_name + + +def configure_logging(): + """Configure the bridge logger.""" + log_level, invalid_log_level, log_level_name = _get_log_level() + + log_format = os.environ.get( + "REGI_LOG_FORMAT", + "%(asctime)s %(levelname)s %(name)s " + "[job=%(aws_batch_job_id)s attempt=%(aws_batch_job_attempt)s] - %(message)s", + ) + + class AwsBatchFilter(logging.Filter): + def filter(self, record): + record.aws_batch_job_id = os.environ.get("AWS_BATCH_JOB_ID", "-") + record.aws_batch_job_attempt = os.environ.get("AWS_BATCH_JOB_ATTEMPT", "-") + return True + + handler = logging.StreamHandler() + handler.setLevel(log_level) + handler.setFormatter(logging.Formatter(log_format)) + handler.addFilter(AwsBatchFilter()) + + logger = logging.getLogger("regi-launcher") + logger.setLevel(log_level) + logger.handlers.clear() + logger.addHandler(handler) + logger.propagate = False + + if invalid_log_level: + logger.warning("Invalid REGI_LOG_LEVEL '%s'; using INFO.", log_level_name) + + return logger + + +def configure_jul_to_python_logging(python_logger): + """Forward Java JUL records into Python logging.""" + global _java_log_sink + + from java.util.logging import Logger + from usace.rowcps.headless import PythonJulHandler + + java_level = _python_level_to_jul_level(python_logger.getEffectiveLevel()) + + root_logger = Logger.getLogger("") + root_logger.setLevel(java_level) + + for handler in root_logger.getHandlers(): + root_logger.removeHandler(handler) + + PythonLogSink = _create_python_log_sink_class() + _java_log_sink = PythonLogSink(python_logger) + + handler = PythonJulHandler(_java_log_sink) + handler.setLevel(java_level) + + root_logger.addHandler(handler) + + +def _python_level_to_jul_level(python_level): + from java.util.logging import Level + + if python_level <= logging.NOTSET: + return Level.ALL + if python_level <= logging.DEBUG: + return Level.FINE + if python_level <= logging.INFO: + return Level.INFO + if python_level <= logging.WARNING: + return Level.WARNING + if python_level <= logging.CRITICAL: + return Level.SEVERE + return Level.OFF + + +def _jul_level_to_python_level(jul_level_name): + mapping = { + "SEVERE": logging.ERROR, + "WARNING": logging.WARNING, + "INFO": logging.INFO, + "CONFIG": logging.INFO, + "FINE": logging.DEBUG, + "FINER": logging.DEBUG, + "FINEST": logging.DEBUG, + "ALL": logging.NOTSET, + "OFF": logging.CRITICAL + 10, + } + return mapping.get(str(jul_level_name), logging.INFO) + + +def _create_python_log_sink_class(): + # JPype resolves @JImplements interfaces immediately, so define this class only + # after the JVM has started; otherwise importing this module would fail. + from jpype import JImplements, JOverride + + @JImplements("usace.rowcps.headless.PythonLogSink") + class PythonLogSink: + def __init__(self, python_logger): + self._logger = python_logger + + @JOverride + def log(self, record, message): + level = _jul_level_to_python_level(record.getLevel().getName()) + name = str(record.getLoggerName() or "java") + message = str(message) + + thrown = record.getThrown() + if thrown is not None: + message = f"{message}\n{thrown}" + + self._logger.log(level, "[%s] %s", name, message) + + return PythonLogSink diff --git a/regi-headless/src/main/resources/usace/rowcps/headless/sigstages/retrieve/xmlmodel/HydroGenData.xsd b/regi-headless/src/main/resources/usace/rowcps/headless/sigstages/retrieve/xmlmodel/HydroGenData.xsd deleted file mode 100644 index e1bb1ea..0000000 --- a/regi-headless/src/main/resources/usace/rowcps/headless/sigstages/retrieve/xmlmodel/HydroGenData.xsd +++ /dev/null @@ -1,218 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/regi-headless/src/test/java/usace/rowcps/headless/TestFrame.java b/regi-headless/src/test/java/usace/rowcps/headless/TestFrame.java deleted file mode 100644 index 2b46ad7..0000000 --- a/regi-headless/src/test/java/usace/rowcps/headless/TestFrame.java +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Copyright (c) 2018 - * United States Army Corps of Engineers - Hydrologic Engineering Center (USACE/HEC) - * All Rights Reserved. USACE PROPRIETARY/CONFIDENTIAL. - * Source may not be released without written approval from HEC - */ -package usace.rowcps.headless; - -import java.awt.GridLayout; -import java.awt.HeadlessException; -import java.awt.event.ActionEvent; -import java.io.IOException; -import java.nio.file.FileVisitOption; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.List; -import java.util.ArrayList; -import static java.util.stream.Collectors.toList; -import javax.swing.AbstractAction; -import javax.swing.JButton; -import javax.swing.JFrame; -import javax.swing.SwingUtilities; - -/** - * - * @author @author Ryan A. Miles (ryanm@rmanet.com) - */ -public class TestFrame extends JFrame -{ - private static final Logger LOGGER = Logger.getLogger(TestFrame.class.getName()); - private final ExecutorService _executor = Executors.newSingleThreadExecutor(); - private final List _buttons = new ArrayList<>(); - - public TestFrame() throws HeadlessException - { - buildComponents(); - } - - public static void main(String[] args) - { - TestFrame frame = new TestFrame(); - frame.pack(); - frame.setDefaultCloseOperation(EXIT_ON_CLOSE); - frame.setMinimumSize(frame.getPreferredSize()); - frame.setVisible(true); - } - - private void buildComponents() - { - setLayout(new GridLayout(0, 3, 2, 2)); - - String[] files = readFilesInTestFolder(); - - for (String testData : files) - { - JButton btn = new JButton(new ButtonAction(testData, testData)); - _buttons.add(btn); - add(btn); - } - } - - private String[] readFilesInTestFolder() - { - Path path = Paths.get(TestHeadless.getJythonTestFolder()); - List paths = new ArrayList<>(); - try - { - paths.addAll(Files.walk(path, FileVisitOption.FOLLOW_LINKS) - .map(Path::getFileName) - .map(Path::toString) - .filter(file -> file.endsWith("py")) - .collect(toList())); - } - catch (IOException | RuntimeException ex) - { - - } - - return paths.toArray(new String[0]); - } - - private void performHeadless(String file) - { - disableButtons(); - - _executor.submit(() -> - { - String[] args = TestHeadless.getArgsForFile(file); - try - { - RegiCLI.runHeadlessTest(args); - //Reset logging options - LoggingOptions.disableFlowGroupCompLogging(); - LoggingOptions.setMetricsEnabled(false); - LoggingOptions.setDbMessageLevel(0); - } - catch (Throwable ex) - { - logThrowable(ex); - } - - SwingUtilities.invokeLater(this::enableButtons); - }); - } - - private void logThrowable(Throwable ex) - { - if (ex.getCause() != null) - { - logThrowable(ex.getCause()); - } - LOGGER.log(Level.SEVERE, "Exception occurred", ex); - } - - private void disableButtons() - { - for (JButton btn : _buttons) - { - btn.setEnabled(false); - } - } - - private void enableButtons() - { - for (JButton btn : _buttons) - { - btn.setEnabled(true); - } - } - - private class ButtonAction extends AbstractAction - { - private final String _file; - public ButtonAction(String name, String file) - { - super(name); - _file = file; - } - - @Override - public void actionPerformed(ActionEvent e) - { - performHeadless(_file); - } - } -} diff --git a/regi-headless/src/test/java/usace/rowcps/headless/TestHeadless.java b/regi-headless/src/test/java/usace/rowcps/headless/TestHeadless.java deleted file mode 100644 index 4cf8ac7..0000000 --- a/regi-headless/src/test/java/usace/rowcps/headless/TestHeadless.java +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Copyright (c) 2018 - * United States Army Corps of Engineers - Hydrologic Engineering Center (USACE/HEC) - * All Rights Reserved. USACE PROPRIETARY/CONFIDENTIAL. - * Source may not be released without written approval from HEC - */ -package usace.rowcps.headless; - -import java.util.TimeZone; -import usace.rowcps.headless.tests.TestVariables; - -/** - * This class is intended for use by developers to run headless after making changes. Run a single test at a time - * otherwise it might be a while... - * - * All files are relative to the JYTHON_FILE_ROOT variable, which should be in this same folder. - * - * @author @author Ryan A. Miles (ryanm@rmanet.com) - */ -public class TestHeadless -{ - static final String JYTHON_FILE_ROOT = "src\\test\\java\\usace\\rowcps\\headless\\"; - static final String CREDENTIALS_FILE = "usace/rowcps/headless/credentials.properties"; - - //I'd like this to be the office, but we haven't connected yet. - //Could get it from the credentials probably. - static final String SUB_FOLDER = "tests"; - -// @Test //23-001 - migration - public void testAssoc_DBExport() throws Exception - { - RegiCLI.runHeadlessTest(getArgsForFile("Assoc_DBExport.py")); - } - -// @Test //23-001 - migration - public void testBasinPieExport() throws Exception - { - RegiCLI.runHeadlessTest(getArgsForFile("BasinPieExport.py")); - } - -// @Test //23-001 - migration - public void testGateFlowPy() throws Exception - { - RegiCLI.runHeadlessTest(getArgsForFile("GateFlow.py")); - } - -// @Test //23-001 - migration - public void testGateFlowCalcPy() throws Exception - { - RegiCLI.runHeadlessTest(getArgsForFile("GateFlowCalc.py")); - } - -// @Test //23-001 - migration - public void testGateSettingsPy() throws Exception - { - RegiCLI.runHeadlessTest(getArgsForFile("GateSettings.py")); - } - -// @Test //23-001 - migration - public void testInflowCalcAutoAdjustPy() throws Exception - { - RegiCLI.runHeadlessTest(getArgsForFile("InflowCalcAutoAdjust.py")); - } - -// @Test //23-001 - migration - public void testInflowCalcBalanceAllPy() throws Exception - { - RegiCLI.runHeadlessTest(getArgsForFile("InflowCalcBalanceAll.py")); - } - -// @Test //23-001 - migration - public void testInflowCalcClonePy() throws Exception - { - RegiCLI.runHeadlessTest(getArgsForFile("InflowCalcClone.py")); - } - -// @Test //23-001 - migration - public void testInflowCalcComputeEvapAsFlowPy() throws Exception - { - RegiCLI.runHeadlessTest(getArgsForFile("InflowCalcComputeEvapAsFlow.py")); - } - - -// @Test //23-001 - migration - public void testInflowCalcComputedInflowPy() throws Exception - { - RegiCLI.runHeadlessTest(getArgsForFile("InflowCalcComputedInflow.py")); - } - -// @Test //23-001 - migration - public void testInflowCalcZeroNegativePy() throws Exception - { - RegiCLI.runHeadlessTest(getArgsForFile("InflowCalcZeroNegative.py")); - } - -// @Test //23-001 - migration - public void testPoolPercentCalcPy() throws Exception - { - RegiCLI.runHeadlessTest(getArgsForFile("PoolPercentCalc.py")); - } - -// @Test //23-001 - migration - public void testSigstages_DBExportPy() throws Exception - { - RegiCLI.runHeadlessTest(getArgsForFile("Sigstages_DBExport.py")); - } - -// @Test //23-001 - migration - public void testSigstages_DBImportPy() throws Exception - { - RegiCLI.runHeadlessTest(getArgsForFile("Sigstages_DBImport.py")); - } - -// @Test //23-001 - migration - public void testSigstages_DownloadPy() throws Exception - { - RegiCLI.runHeadlessTest(getArgsForFile("Sigstages_Download.py")); - } - -// @Test //23-001 - migration - public void testStatusDemoPy() throws Exception - { - RegiCLI.runHeadlessTest(getArgsForFile("StatusDemo.py")); - } - - static String[] getArgsForFile(String file) - { - return getArgsForFileAndTimeZone(SUB_FOLDER + "\\" + file, TimeZone.getTimeZone("US/Central")); - } - - static String getJythonTestFolder() - { - return JYTHON_FILE_ROOT + SUB_FOLDER; - } - - private static String[] getArgsForFileAndTimeZone(String file, TimeZone tz) - { - TestVariables.init(); - String[] args = new String[] - { - "-Drowcps.timezone=" + tz.getID(), - "-p", JYTHON_FILE_ROOT + CREDENTIALS_FILE, - "-f", JYTHON_FILE_ROOT + file, - }; - - return args; - } -} diff --git a/regi-headless/src/test/java/usace/rowcps/headless/tests/TestVariables.java b/regi-headless/src/test/java/usace/rowcps/headless/tests/TestVariables.java deleted file mode 100644 index 417caeb..0000000 --- a/regi-headless/src/test/java/usace/rowcps/headless/tests/TestVariables.java +++ /dev/null @@ -1,39 +0,0 @@ -package usace.rowcps.headless.tests; - -/** - * I couldn't get this into a python file...it kept saying it couldn't find the module. I think it's because the - * evaluator only takes in one file? - * - * Anyway, this is intended to provide the global test information for all of the python scripts. I set this up for the - * SWT office and locations. I've also updated these tests to use an output file location that we can get all our files - * to so it's not so scattered. - * - * @author Ryan A. Miles (ryanm@rmanet.com) - */ -public final class TestVariables -{ - - //Office ID for current tests. - public static final String OFFICE_ID = "SWT"; - //These are intended to be unique, please don't use the same variables. - public static final String GATE_LOCATION = "FGIB"; - public static final String INFLOW_LOCATION = "EUFA"; - public static final String POOL_LOCATION = "ARBU"; - public static final String LOCATION_4 = "ALTU"; - public static final String[] ALL_PROJECTS = new String[] - { - GATE_LOCATION, INFLOW_LOCATION, POOL_LOCATION, LOCATION_4 - }; - public static final String STREAM_GAGE_LOCATION = "RIPL"; - public static final String HEADLESS_FILE_LOCATION = "C:\\Temp\\Headless\\"; - - public static void init() - { - - } - - private TestVariables() - { - throw new AssertionError("Don't instantiate this class"); - } -} diff --git a/regi-headless/src/test/python/test_district_scripts.py b/regi-headless/src/test/python/test_district_scripts.py new file mode 100644 index 0000000..44704d4 --- /dev/null +++ b/regi-headless/src/test/python/test_district_scripts.py @@ -0,0 +1,244 @@ +import importlib.util +import io +import logging +import re +import sys +import types +from contextlib import redirect_stdout +from pathlib import Path + + +MODULE_ROOT = Path(__file__).resolve().parents[3] +REPOSITORY_ROOT = MODULE_ROOT.parent +DISTRICT_SCRIPTS_ROOT = REPOSITORY_ROOT / "district-scripts" +EXAMPLE_SCRIPTS_ROOT = MODULE_ROOT / "src" / "test" / "resources" / "usace" / "rowcps" / "headless" / "examples" +JAVA_SOURCE_ROOT = MODULE_ROOT / "src" / "main" / "java" + + +JAVA_METHOD_PATTERN = re.compile( + r"\bpublic\s+(?:static\s+)?(?:[\w<>\[\], ?]+\s+)+(?P[A-Za-z_]\w*)\s*\(" +) +LOGGER = logging.getLogger(__name__) + + +def test_migrated_scripts_only_call_known_scriptable_api(monkeypatch): + java_api = _load_java_api() + _install_fake_modules(monkeypatch, java_api) + + _validate_scripts("district", DISTRICT_SCRIPTS_ROOT, _district_scripts(), java_api) + _validate_scripts("example", EXAMPLE_SCRIPTS_ROOT, _example_scripts(), java_api) + + +def _validate_scripts(label, root, scripts, java_api): + assert scripts, f"No {label} scripts found under {root}" + LOGGER.info("Validating %s %s script(s)", len(scripts), label) + + failures = [] + for script in scripts: + relative_script = script.relative_to(REPOSITORY_ROOT) + LOGGER.info("Validating %s script: %s", label, relative_script) + try: + module = _load_script(script) + with redirect_stdout(io.StringIO()): + _script_callback(module)(FakeRegistry(java_api)) + except Exception as exc: + failures.append(f"{relative_script}: {type(exc).__name__}: {exc}") + else: + LOGGER.info("Validated %s script: %s", label, relative_script) + + assert not failures, f"{label.title()} script API validation failed:\n" + "\n".join(failures) + + +def _district_scripts(): + return sorted( + path + for path in DISTRICT_SCRIPTS_ROOT.rglob("*.py") + if path.name != "__init__.py" + ) + + +def _example_scripts(): + return sorted( + path + for path in EXAMPLE_SCRIPTS_ROOT.rglob("*.py") + if path.name != "__init__.py" + ) + + +def _script_callback(module): + for name in ( + "run_calculations", + "calculate_inflow", + "calculate_gate_flow", + "calculate_gate_settings", + "configure_logging_options", + ): + callback = getattr(module, name, None) + if callback is not None: + return callback + raise AssertionError(f"No migrated script callback found in {module.__file__}") + + +def _load_java_api(): + return { + "Inflow": _java_methods( + "usace/rowcps/headless/calculator/inflow/ScriptableInflowImpl.java" + ), + "Gate Flow": _java_methods( + "usace/rowcps/headless/calculator/flowgroup/ScriptableGateFlowImpl.java" + ), + "Gate Settings": _java_methods( + "usace/rowcps/headless/calculator/gatesettings/ScriptableGateSettingsImpl.java" + ), + "LoggingOptions": _java_methods("usace/rowcps/headless/LoggingOptions.java"), + } + + +def _java_methods(relative_path): + source = (JAVA_SOURCE_ROOT / relative_path).read_text(encoding="utf-8") + return { + match.group("name") + for match in JAVA_METHOD_PATTERN.finditer(_strip_java_comments(source)) + } + + +def _strip_java_comments(source): + source = re.sub(r"/\*.*?\*/", "", source, flags=re.DOTALL) + return re.sub(r"//.*", "", source) + + +def _load_script(path): + module_name = "district_script_" + re.sub(r"\W+", "_", str(path.relative_to(REPOSITORY_ROOT))) + spec = importlib.util.spec_from_file_location(module_name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _install_fake_modules(monkeypatch, java_api): + modules = {} + + def module(name): + value = modules.get(name) + if value is None: + value = types.ModuleType(name) + modules[name] = value + monkeypatch.setitem(sys.modules, name, value) + if "." in name: + parent_name, child_name = name.rsplit(".", 1) + setattr(module(parent_name), child_name, value) + return value + + regi_python = module("regi_python") + regi_python.regi_session = _fake_regi_session + regi_python.run_headless = lambda callback: callback(FakeRegistry(java_api)) + + java_util = module("java.util") + java_util.Calendar = FakeCalendar + java_util.TimeZone = FakeTimeZone + + java_lang = module("java.lang") + java_lang.System = FakeSystem + + headless = module("usace.rowcps.headless") + headless.LoggingOptions = type( + "LoggingOptions", + (), + {name: staticmethod(_noop) for name in java_api["LoggingOptions"]}, + ) + + inflow = module("usace.rowcps.headless.calculator.inflow") + inflow.InflowComputationStorageOption = types.SimpleNamespace( + EVAP_AS_FLOW="EVAP_AS_FLOW", + PROJECT_RELEASES="PROJECT_RELEASES", + ) + + +class _fake_regi_session: + def __enter__(self): + return None + + def __exit__(self, exc_type, exc, traceback): + return False + + +class FakeRegistry: + def __init__(self, java_api): + self._java_api = java_api + + def getNames(self, version): + return ["Inflow", "Gate Flow", "Gate Settings"] + + def getCalculation(self, version, name): + if name not in self._java_api: + raise AssertionError(f"Unknown calculation requested: {name!r}") + return FakeJavaObject(name, self._java_api[name]) + + +class FakeJavaObject: + def __init__(self, display_name, method_names): + self._display_name = display_name + self._method_names = method_names + + def __getattr__(self, name): + if name not in self._method_names: + raise AttributeError(f"{self._display_name} has no Java method {name!r}") + return _noop + + +class FakeTimeZone: + @staticmethod + def getTimeZone(name): + return FakeTimeZone() + + +class FakeSystem: + @staticmethod + def getProperty(name): + return "" + + +class FakeCalendar: + DATE = 1 + DAY_OF_MONTH = 2 + HOUR = 3 + HOUR_OF_DAY = 4 + MILLISECOND = 5 + MINUTE = 6 + MONTH = 7 + SECOND = 8 + YEAR = 9 + + @staticmethod + def getInstance(time_zone=None): + return FakeCalendar() + + def add(self, field, amount): + return None + + def clear(self): + return None + + def getTime(self): + return FakeDate() + + def getTimeInMillis(self): + return 0 + + def get(self, field): + return 0 + + def set(self, field, value): + return None + + +class FakeDate: + def getTime(self): + return 0 + + def toString(self): + return "FakeDate" + + +def _noop(*args, **kwargs): + return None diff --git a/regi-headless/src/test/python/test_regi_python.py b/regi-headless/src/test/python/test_regi_python.py new file mode 100644 index 0000000..81b9bb2 --- /dev/null +++ b/regi-headless/src/test/python/test_regi_python.py @@ -0,0 +1,84 @@ +# Copyright (c) 2026 +# United States Army Corps of Engineers - Hydrologic Engineering Center (USACE/HEC) +# All Rights Reserved. USACE PROPRIETARY/CONFIDENTIAL. +# Source may not be released without written approval from HEC + +import importlib.metadata +import importlib +from pathlib import Path + + +def test_wheel_distribution_metadata_is_installed(): + dist = importlib.metadata.distribution("regi-python") + + assert dist.metadata["Name"] == "regi-python" + assert dist.version + + +def test_wheel_can_import_top_level_package(): + import regi_python + + assert regi_python is not None + + +def test_public_api_is_exposed(): + import regi_python + + assert callable(regi_python.regi_session) + assert callable(regi_python.run_headless) + + +def test_run_headless_requires_cda_environment(monkeypatch): + import regi_python + + monkeypatch.delenv("CDA_URL", raising=False) + monkeypatch.delenv("CDA_API_KEY", raising=False) + monkeypatch.delenv("OFFICE_ID", raising=False) + + try: + regi_python.run_headless(lambda registry: None) + except RuntimeError as exc: + assert str(exc) == ( + "Missing required environment variables: CDA_URL, CDA_API_KEY, OFFICE_ID" + ) + else: + raise AssertionError("run_headless should fail when CDA env vars are missing") + + +def test_bundled_java_libraries_are_present(): + import regi_python + package_dir = Path(regi_python.__file__).parent + lib_dir = package_dir / "lib" + + assert lib_dir.is_dir() + assert any(lib_dir.glob("*.jar")) + + +def test_bridge_import_does_not_require_java_home(monkeypatch): + import regi_python.regi_python as bridge + + monkeypatch.delenv("JAVA_HOME", raising=False) + + reloaded = importlib.reload(bridge) + + assert reloaded is bridge + + +def test_regi_session_only_shuts_down_jvm_it_started(monkeypatch): + import regi_python.regi_python as bridge + + started = [] + stopped = [] + jul_configured = [] + + monkeypatch.setattr(bridge.jpype, "isJVMStarted", lambda: True) + monkeypatch.setattr(bridge.jpype, "startJVM", lambda *args, **kwargs: started.append((args, kwargs))) + monkeypatch.setattr(bridge.jpype, "shutdownJVM", lambda: stopped.append(True)) + monkeypatch.setattr(bridge, "configure_jul_to_python_logging", lambda logger: jul_configured.append(logger)) + + with bridge.regi_session(): + pass + + assert started == [] + assert stopped == [] + assert jul_configured == [] From e0bf121cf8f4d9c4292e106202ad58bb0ae0bbdd Mon Sep 17 00:00:00 2001 From: Adam Korynta Date: Thu, 16 Jul 2026 13:49:51 -0700 Subject: [PATCH 2/4] disable smoke testing until district script migration is complete --- regi-headless/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regi-headless/build.gradle b/regi-headless/build.gradle index fe64972..8f3c4f7 100644 --- a/regi-headless/build.gradle +++ b/regi-headless/build.gradle @@ -186,5 +186,5 @@ tasks.register('smokeTestDistrictScripts', VenvTask) { check { dependsOn testPythonWheel - dependsOn smokeTestDistrictScripts +// dependsOn smokeTestDistrictScripts } From 028b606944093f455dd4e83854e9b00e09882e4e Mon Sep 17 00:00:00 2001 From: Adam Korynta Date: Thu, 16 Jul 2026 13:50:36 -0700 Subject: [PATCH 3/4] disable smoke testing until district script migration is complete --- regi-headless/build.gradle | 22 -- .../src/test/python/test_district_scripts.py | 244 ------------------ 2 files changed, 266 deletions(-) delete mode 100644 regi-headless/src/test/python/test_district_scripts.py diff --git a/regi-headless/build.gradle b/regi-headless/build.gradle index 8f3c4f7..edc66ab 100644 --- a/regi-headless/build.gradle +++ b/regi-headless/build.gradle @@ -163,28 +163,6 @@ tasks.register('testPythonWheel', VenvTask) { outputs.upToDateWhen { false } } -tasks.register('smokeTestDistrictScripts', VenvTask) { - group = 'verification' - description = 'Validates migrated district and example scripts against the Python entrypoint shape and Java scriptable APIs.' - - dependsOn installPythonBuildTools - - venvExec = 'python' - args = [ - '-m', 'pytest', - '-o', 'log_cli=true', - '--log-cli-level=INFO', - 'src/test/python/test_district_scripts.py' - ] - - inputs.files(fileTree(dir: '../district-scripts', include: '**/*.py')) - inputs.files(fileTree(dir: 'src/test/resources/usace/rowcps/headless/examples', include: '**/*.py')) - inputs.files(fileTree(dir: 'src/main/java/usace/rowcps/headless', include: '**/*.java')) - inputs.file('src/test/python/test_district_scripts.py') - outputs.upToDateWhen { false } -} - check { dependsOn testPythonWheel -// dependsOn smokeTestDistrictScripts } diff --git a/regi-headless/src/test/python/test_district_scripts.py b/regi-headless/src/test/python/test_district_scripts.py deleted file mode 100644 index 44704d4..0000000 --- a/regi-headless/src/test/python/test_district_scripts.py +++ /dev/null @@ -1,244 +0,0 @@ -import importlib.util -import io -import logging -import re -import sys -import types -from contextlib import redirect_stdout -from pathlib import Path - - -MODULE_ROOT = Path(__file__).resolve().parents[3] -REPOSITORY_ROOT = MODULE_ROOT.parent -DISTRICT_SCRIPTS_ROOT = REPOSITORY_ROOT / "district-scripts" -EXAMPLE_SCRIPTS_ROOT = MODULE_ROOT / "src" / "test" / "resources" / "usace" / "rowcps" / "headless" / "examples" -JAVA_SOURCE_ROOT = MODULE_ROOT / "src" / "main" / "java" - - -JAVA_METHOD_PATTERN = re.compile( - r"\bpublic\s+(?:static\s+)?(?:[\w<>\[\], ?]+\s+)+(?P[A-Za-z_]\w*)\s*\(" -) -LOGGER = logging.getLogger(__name__) - - -def test_migrated_scripts_only_call_known_scriptable_api(monkeypatch): - java_api = _load_java_api() - _install_fake_modules(monkeypatch, java_api) - - _validate_scripts("district", DISTRICT_SCRIPTS_ROOT, _district_scripts(), java_api) - _validate_scripts("example", EXAMPLE_SCRIPTS_ROOT, _example_scripts(), java_api) - - -def _validate_scripts(label, root, scripts, java_api): - assert scripts, f"No {label} scripts found under {root}" - LOGGER.info("Validating %s %s script(s)", len(scripts), label) - - failures = [] - for script in scripts: - relative_script = script.relative_to(REPOSITORY_ROOT) - LOGGER.info("Validating %s script: %s", label, relative_script) - try: - module = _load_script(script) - with redirect_stdout(io.StringIO()): - _script_callback(module)(FakeRegistry(java_api)) - except Exception as exc: - failures.append(f"{relative_script}: {type(exc).__name__}: {exc}") - else: - LOGGER.info("Validated %s script: %s", label, relative_script) - - assert not failures, f"{label.title()} script API validation failed:\n" + "\n".join(failures) - - -def _district_scripts(): - return sorted( - path - for path in DISTRICT_SCRIPTS_ROOT.rglob("*.py") - if path.name != "__init__.py" - ) - - -def _example_scripts(): - return sorted( - path - for path in EXAMPLE_SCRIPTS_ROOT.rglob("*.py") - if path.name != "__init__.py" - ) - - -def _script_callback(module): - for name in ( - "run_calculations", - "calculate_inflow", - "calculate_gate_flow", - "calculate_gate_settings", - "configure_logging_options", - ): - callback = getattr(module, name, None) - if callback is not None: - return callback - raise AssertionError(f"No migrated script callback found in {module.__file__}") - - -def _load_java_api(): - return { - "Inflow": _java_methods( - "usace/rowcps/headless/calculator/inflow/ScriptableInflowImpl.java" - ), - "Gate Flow": _java_methods( - "usace/rowcps/headless/calculator/flowgroup/ScriptableGateFlowImpl.java" - ), - "Gate Settings": _java_methods( - "usace/rowcps/headless/calculator/gatesettings/ScriptableGateSettingsImpl.java" - ), - "LoggingOptions": _java_methods("usace/rowcps/headless/LoggingOptions.java"), - } - - -def _java_methods(relative_path): - source = (JAVA_SOURCE_ROOT / relative_path).read_text(encoding="utf-8") - return { - match.group("name") - for match in JAVA_METHOD_PATTERN.finditer(_strip_java_comments(source)) - } - - -def _strip_java_comments(source): - source = re.sub(r"/\*.*?\*/", "", source, flags=re.DOTALL) - return re.sub(r"//.*", "", source) - - -def _load_script(path): - module_name = "district_script_" + re.sub(r"\W+", "_", str(path.relative_to(REPOSITORY_ROOT))) - spec = importlib.util.spec_from_file_location(module_name, path) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - -def _install_fake_modules(monkeypatch, java_api): - modules = {} - - def module(name): - value = modules.get(name) - if value is None: - value = types.ModuleType(name) - modules[name] = value - monkeypatch.setitem(sys.modules, name, value) - if "." in name: - parent_name, child_name = name.rsplit(".", 1) - setattr(module(parent_name), child_name, value) - return value - - regi_python = module("regi_python") - regi_python.regi_session = _fake_regi_session - regi_python.run_headless = lambda callback: callback(FakeRegistry(java_api)) - - java_util = module("java.util") - java_util.Calendar = FakeCalendar - java_util.TimeZone = FakeTimeZone - - java_lang = module("java.lang") - java_lang.System = FakeSystem - - headless = module("usace.rowcps.headless") - headless.LoggingOptions = type( - "LoggingOptions", - (), - {name: staticmethod(_noop) for name in java_api["LoggingOptions"]}, - ) - - inflow = module("usace.rowcps.headless.calculator.inflow") - inflow.InflowComputationStorageOption = types.SimpleNamespace( - EVAP_AS_FLOW="EVAP_AS_FLOW", - PROJECT_RELEASES="PROJECT_RELEASES", - ) - - -class _fake_regi_session: - def __enter__(self): - return None - - def __exit__(self, exc_type, exc, traceback): - return False - - -class FakeRegistry: - def __init__(self, java_api): - self._java_api = java_api - - def getNames(self, version): - return ["Inflow", "Gate Flow", "Gate Settings"] - - def getCalculation(self, version, name): - if name not in self._java_api: - raise AssertionError(f"Unknown calculation requested: {name!r}") - return FakeJavaObject(name, self._java_api[name]) - - -class FakeJavaObject: - def __init__(self, display_name, method_names): - self._display_name = display_name - self._method_names = method_names - - def __getattr__(self, name): - if name not in self._method_names: - raise AttributeError(f"{self._display_name} has no Java method {name!r}") - return _noop - - -class FakeTimeZone: - @staticmethod - def getTimeZone(name): - return FakeTimeZone() - - -class FakeSystem: - @staticmethod - def getProperty(name): - return "" - - -class FakeCalendar: - DATE = 1 - DAY_OF_MONTH = 2 - HOUR = 3 - HOUR_OF_DAY = 4 - MILLISECOND = 5 - MINUTE = 6 - MONTH = 7 - SECOND = 8 - YEAR = 9 - - @staticmethod - def getInstance(time_zone=None): - return FakeCalendar() - - def add(self, field, amount): - return None - - def clear(self): - return None - - def getTime(self): - return FakeDate() - - def getTimeInMillis(self): - return 0 - - def get(self, field): - return 0 - - def set(self, field, value): - return None - - -class FakeDate: - def getTime(self): - return 0 - - def toString(self): - return "FakeDate" - - -def _noop(*args, **kwargs): - return None From 6688c1bb7b0408dc20982ba3188f6352985eedaf Mon Sep 17 00:00:00 2001 From: Adam Korynta Date: Thu, 16 Jul 2026 13:39:46 -0700 Subject: [PATCH 4/4] modularize GitHub Actions workflows: separate build-wheel and release processes --- .github/workflows/build-wheel.yml | 50 +++++++++++++++++++++++++++++++ .github/workflows/build.yml | 31 +++++-------------- .github/workflows/release.yml | 46 ++++++++++++++++++++++++++++ 3 files changed, 103 insertions(+), 24 deletions(-) create mode 100644 .github/workflows/build-wheel.yml create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/build-wheel.yml b/.github/workflows/build-wheel.yml new file mode 100644 index 0000000..7b3fdae --- /dev/null +++ b/.github/workflows/build-wheel.yml @@ -0,0 +1,50 @@ +name: Build Python Wheel + +on: + workflow_call: + inputs: + ref: + description: Git ref to check out + required: false + type: string + default: "" + +jobs: + build-wheel: + name: Build Python wheel + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v7 + with: + fetch-depth: 0 + ref: ${{ inputs.ref || github.ref }} + + - name: Set up Java + uses: actions/setup-java@v5 + with: + distribution: temurin + java-version: 21 + + - name: Set up Gradle + uses: gradle/actions/setup-gradle@v6 + + - name: Make Gradle wrapper executable + run: chmod +x ./gradlew + + - name: Build and test + run: ./gradlew clean build buildPythonWheel + + - name: Collect wheel + run: | + mkdir -p dist + find . -path "*/build/install/*/dist/*.whl" -exec cp {} dist/ \; + + - name: Upload Python wheel artifact + uses: actions/upload-artifact@v4 + with: + name: python-wheel + path: dist/*.whl + if-no-files-found: error + retention-days: 14 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0be5339..c9f1b40 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,35 +9,18 @@ on: permissions: contents: read +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: - build: + build-wheel: name: Build and test - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v7 - with: - fetch-depth: 0 - - - name: Set up Java - uses: actions/setup-java@v5 - with: - distribution: temurin - java-version: 21 - - - name: Set up Gradle - uses: gradle/actions/setup-gradle@v6 - - - name: Make Gradle wrapper executable - run: chmod +x ./gradlew - - - name: Build and test - run: ./gradlew clean build + uses: ./.github/workflows/build-wheel.yml dependency-submission: name: Submit Gradle dependencies - needs: build + needs: build-wheel runs-on: ubuntu-latest if: github.event_name == 'push' && github.ref == 'refs/heads/main' permissions: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..e0833f6 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,46 @@ +name: Release + +on: + release: + types: + - published + +permissions: + contents: write + actions: read + +concurrency: + group: release-${{ github.event.release.id }} + cancel-in-progress: false + +jobs: + build-wheel: + name: Build wheel from release tag + uses: ./.github/workflows/build-wheel.yml + with: + ref: ${{ github.event.release.tag_name }} + + publish-wheel: + name: Publish Python wheel to GitHub Release + needs: build-wheel + runs-on: ubuntu-latest + + steps: + - name: Download Python wheel artifact + uses: actions/download-artifact@v4 + with: + name: python-wheel + path: dist + + - name: Generate checksums + run: | + cd dist + sha256sum *.whl > SHA256SUMS.txt + + - name: Publish wheel to GitHub Release + uses: softprops/action-gh-release@v2 + with: + tag_name: ${{ github.event.release.tag_name }} + files: | + dist/*.whl + dist/SHA256SUMS.txt \ No newline at end of file