From ab9010bca444cdc6c201fbf0cca7268ed0ad92cb Mon Sep 17 00:00:00 2001 From: Jasmin Makwana Date: Fri, 1 May 2026 16:12:20 +0530 Subject: [PATCH 1/6] [minor] Add support for configuring scheduling constraints for AI Service tenant --- .secrets.baseline | 2 +- python/src/mas/cli/aiservice/install/app.py | 19 +++++++++++++++++++ .../mas/cli/aiservice/install/argBuilder.py | 2 ++ .../mas/cli/aiservice/install/argParser.py | 9 ++++++++- .../src/mas/cli/aiservice/install/params.py | 3 +++ .../mas/cli/aiservice/install/summarizer.py | 3 +++ .../cli/install/settings/additionalConfigs.py | 15 +++++++++++++++ tekton/src/params/install-aiservice.yml.j2 | 4 ++++ tekton/src/pipelines/mas-install.yml.j2 | 2 ++ .../taskdefs/aiservice/aiservice.yml.j2 | 4 ++++ tekton/src/tasks/aiservice/aiservice.yml.j2 | 8 ++++++++ 11 files changed, 69 insertions(+), 2 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index c177b52bba9..90007c59841 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,7 +3,7 @@ "files": "build/bin/config/oscap/ssg-rhel9-ds.xml|^.secrets.baseline$|^docs/catalogs/", "lines": null }, - "generated_at": "2026-04-26T12:19:35Z", + "generated_at": "2026-05-01T10:41:14Z", "plugins_used": [ { "name": "AWSKeyDetector" diff --git a/python/src/mas/cli/aiservice/install/app.py b/python/src/mas/cli/aiservice/install/app.py index 827e115c1fa..f8fd04f4881 100644 --- a/python/src/mas/cli/aiservice/install/app.py +++ b/python/src/mas/cli/aiservice/install/app.py @@ -447,6 +447,8 @@ def install(self, argv): # Set up the sls license file self.slsLicenseFile() + self.aiserviceConfig() + # Show a summary of the installation configuration self.printH1("Non-Interactive Install Command") self.printDescription([ @@ -494,6 +496,7 @@ def install(self, argv): additionalConfigs=self.additionalConfigsSecret, podTemplates=self.podTemplatesSecret, certs=self.certsSecret, + aiserviceConfig=self.aiserviceConfigSecret, slack_token=self.getParam("slack_token"), slack_channel=self.getParam("slack_channel") ) @@ -598,6 +601,22 @@ def aiServiceTenantSettings(self) -> None: self.setParam("tenant_entitlement_start_date", today.strftime('%Y-%m-%d')) self.promptForString("Entitlement end date (YYYY-MM-DD)", "tenant_entitlement_end_date", default=oneyear.strftime('%Y-%m-%d')) + self.configSchedulingConstraints() + + @logMethodCall + def configSchedulingConstraints(self): + if self.showAdvancedOptions: + self.printH1("Scheduling constraints for AI Workloads") + self.printDescription(content=[ + "AI Service supports configuring tolerations and nodeSelector per tenant to schedule AI workloads(training pipelines & Inference services) on dedicated nodes.", + "To configure tolerations and nodeSelector, create a YAML configuration file", + "The YAML file must contain `pipeline` and/or `predictor` objects. Each object can have:", + " `tolerations`: List of Kubernetes tolerations (required fields: `key`, `operator`, `effect`)", + " `nodeSelector`: Dictionary of node label key-value pairs", + ]) + + self.aiserviceTenantSchedulingConfigFileLocal = self.promptForFile("Scheduling constraints YAML file", mustExist=True, envVar="AISERVICE_TENANT_SCHEDULING_CONFIG_FILE") + def _setMinioStorageDefaults(self) -> None: """ Set MinIO storage defaults when MinIO is being installed in-cluster. diff --git a/python/src/mas/cli/aiservice/install/argBuilder.py b/python/src/mas/cli/aiservice/install/argBuilder.py index 61a84d5f4d4..7d8f22f66b8 100644 --- a/python/src/mas/cli/aiservice/install/argBuilder.py +++ b/python/src/mas/cli/aiservice/install/argBuilder.py @@ -172,6 +172,8 @@ def buildCommand(self) -> str: command += f" --tenant-entitlement-start-date \"{self.getParam('tenant_entitlement_start_date')}\"{newline}" if self.getParam('tenant_entitlement_end_date') != "": command += f" --tenant-entitlement-end-date \"{self.getParam('tenant_entitlement_end_date')}\"{newline}" + if self.aiserviceTenantSchedulingConfigFileLocal: + command += f" --tenant-scheduling-constraints-file \"{self.aiserviceTenantSchedulingConfigFileLocal}\"{newline}" if self.getParam('rsl_url') != "": command += f" --rsl-url \"{self.getParam('rsl_url')}\"{newline}" diff --git a/python/src/mas/cli/aiservice/install/argParser.py b/python/src/mas/cli/aiservice/install/argParser.py index 6deb68edffc..fb471e96501 100644 --- a/python/src/mas/cli/aiservice/install/argParser.py +++ b/python/src/mas/cli/aiservice/install/argParser.py @@ -23,7 +23,7 @@ def isValidFile(parser, arg) -> str: aiServiceinstallArgParser = argparse.ArgumentParser( - prog="mas install-aiservice", + prog="mas aiservice-install", description="\n".join([ f"IBM Maximo Application Suite Admin CLI v{packageVersion}", "Install Aiservice by configuring and launching the Tekton Pipeline.\n", @@ -444,6 +444,13 @@ def isValidFile(parser, arg) -> str: action="store_const", const="true" ) +aiserviceAdvancedArgGroup.add_argument( + "--tenant-scheduling-constraints-file", + dest="tenant_scheduling_config_file", + required=False, + help="Path to the YAML file that contains the scheduling constraints for tenant", + type=lambda x: isValidFile(aiServiceinstallArgParser, x) +) # IBM Db2 Universal Operator diff --git a/python/src/mas/cli/aiservice/install/params.py b/python/src/mas/cli/aiservice/install/params.py index e2050315db0..a5811b5e1d1 100644 --- a/python/src/mas/cli/aiservice/install/params.py +++ b/python/src/mas/cli/aiservice/install/params.py @@ -110,4 +110,7 @@ # Slack "slack_token", "slack_channel", + + # Scheduling constraints + "tenant_scheduling_config_file", ] diff --git a/python/src/mas/cli/aiservice/install/summarizer.py b/python/src/mas/cli/aiservice/install/summarizer.py index 75cb435ca56..fc5a14ebd1f 100644 --- a/python/src/mas/cli/aiservice/install/summarizer.py +++ b/python/src/mas/cli/aiservice/install/summarizer.py @@ -57,6 +57,9 @@ def aiServiceSummary(self) -> None: self.printParamSummary("Start Date", "tenant_entitlement_start_date") self.printParamSummary("End Date", "tenant_entitlement_end_date") + if self.aiserviceTenantSchedulingConfigFileLocal: + self.printParamSummary("Scheduling constraints config file", "tenant_scheduling_config_file") + self.printH2("S3 Configuration") # self.printParamSummary("Storage provider", "aiservice_s3_provider") if self.getParam("minio_root_user") is not None and self.getParam("minio_root_user") != "": diff --git a/python/src/mas/cli/install/settings/additionalConfigs.py b/python/src/mas/cli/install/settings/additionalConfigs.py index 52f978f0a32..fb7fa37c20f 100644 --- a/python/src/mas/cli/install/settings/additionalConfigs.py +++ b/python/src/mas/cli/install/settings/additionalConfigs.py @@ -34,10 +34,12 @@ class AdditionalConfigsMixin(): slsLicenseFileLocal: str | None manualCertsDir: str | None showAdvancedOptions: bool + aiserviceTenantSchedulingConfigFileLocal: str | None additionalConfigsSecret: Dict[str, Any] | None podTemplatesSecret: Dict[str, Any] | None slsLicenseFileSecret: Dict[str, Any] | None certsSecret: Dict[str, Any] | None + aiserviceConfigSecret: Dict[str, Any] | None # Methods from BaseApp def setParam(self, param: str, value: str) -> None: @@ -272,6 +274,19 @@ def slsLicenseFile(self) -> None: self.setParam("sls_entitlement_file", f"/workspace/entitlement/{path.basename(self.slsLicenseFileLocal)}") self.slsLicenseFileSecret = self.addFilesToSecret(slsLicenseFileSecret, self.slsLicenseFileLocal, '') + def aiserviceConfig(self) -> None: + if self.aiserviceTenantSchedulingConfigFileLocal: + aiserviceConfigSecret: dict[str, Any] = { + "apiVersion": "v1", + "kind": "Secret", + "type": "Opaque", + "metadata": { + "name": "pipeline-aiservice-config" + } + } + self.setParam("tenant_scheduling_config_file", f"/workspace/aiservice/{path.basename(self.aiserviceTenantSchedulingConfigFileLocal)}") + self.aiserviceConfigSecret = self.addFilesToSecret(aiserviceConfigSecret, self.aiserviceTenantSchedulingConfigFileLocal, 'yaml') + def addFilesToSecret(self, secretDict: dict, configPath: str, extension: str, keyPrefix: str = '') -> dict: """ Add file (or files) to pipeline-additional-configs diff --git a/tekton/src/params/install-aiservice.yml.j2 b/tekton/src/params/install-aiservice.yml.j2 index 3da99e59c99..d415bf930d2 100644 --- a/tekton/src/params/install-aiservice.yml.j2 +++ b/tekton/src/params/install-aiservice.yml.j2 @@ -96,6 +96,10 @@ type: string description: define end date for tenant default: "" +- name: tenant_scheduling_cfg_file + type: string + description: Path to YAML file that contains scheduling constraints for AI workloads for tenant + default: "" # MAS Application Configuration - IBM Maximo AI Service - Watstonx # ----------------------------------------------------------------------------- diff --git a/tekton/src/pipelines/mas-install.yml.j2 b/tekton/src/pipelines/mas-install.yml.j2 index 488d8157a8e..7857caab965 100644 --- a/tekton/src/pipelines/mas-install.yml.j2 +++ b/tekton/src/pipelines/mas-install.yml.j2 @@ -15,6 +15,8 @@ spec: - name: shared-certificates # PodTemplates configurations - name: shared-pod-templates + # AIService configurations. Contains Scheduling config file for AI workloads for tenant. + - name: shared-aiservice-config params: # 1. Common Parameters diff --git a/tekton/src/pipelines/taskdefs/aiservice/aiservice.yml.j2 b/tekton/src/pipelines/taskdefs/aiservice/aiservice.yml.j2 index 36d8598b69a..50f35276e72 100644 --- a/tekton/src/pipelines/taskdefs/aiservice/aiservice.yml.j2 +++ b/tekton/src/pipelines/taskdefs/aiservice/aiservice.yml.j2 @@ -95,6 +95,8 @@ value: $(params.aiservice_certificate_issuer) - name: enable_ipv6 value: $(params.enable_ipv6) + - name: tenant_scheduling_cfg_file + value: $(params.tenant_scheduling_cfg_file) taskRef: name: mas-devops-aiservice @@ -106,3 +108,5 @@ workspaces: - name: configs workspace: shared-configs + - name: aiservice + workspace: shared-aiservice-config diff --git a/tekton/src/tasks/aiservice/aiservice.yml.j2 b/tekton/src/tasks/aiservice/aiservice.yml.j2 index c2c15c03009..84c677f9c4f 100644 --- a/tekton/src/tasks/aiservice/aiservice.yml.j2 +++ b/tekton/src/tasks/aiservice/aiservice.yml.j2 @@ -143,6 +143,9 @@ spec: type: string - name: tenant_entitlement_end_date type: string + # Config file for scheduling constraints for AI workloads + - name: tenant_scheduling_cfg_file + type: string # RSL - name: rsl_url @@ -253,6 +256,8 @@ spec: value: $(params.tenant_entitlement_start_date) - name: AISERVICE_TENANT_ENTITLEMENT_END_DATE value: $(params.tenant_entitlement_end_date) + - name: AISERVICE_TENANT_SCHEDULING_CONFIG_FILE + value: $(params.tenant_scheduling_cfG_file) # RSL - name: RSL_URL @@ -324,3 +329,6 @@ spec: workspaces: - name: configs optional: true + - name: aiservice + workspace: shared-aiservice-config + optional: true From f2b813bc603098ed80fc008ba2cd8db8c4173be6 Mon Sep 17 00:00:00 2001 From: Jasmin Makwana Date: Fri, 1 May 2026 21:26:30 +0530 Subject: [PATCH 2/6] Bug fixes, Fix tests and code refactor --- .gitignore | 2 +- python/src/mas/cli/aiservice/install/app.py | 15 +++++++++++---- python/src/mas/cli/aiservice/install/params.py | 3 --- .../src/mas/cli/aiservice/install/summarizer.py | 2 +- .../mas/cli/install/settings/additionalConfigs.py | 2 ++ python/test/aiservice/install/test_app.py | 5 +++++ 6 files changed, 20 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 4c88e2e8aca..52b8d0cedfd 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,7 @@ .env .venv kubectl.exe -mas.log +mas.log* site/ report/ diff --git a/python/src/mas/cli/aiservice/install/app.py b/python/src/mas/cli/aiservice/install/app.py index f8fd04f4881..4144e50ddb3 100644 --- a/python/src/mas/cli/aiservice/install/app.py +++ b/python/src/mas/cli/aiservice/install/app.py @@ -37,9 +37,7 @@ # AI Service utilizes two distinct databases: DB2 is employed by the AiBroker component. # By default, AiService will deploy DB2 within the same namespace as MAS (db2u), but it will be configured as a separate DB2 instance. -from ...install.settings.mongodbSettings import MongoDbSettingsMixin -from ...install.settings.db2Settings import Db2SettingsMixin -from ...install.settings.additionalConfigs import AdditionalConfigsMixin +from ...install.settings import InstallSettingsMixin from mas.cli.validators import ( InstanceIDFormatValidator, @@ -75,7 +73,7 @@ def wrapper(self, *args, **kwargs): return wrapper -class AiServiceInstallApp(BaseApp, aiServiceInstallArgBuilderMixin, aiServiceInstallSummarizerMixin, MongoDbSettingsMixin, Db2SettingsMixin, AdditionalConfigsMixin, ConfigGeneratorMixin): +class AiServiceInstallApp(BaseApp, aiServiceInstallArgBuilderMixin, aiServiceInstallSummarizerMixin, InstallSettingsMixin, ConfigGeneratorMixin): @logMethodCall def processCatalogChoice(self) -> list: self.catalogDigest = self.chosenCatalog["catalog_digest"] @@ -131,6 +129,7 @@ def chooseInstallFlavour(self) -> None: "There are two flavours of the interactive install to choose from: Simplified and Advanced. The simplified option will present fewer dialogs, but you lose the ability to configure the following aspects of the installation:", " - Configure certificate issuer", " - Enable IPv6 SingleStack networking for services", + " - Configure Tolerations & nodeSelector for AI Service tenant" ]) self.showAdvancedOptions = self.yesOrNo("Show advanced installation options") @@ -189,6 +188,8 @@ def nonInteractiveMode(self) -> None: self.storageClassProvider = "custom" self.slsLicenseFileLocal = None + self.aiserviceTenantSchedulingConfigFileLocal = None + self.approvals = { "approval_aiservice": {"id": "aiservice"}, } @@ -250,6 +251,11 @@ def nonInteractiveMode(self) -> None: if len(value) == 0 or len(value) > 4: self.fatalError(f"Unsupported value for --s3-bucket-prefix(Must be 1-4 characters long): {value}") + elif key == "tenant_scheduling_config_file": + # No need to perform validation if file exist here, as it has been already validated by argParser type check. + if value is not None and value != "": + self.aiserviceTenantSchedulingConfigFileLocal = value + elif key == "non_prod": if not value: self.operationalMode = 1 @@ -601,6 +607,7 @@ def aiServiceTenantSettings(self) -> None: self.setParam("tenant_entitlement_start_date", today.strftime('%Y-%m-%d')) self.promptForString("Entitlement end date (YYYY-MM-DD)", "tenant_entitlement_end_date", default=oneyear.strftime('%Y-%m-%d')) + self.aiserviceTenantSchedulingConfigFileLocal = None self.configSchedulingConstraints() @logMethodCall diff --git a/python/src/mas/cli/aiservice/install/params.py b/python/src/mas/cli/aiservice/install/params.py index a5811b5e1d1..e2050315db0 100644 --- a/python/src/mas/cli/aiservice/install/params.py +++ b/python/src/mas/cli/aiservice/install/params.py @@ -110,7 +110,4 @@ # Slack "slack_token", "slack_channel", - - # Scheduling constraints - "tenant_scheduling_config_file", ] diff --git a/python/src/mas/cli/aiservice/install/summarizer.py b/python/src/mas/cli/aiservice/install/summarizer.py index fc5a14ebd1f..90c5335ee5f 100644 --- a/python/src/mas/cli/aiservice/install/summarizer.py +++ b/python/src/mas/cli/aiservice/install/summarizer.py @@ -58,7 +58,7 @@ def aiServiceSummary(self) -> None: self.printParamSummary("End Date", "tenant_entitlement_end_date") if self.aiserviceTenantSchedulingConfigFileLocal: - self.printParamSummary("Scheduling constraints config file", "tenant_scheduling_config_file") + self.printSummary("Scheduling constraints config file", self.aiserviceTenantSchedulingConfigFileLocal) self.printH2("S3 Configuration") # self.printParamSummary("Storage provider", "aiservice_s3_provider") diff --git a/python/src/mas/cli/install/settings/additionalConfigs.py b/python/src/mas/cli/install/settings/additionalConfigs.py index fb7fa37c20f..949861ebe7d 100644 --- a/python/src/mas/cli/install/settings/additionalConfigs.py +++ b/python/src/mas/cli/install/settings/additionalConfigs.py @@ -275,6 +275,8 @@ def slsLicenseFile(self) -> None: self.slsLicenseFileSecret = self.addFilesToSecret(slsLicenseFileSecret, self.slsLicenseFileLocal, '') def aiserviceConfig(self) -> None: + self.aiserviceConfigSecret = None + if self.aiserviceTenantSchedulingConfigFileLocal: aiserviceConfigSecret: dict[str, Any] = { "apiVersion": "v1", diff --git a/python/test/aiservice/install/test_app.py b/python/test/aiservice/install/test_app.py index d499ab2afa7..6a1d76e7e16 100644 --- a/python/test/aiservice/install/test_app.py +++ b/python/test/aiservice/install/test_app.py @@ -25,6 +25,7 @@ def test_install_noninteractive(tmpdir): tmpdir.join('authorized_entitlement.lic').write('testLicense') + tmpdir.join('aiservice-tenant-affinity-config.yaml').write('#') with mock.patch('mas.cli.cli.config'): dynamic_client = MagicMock(DynamicClient) resources = MagicMock() @@ -103,6 +104,7 @@ def test_install_noninteractive(tmpdir): '--tenant-entitlement-type', 'standard', '--tenant-entitlement-start-date', '2025-08-28', '--tenant-entitlement-end-date', '2026-08-28', + '--tenant-scheduling-constraints-file', f'{tmpdir}/aiservice-tenant-affinity-config.yaml', '--rsl-url', 'https:/test.rsl.maximo.ibm.com/api/v3/vector/query', '--rsl-org-id', 'testOrgId', '--rsl-token', 'testRslToken', @@ -113,6 +115,7 @@ def test_install_noninteractive(tmpdir): def test_install_interactive_advanced(tmpdir): tmpdir.join('authorized_entitlement.lic').write('testLicense') + tmpdir.join('aiservice-tenant-affinity-config.yaml').write('#') tmpdir.join('mongodb-system.yaml').write('#') tmpdir.join('cert.crt').write('#') with mock.patch('mas.cli.cli.config'): @@ -177,6 +180,8 @@ def set_mixin_prompt_input(**kwargs): return f'{tmpdir}/authorized_entitlement.lic' if re.match('.*Instance ID.*', message): return 'apmdevops' + if re.match('.*Scheduling constraints YAML file.*', message): + return f'{tmpdir}/aiservice-tenant-affinity-config.yaml' if re.match('.*Operational Mode.*', message): return '1' if re.match('.*Install Minio.*', message): From 8c5e84538bb76b7884d5595ed1b5e00649bf5821 Mon Sep 17 00:00:00 2001 From: Jasmin Makwana Date: Mon, 4 May 2026 11:57:02 +0530 Subject: [PATCH 3/6] Fix bug in aiservice task --- tekton/src/tasks/aiservice/aiservice.yml.j2 | 1 - 1 file changed, 1 deletion(-) diff --git a/tekton/src/tasks/aiservice/aiservice.yml.j2 b/tekton/src/tasks/aiservice/aiservice.yml.j2 index 84c677f9c4f..d389a7008f6 100644 --- a/tekton/src/tasks/aiservice/aiservice.yml.j2 +++ b/tekton/src/tasks/aiservice/aiservice.yml.j2 @@ -330,5 +330,4 @@ spec: - name: configs optional: true - name: aiservice - workspace: shared-aiservice-config optional: true From 7d776656c3bf2d99b28c206ca26bb3136664f101 Mon Sep 17 00:00:00 2001 From: Jasmin Makwana Date: Mon, 4 May 2026 12:18:40 +0530 Subject: [PATCH 4/6] Bug fix --- tekton/src/tasks/aiservice/aiservice.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tekton/src/tasks/aiservice/aiservice.yml.j2 b/tekton/src/tasks/aiservice/aiservice.yml.j2 index d389a7008f6..91849553dba 100644 --- a/tekton/src/tasks/aiservice/aiservice.yml.j2 +++ b/tekton/src/tasks/aiservice/aiservice.yml.j2 @@ -257,7 +257,7 @@ spec: - name: AISERVICE_TENANT_ENTITLEMENT_END_DATE value: $(params.tenant_entitlement_end_date) - name: AISERVICE_TENANT_SCHEDULING_CONFIG_FILE - value: $(params.tenant_scheduling_cfG_file) + value: $(params.tenant_scheduling_cfg_file) # RSL - name: RSL_URL From b8ba6e81bc4ae05e02a17b9002849e68bf353f50 Mon Sep 17 00:00:00 2001 From: Jasmin Makwana Date: Mon, 4 May 2026 15:42:11 +0530 Subject: [PATCH 5/6] Add support for scheduling configuration in mas install command --- python/src/mas/cli/aiservice/install/app.py | 6 ++-- .../mas/cli/aiservice/install/argBuilder.py | 2 +- .../mas/cli/aiservice/install/argParser.py | 4 +-- .../mas/cli/aiservice/install/summarizer.py | 4 +-- python/src/mas/cli/install/app.py | 31 ++++++++++++++++++- python/src/mas/cli/install/argBuilder.py | 2 ++ python/src/mas/cli/install/argParser.py | 7 +++++ python/src/mas/cli/install/summarizer.py | 4 ++- python/test/aiservice/install/test_app.py | 2 +- python/test/utils/install_test_helper.py | 1 + 10 files changed, 52 insertions(+), 11 deletions(-) diff --git a/python/src/mas/cli/aiservice/install/app.py b/python/src/mas/cli/aiservice/install/app.py index 4144e50ddb3..7f2303283af 100644 --- a/python/src/mas/cli/aiservice/install/app.py +++ b/python/src/mas/cli/aiservice/install/app.py @@ -129,7 +129,7 @@ def chooseInstallFlavour(self) -> None: "There are two flavours of the interactive install to choose from: Simplified and Advanced. The simplified option will present fewer dialogs, but you lose the ability to configure the following aspects of the installation:", " - Configure certificate issuer", " - Enable IPv6 SingleStack networking for services", - " - Configure Tolerations & nodeSelector for AI Service tenant" + " - Customize Scheduling configuration for AI workloads(Training pipeline & Inference services) for AI Service tenant" ]) self.showAdvancedOptions = self.yesOrNo("Show advanced installation options") @@ -613,7 +613,7 @@ def aiServiceTenantSettings(self) -> None: @logMethodCall def configSchedulingConstraints(self): if self.showAdvancedOptions: - self.printH1("Scheduling constraints for AI Workloads") + self.printH1("Scheduling configuration for AI Workloads") self.printDescription(content=[ "AI Service supports configuring tolerations and nodeSelector per tenant to schedule AI workloads(training pipelines & Inference services) on dedicated nodes.", "To configure tolerations and nodeSelector, create a YAML configuration file", @@ -622,7 +622,7 @@ def configSchedulingConstraints(self): " `nodeSelector`: Dictionary of node label key-value pairs", ]) - self.aiserviceTenantSchedulingConfigFileLocal = self.promptForFile("Scheduling constraints YAML file", mustExist=True, envVar="AISERVICE_TENANT_SCHEDULING_CONFIG_FILE") + self.aiserviceTenantSchedulingConfigFileLocal = self.promptForFile("Scheduling configuration YAML file", mustExist=True, envVar="AISERVICE_TENANT_SCHEDULING_CONFIG_FILE") def _setMinioStorageDefaults(self) -> None: """ diff --git a/python/src/mas/cli/aiservice/install/argBuilder.py b/python/src/mas/cli/aiservice/install/argBuilder.py index 7d8f22f66b8..843a67db70c 100644 --- a/python/src/mas/cli/aiservice/install/argBuilder.py +++ b/python/src/mas/cli/aiservice/install/argBuilder.py @@ -173,7 +173,7 @@ def buildCommand(self) -> str: if self.getParam('tenant_entitlement_end_date') != "": command += f" --tenant-entitlement-end-date \"{self.getParam('tenant_entitlement_end_date')}\"{newline}" if self.aiserviceTenantSchedulingConfigFileLocal: - command += f" --tenant-scheduling-constraints-file \"{self.aiserviceTenantSchedulingConfigFileLocal}\"{newline}" + command += f" --tenant-scheduling-config-file \"{self.aiserviceTenantSchedulingConfigFileLocal}\"{newline}" if self.getParam('rsl_url') != "": command += f" --rsl-url \"{self.getParam('rsl_url')}\"{newline}" diff --git a/python/src/mas/cli/aiservice/install/argParser.py b/python/src/mas/cli/aiservice/install/argParser.py index fb471e96501..3537e926041 100644 --- a/python/src/mas/cli/aiservice/install/argParser.py +++ b/python/src/mas/cli/aiservice/install/argParser.py @@ -445,10 +445,10 @@ def isValidFile(parser, arg) -> str: const="true" ) aiserviceAdvancedArgGroup.add_argument( - "--tenant-scheduling-constraints-file", + "--tenant-scheduling-config-file", dest="tenant_scheduling_config_file", required=False, - help="Path to the YAML file that contains the scheduling constraints for tenant", + help="Path to the YAML file that contains the scheduling configuration for tenant", type=lambda x: isValidFile(aiServiceinstallArgParser, x) ) diff --git a/python/src/mas/cli/aiservice/install/summarizer.py b/python/src/mas/cli/aiservice/install/summarizer.py index 90c5335ee5f..b32a4b905a8 100644 --- a/python/src/mas/cli/aiservice/install/summarizer.py +++ b/python/src/mas/cli/aiservice/install/summarizer.py @@ -52,13 +52,13 @@ def aiServiceSummary(self) -> None: self.printParamSummary("Configure AI Service to run in IPv6 mode", "enable_ipv6") - self.printH2("AI Service Tenant Entitlement") + self.printH2("AI Service Tenant Configuration") self.printParamSummary("Entitlement Type", "tenant_entitlement_type") self.printParamSummary("Start Date", "tenant_entitlement_start_date") self.printParamSummary("End Date", "tenant_entitlement_end_date") if self.aiserviceTenantSchedulingConfigFileLocal: - self.printSummary("Scheduling constraints config file", self.aiserviceTenantSchedulingConfigFileLocal) + self.printSummary("Scheduling configuration file", self.aiserviceTenantSchedulingConfigFileLocal) self.printH2("S3 Configuration") # self.printParamSummary("Storage provider", "aiservice_s3_provider") diff --git a/python/src/mas/cli/install/app.py b/python/src/mas/cli/install/app.py index 2ab1810ed2d..d3eca69f49a 100644 --- a/python/src/mas/cli/install/app.py +++ b/python/src/mas/cli/install/app.py @@ -1401,6 +1401,23 @@ def aiServiceTenantSettings(self) -> None: self.setParam("tenant_entitlement_start_date", today.strftime('%Y-%m-%d')) self.promptForString("Entitlement end date (YYYY-MM-DD)", "tenant_entitlement_end_date", default=oneyear.strftime('%Y-%m-%d')) + self.aiserviceTenantSchedulingConfigFileLocal = None + self.configSchedulingConstraints() + + @logMethodCall + def configSchedulingConstraints(self): + if self.showAdvancedOptions: + self.printH1("Scheduling configuration for AI Workloads") + self.printDescription(content=[ + "AI Service supports configuring tolerations and nodeSelector per tenant to schedule AI workloads(training pipelines & Inference services) on dedicated nodes.", + "To configure tolerations and nodeSelector, create a YAML configuration file", + "The YAML file must contain `pipeline` and/or `predictor` objects. Each object can have:", + " `tolerations`: List of Kubernetes tolerations (required fields: `key`, `operator`, `effect`)", + " `nodeSelector`: Dictionary of node label key-value pairs", + ]) + + self.aiserviceTenantSchedulingConfigFileLocal = self.promptForFile("Scheduling configuration YAML file", mustExist=True, envVar="AISERVICE_TENANT_SCHEDULING_CONFIG_FILE") + @logMethodCall def _setMinioStorageDefaults(self) -> None: """ @@ -1496,7 +1513,8 @@ def chooseInstallFlavour(self) -> None: " - Enable optional Real Estate and Facilities configurations", " - Customize Db2 node affinity and tolerations, memory, cpu, and storage settings (when using the IBM Db2 Universal Operator)", " - Choose alternative Apache Kafka providers (default to Strimzi)", - " - Customize Grafana storage settings" + " - Customize Grafana storage settings", + " - Customize Scheduling configuration for AI workloads(Training pipeline & Inference services) for AI Service tenant" ]) self.showAdvancedOptions = self.yesOrNo("Show advanced installation options") @@ -1505,6 +1523,9 @@ def interactiveMode(self, simplified: bool, advanced: bool) -> None: # Interactive mode self.isInteractiveMode = True + # Initialize attributes that may be used later + self.aiserviceTenantSchedulingConfigFileLocal = None + if simplified: self.showAdvancedOptions = False elif advanced: @@ -1580,6 +1601,7 @@ def nonInteractiveMode(self) -> None: self.db2SetTolerations = False self.installAIService = False self.slsLicenseFileLocal = None + self.aiserviceTenantSchedulingConfigFileLocal = None self.approvals: Dict[str, Dict[str, Any]] = { "approval_core": {"id": "suite-verify"}, # After Core Platform verification has completed @@ -1842,6 +1864,11 @@ def nonInteractiveMode(self) -> None: if len(value) == 0 or len(value) > 4: self.fatalError(f"Unsupported value for --s3-bucket-prefix(Must be 1-4 characters long): {value}") + elif key == "tenant_scheduling_config_file": + # No need to perform validation if file exist here, as it has been already validated by argParser type check. + if value is not None and value != "": + self.aiserviceTenantSchedulingConfigFileLocal = value + # Fail if there's any arguments we don't know how to handle else: print(f"Unknown option: {key} {value}") @@ -1989,6 +2016,7 @@ def install(self, argv): self.podTemplates() self.slsLicenseFile() self.manualCertificates() + self.aiserviceConfig() # Show a summary of the installation configuration self.printH1("Non-Interactive Install Command") @@ -2157,6 +2185,7 @@ def install(self, argv): additionalConfigs=self.additionalConfigsSecret, podTemplates=self.podTemplatesSecret, certs=self.certsSecret, + aiserviceConfig=self.aiserviceConfigSecret, slack_token=self.getParam("slack_token"), slack_channel=self.getParam("slack_channel") ) diff --git a/python/src/mas/cli/install/argBuilder.py b/python/src/mas/cli/install/argBuilder.py index 05bbe90fdf9..b602365df90 100644 --- a/python/src/mas/cli/install/argBuilder.py +++ b/python/src/mas/cli/install/argBuilder.py @@ -411,6 +411,8 @@ def buildCommand(self) -> str: command += f" --tenant-entitlement-start-date \"{self.getParam('tenant_entitlement_start_date')}\"{newline}" if self.getParam('tenant_entitlement_end_date') != "": command += f" --tenant-entitlement-end-date \"{self.getParam('tenant_entitlement_end_date')}\"{newline}" + if self.aiserviceTenantSchedulingConfigFileLocal: + command += f" --tenant-scheduling-config-file \"{self.aiserviceTenantSchedulingConfigFileLocal}\"{newline}" if self.getParam('rsl_url') != "": command += f" --rsl-url \"{self.getParam('rsl_url')}\"{newline}" diff --git a/python/src/mas/cli/install/argParser.py b/python/src/mas/cli/install/argParser.py index bd5a7922fa8..16a72ad46aa 100644 --- a/python/src/mas/cli/install/argParser.py +++ b/python/src/mas/cli/install/argParser.py @@ -1103,6 +1103,13 @@ def isValidFile(parser: argparse.ArgumentParser, arg: str) -> str: required=False, help="Provide the name of the Issuer to configure AI Service to issue certificates", ) +aiServiceArgGroup.add_argument( + "--tenant-scheduling-config-file", + dest="tenant_scheduling_config_file", + required=False, + help="Path to the YAML file that contains the scheduling configuration for tenant", + type=lambda x: isValidFile(installArgParser, x) +) # IBM Cloud Pak for Data # ----------------------------------------------------------------------------- diff --git a/python/src/mas/cli/install/summarizer.py b/python/src/mas/cli/install/summarizer.py index c0c75bfc326..416f55fc459 100644 --- a/python/src/mas/cli/install/summarizer.py +++ b/python/src/mas/cli/install/summarizer.py @@ -257,10 +257,12 @@ def aiServiceSummary(self) -> None: if "aiservice_certificate_issuer" in self.params: self.printParamSummary("Certificate Issuer", "aiservice_certificate_issuer") - self.printH2("AI Service Tenant Entitlement") + self.printH2("AI Service Tenant Configuration") self.printParamSummary("Entitlement Type", "tenant_entitlement_type") self.printParamSummary("Start Date", "tenant_entitlement_start_date") self.printParamSummary("End Date", "tenant_entitlement_end_date") + if self.aiserviceTenantSchedulingConfigFileLocal: + self.printSummary("Scheduling configuration file", self.aiserviceTenantSchedulingConfigFileLocal) self.printH2("S3 Configuration") # self.printParamSummary("Storage provider", "aiservice_s3_provider") diff --git a/python/test/aiservice/install/test_app.py b/python/test/aiservice/install/test_app.py index 6a1d76e7e16..b53bac9f60f 100644 --- a/python/test/aiservice/install/test_app.py +++ b/python/test/aiservice/install/test_app.py @@ -104,7 +104,7 @@ def test_install_noninteractive(tmpdir): '--tenant-entitlement-type', 'standard', '--tenant-entitlement-start-date', '2025-08-28', '--tenant-entitlement-end-date', '2026-08-28', - '--tenant-scheduling-constraints-file', f'{tmpdir}/aiservice-tenant-affinity-config.yaml', + '--tenant-scheduling-config-file', f'{tmpdir}/aiservice-tenant-affinity-config.yaml', '--rsl-url', 'https:/test.rsl.maximo.ibm.com/api/v3/vector/query', '--rsl-org-id', 'testOrgId', '--rsl-token', 'testRslToken', diff --git a/python/test/utils/install_test_helper.py b/python/test/utils/install_test_helper.py index 6edfcff20c1..98cda3bf437 100644 --- a/python/test/utils/install_test_helper.py +++ b/python/test/utils/install_test_helper.py @@ -96,6 +96,7 @@ def setup_test_files(self): self.tmpdir.join('authorized_entitlement.lic').write('testLicense') self.tmpdir.join('mongodb-system.yaml').write('#') self.tmpdir.join('cert.crt').write('#') + self.tmpdir.join('aiservice-tenant-affinity-config.yaml').write('#') def start_watchdog(self): """Start watchdog thread to detect hanging prompts.""" From c0eee5d0085aae02cdc561700fbd995058961cb4 Mon Sep 17 00:00:00 2001 From: Jasmin Makwana Date: Mon, 4 May 2026 19:38:29 +0530 Subject: [PATCH 6/6] Update doc --- docs/guides/aiservice-install.md | 47 ++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/docs/guides/aiservice-install.md b/docs/guides/aiservice-install.md index d262050e0bb..bc6940ef211 100644 --- a/docs/guides/aiservice-install.md +++ b/docs/guides/aiservice-install.md @@ -88,6 +88,7 @@ The interactive install will guide you through the following steps:
  • Database Configuration: Set up database connection for AI Service
  • RSL Configuration: Configure Red Hat Service Locator integration
  • Tenant Configuration: Set up AI Service tenant(s)
    • +
  • Customize pod scheduling configuration for AI Workloads (Advanced Mode Only): Configure tolerations & nodeSelector for AI workloads (Training pipeline & Inference Service). See Scheduling Configuration File Format for file configuration details.
  • Operational Mode: Choose between production or non-production mode
    • @@ -153,6 +154,7 @@ docker run -e IBM_ENTITLEMENT_KEY -ti --rm -v ~:/mnt/home quay.io/ibmmas/cli:@@C --tenant-entitlement-type standard \ --tenant-entitlement-start-date 2025-01-01 \ --tenant-entitlement-end-date 2026-01-01 \ + --tenant-scheduling-config-file "/mnt/home/aiservice-tenant-affinity.yaml" \ \ --rsl-url http://your-rsl-host:3001/api/v3/vector/query \ --rsl-org-id your_org_id \ @@ -269,6 +271,51 @@ docker run -e IBM_ENTITLEMENT_KEY -ti --rm -v ~:/mnt/home quay.io/ibmmas/cli:@@C | `--rsl-org-id` | RSL organization ID | Optional | `your_org_id` | | `--rsl-token` | RSL authentication token | Optional | `Bearer your_token` | +### AI Workload Scheduling Configuration + +| Parameter | Description | Required | Example | +|-----------|-------------|----------|---------| +| `--aiservice-scheduling-config-file` | Path to YAML file containing scheduling configuration for AI workloads (training pipeline & inference service) | Optional | `/mnt/home/scheduling-config.yaml` | + +!!! note "Scheduling Configuration" + The scheduling configuration allows you to customize pod placement for AI workloads using tolerations and nodeSelector. This is useful for dedicating specific nodes to AI workloads or ensuring proper resource allocation. + +#### Scheduling Configuration File Format + +The scheduling configuration file must be a YAML file with the following structure: + +```yaml +pipeline: + tolerations: + - key: "kmodels" + operator: "Equal" + value: "pipeline" + effect: "NoSchedule" + nodeSelector: + kmodels: pipeline +predictor: + tolerations: + - key: "kmodels" + operator: "Equal" + value: "inference" + effect: "NoSchedule" + nodeSelector: + kmodels: inference +``` + +**Configuration File Structure**: + +The YAML file must contain `pipeline` and/or `predictor` objects. Each object can have: + - `tolerations`: List of Kubernetes tolerations (required fields: `key`, `operator`, `effect`) + - `nodeSelector`: Dictionary of node label key-value pairs +At least one of `tolerations` or `nodeSelector` must be defined for each non-empty object. + + +!!! tip "Use Cases" + - **Dedicated GPU Nodes**: Use nodeSelector to schedule AI workloads on GPU-enabled nodes + - **Resource Isolation**: Use tolerations to ensure AI workloads run on dedicated nodes with specific taints + - **Multi-tenant Environments**: Separate AI workloads from other cluster workloads using node affinity + ### Additional Options | Parameter | Description | Required |