[minor] Support pod scheduling configuration for AI Workloads for AI Service tenant

.gitignore

@@ -5,7 +5,7 @@
 .env
 .venv
 kubectl.exe
-mas.log
+mas.log*
 
 site/
 report/

.secrets.baseline

@@ -3,7 +3,7 @@
     "files": "build/bin/config/oscap/ssg-rhel9-ds.xml|^.secrets.baseline$|^docs/catalogs/",
     "lines": null
   },
-  "generated_at": "2026-04-26T12:19:35Z",
+  "generated_at": "2026-05-01T10:41:14Z",
   "plugins_used": [
     {
       "name": "AWSKeyDetector"

docs/guides/aiservice-install.md

@@ -88,6 +88,7 @@ The interactive install will guide you through the following steps:
         <li><strong>Database Configuration:</strong> Set up database connection for AI Service</li>
         <li><strong>RSL Configuration:</strong> Configure Red Hat Service Locator integration</li>
         <li><strong>Tenant Configuration:</strong> Set up AI Service tenant(s)</li>
+        <li><strong>Customize pod scheduling configuration for AI Workloads (Advanced Mode Only):</strong> Configure tolerations & nodeSelector for AI workloads (Training pipeline & Inference Service). See <a href="#scheduling-configuration-file-format">Scheduling Configuration File Format</a> for file configuration details.</li>
         <li><strong>Operational Mode:</strong> Choose between production or non-production mode</li>
       </ul>
     </cds-accordion-item>
@@ -153,6 +154,7 @@ docker run -e IBM_ENTITLEMENT_KEY -ti --rm -v ~:/mnt/home quay.io/ibmmas/cli:@@C
     --tenant-entitlement-type standard \
     --tenant-entitlement-start-date 2025-01-01 \
     --tenant-entitlement-end-date 2026-01-01 \
+    --tenant-scheduling-config-file "/mnt/home/aiservice-tenant-affinity.yaml" \
     \
     --rsl-url http://your-rsl-host:3001/api/v3/vector/query \
     --rsl-org-id your_org_id \
@@ -269,6 +271,51 @@ docker run -e IBM_ENTITLEMENT_KEY -ti --rm -v ~:/mnt/home quay.io/ibmmas/cli:@@C
 | `--rsl-org-id` | RSL organization ID | Optional | `your_org_id` |
 | `--rsl-token` | RSL authentication token | Optional | `Bearer your_token` |
 
+### AI Workload Scheduling Configuration
+
+| Parameter | Description | Required | Example |
+|-----------|-------------|----------|---------|
+| `--aiservice-scheduling-config-file` | Path to YAML file containing scheduling configuration for AI workloads (training pipeline & inference service) | Optional | `/mnt/home/scheduling-config.yaml` |
+
+!!! note "Scheduling Configuration"
+    The scheduling configuration allows you to customize pod placement for AI workloads using tolerations and nodeSelector. This is useful for dedicating specific nodes to AI workloads or ensuring proper resource allocation.
+
+#### Scheduling Configuration File Format
+
+The scheduling configuration file must be a YAML file with the following structure:
+
+```yaml
+pipeline:
+  tolerations:
+    - key: "kmodels"
+      operator: "Equal"
+      value: "pipeline"
+      effect: "NoSchedule"
+  nodeSelector:
+    kmodels: pipeline
+predictor:
+  tolerations:
+    - key: "kmodels"
+      operator: "Equal"
+      value: "inference"
+      effect: "NoSchedule"
+  nodeSelector:
+    kmodels: inference
+```
+
+**Configuration File Structure**:
+
+The YAML file must contain `pipeline` and/or `predictor` objects. Each object can have:
+  - `tolerations`: List of Kubernetes tolerations (required fields: `key`, `operator`, `effect`)
+  - `nodeSelector`: Dictionary of node label key-value pairs
+At least one of `tolerations` or `nodeSelector` must be defined for each non-empty object.
+
+
+!!! tip "Use Cases"
+    - **Dedicated GPU Nodes**: Use nodeSelector to schedule AI workloads on GPU-enabled nodes
+    - **Resource Isolation**: Use tolerations to ensure AI workloads run on dedicated nodes with specific taints
+    - **Multi-tenant Environments**: Separate AI workloads from other cluster workloads using node affinity
+
 ### Additional Options
 
 | Parameter | Description | Required |

python/src/mas/cli/aiservice/install/app.py

@@ -37,9 +37,7 @@
 # AI Service utilizes two distinct databases: DB2 is employed by the AiBroker component.
 # By default, AiService will deploy DB2 within the same namespace as MAS (db2u), but it will be configured as a separate DB2 instance.
 
-from ...install.settings.mongodbSettings import MongoDbSettingsMixin
-from ...install.settings.db2Settings import Db2SettingsMixin
-from ...install.settings.additionalConfigs import AdditionalConfigsMixin
+from ...install.settings import InstallSettingsMixin
 
 from mas.cli.validators import (
     InstanceIDFormatValidator,
@@ -75,7 +73,7 @@ def wrapper(self, *args, **kwargs):
     return wrapper
 
 
-class AiServiceInstallApp(BaseApp, aiServiceInstallArgBuilderMixin, aiServiceInstallSummarizerMixin, MongoDbSettingsMixin, Db2SettingsMixin, AdditionalConfigsMixin, ConfigGeneratorMixin):
+class AiServiceInstallApp(BaseApp, aiServiceInstallArgBuilderMixin, aiServiceInstallSummarizerMixin, InstallSettingsMixin, ConfigGeneratorMixin):
     @logMethodCall
     def processCatalogChoice(self) -> list:
         self.catalogDigest = self.chosenCatalog["catalog_digest"]
@@ -131,6 +129,7 @@ def chooseInstallFlavour(self) -> None:
             "There are two flavours of the interactive install to choose from: <u>Simplified</u> and <u>Advanced</u>.  The simplified option will present fewer dialogs, but you lose the ability to configure the following aspects of the installation:",
             " - Configure certificate issuer",
             " - Enable IPv6 SingleStack networking for services",
+            " - Customize Scheduling configuration for AI workloads(Training pipeline & Inference services) for AI Service tenant"
         ])
         self.showAdvancedOptions = self.yesOrNo("Show advanced installation options")
 
@@ -189,6 +188,8 @@ def nonInteractiveMode(self) -> None:
         self.storageClassProvider = "custom"
         self.slsLicenseFileLocal = None
 
+        self.aiserviceTenantSchedulingConfigFileLocal = None
+
         self.approvals = {
             "approval_aiservice": {"id": "aiservice"},
         }
@@ -250,6 +251,11 @@ def nonInteractiveMode(self) -> None:
                 if len(value) == 0 or len(value) > 4:
                     self.fatalError(f"Unsupported value for --s3-bucket-prefix(Must be 1-4 characters long): {value}")
 
+            elif key == "tenant_scheduling_config_file":
+                # No need to perform validation if file exist here, as it has been already validated by argParser type check.
+                if value is not None and value != "":
+                    self.aiserviceTenantSchedulingConfigFileLocal = value
+
             elif key == "non_prod":
                 if not value:
                     self.operationalMode = 1
@@ -447,6 +453,8 @@ def install(self, argv):
         # Set up the sls license file
         self.slsLicenseFile()
 
+        self.aiserviceConfig()
+
         # Show a summary of the installation configuration
         self.printH1("Non-Interactive Install Command")
         self.printDescription([
@@ -494,6 +502,7 @@ def install(self, argv):
                     additionalConfigs=self.additionalConfigsSecret,
                     podTemplates=self.podTemplatesSecret,
                     certs=self.certsSecret,
+                    aiserviceConfig=self.aiserviceConfigSecret,
                     slack_token=self.getParam("slack_token"),
                     slack_channel=self.getParam("slack_channel")
                 )
@@ -598,6 +607,23 @@ def aiServiceTenantSettings(self) -> None:
         self.setParam("tenant_entitlement_start_date", today.strftime('%Y-%m-%d'))
         self.promptForString("Entitlement end date (YYYY-MM-DD)", "tenant_entitlement_end_date", default=oneyear.strftime('%Y-%m-%d'))
 
+        self.aiserviceTenantSchedulingConfigFileLocal = None
+        self.configSchedulingConstraints()
+
+    @logMethodCall
+    def configSchedulingConstraints(self):
+        if self.showAdvancedOptions:
+            self.printH1("Scheduling configuration for AI Workloads")
+            self.printDescription(content=[
+                "AI Service supports configuring tolerations and nodeSelector per tenant to schedule AI workloads(training pipelines & Inference services) on dedicated nodes.",
+                "To configure tolerations and nodeSelector, create a YAML configuration file",
+                "The YAML file must contain `pipeline` and/or `predictor` objects. Each object can have:",
+                "  `tolerations`: List of Kubernetes tolerations (required fields: `key`, `operator`, `effect`)",
+                "  `nodeSelector`: Dictionary of node label key-value pairs",
+            ])
+
+            self.aiserviceTenantSchedulingConfigFileLocal = self.promptForFile("Scheduling configuration YAML file", mustExist=True, envVar="AISERVICE_TENANT_SCHEDULING_CONFIG_FILE")
+
     def _setMinioStorageDefaults(self) -> None:
         """
         Set MinIO storage defaults when MinIO is being installed in-cluster.

python/src/mas/cli/aiservice/install/argBuilder.py

@@ -172,6 +172,8 @@ def buildCommand(self) -> str:
             command += f"  --tenant-entitlement-start-date \"{self.getParam('tenant_entitlement_start_date')}\"{newline}"
         if self.getParam('tenant_entitlement_end_date') != "":
             command += f"  --tenant-entitlement-end-date \"{self.getParam('tenant_entitlement_end_date')}\"{newline}"
+        if self.aiserviceTenantSchedulingConfigFileLocal:
+            command += f"  --tenant-scheduling-config-file \"{self.aiserviceTenantSchedulingConfigFileLocal}\"{newline}"
 
         if self.getParam('rsl_url') != "":
             command += f"  --rsl-url \"{self.getParam('rsl_url')}\"{newline}"

python/src/mas/cli/aiservice/install/argParser.py

@@ -23,7 +23,7 @@ def isValidFile(parser, arg) -> str:
 
 
 aiServiceinstallArgParser = argparse.ArgumentParser(
-    prog="mas install-aiservice",
+    prog="mas aiservice-install",
     description="\n".join([
         f"IBM Maximo Application Suite Admin CLI v{packageVersion}",
         "Install Aiservice by configuring and launching the Tekton Pipeline.\n",
@@ -444,6 +444,13 @@ def isValidFile(parser, arg) -> str:
     action="store_const",
     const="true"
 )
+aiserviceAdvancedArgGroup.add_argument(
+    "--tenant-scheduling-config-file",
+    dest="tenant_scheduling_config_file",
+    required=False,
+    help="Path to the YAML file that contains the scheduling configuration for tenant",
+    type=lambda x: isValidFile(aiServiceinstallArgParser, x)
+)
 
 
 # IBM Db2 Universal Operator

python/src/mas/cli/aiservice/install/summarizer.py

@@ -52,11 +52,14 @@ def aiServiceSummary(self) -> None:
 
         self.printParamSummary("Configure AI Service to run in IPv6 mode", "enable_ipv6")
 
-        self.printH2("AI Service Tenant Entitlement")
+        self.printH2("AI Service Tenant Configuration")
         self.printParamSummary("Entitlement Type", "tenant_entitlement_type")
         self.printParamSummary("Start Date", "tenant_entitlement_start_date")
         self.printParamSummary("End Date", "tenant_entitlement_end_date")
 
+        if self.aiserviceTenantSchedulingConfigFileLocal:
+            self.printSummary("Scheduling configuration file", self.aiserviceTenantSchedulingConfigFileLocal)
+
         self.printH2("S3 Configuration")
         # self.printParamSummary("Storage provider", "aiservice_s3_provider")
         if self.getParam("minio_root_user") is not None and self.getParam("minio_root_user") != "":

python/src/mas/cli/install/app.py

@@ -1401,6 +1401,23 @@ def aiServiceTenantSettings(self) -> None:
             self.setParam("tenant_entitlement_start_date", today.strftime('%Y-%m-%d'))
             self.promptForString("Entitlement end date (YYYY-MM-DD)", "tenant_entitlement_end_date", default=oneyear.strftime('%Y-%m-%d'))
 
+            self.aiserviceTenantSchedulingConfigFileLocal = None
+            self.configSchedulingConstraints()
+
+    @logMethodCall
+    def configSchedulingConstraints(self):
+        if self.showAdvancedOptions:
+            self.printH1("Scheduling configuration for AI Workloads")
+            self.printDescription(content=[
+                "AI Service supports configuring tolerations and nodeSelector per tenant to schedule AI workloads(training pipelines & Inference services) on dedicated nodes.",
+                "To configure tolerations and nodeSelector, create a YAML configuration file",
+                "The YAML file must contain `pipeline` and/or `predictor` objects. Each object can have:",
+                "  `tolerations`: List of Kubernetes tolerations (required fields: `key`, `operator`, `effect`)",
+                "  `nodeSelector`: Dictionary of node label key-value pairs",
+            ])
+
+            self.aiserviceTenantSchedulingConfigFileLocal = self.promptForFile("Scheduling configuration YAML file", mustExist=True, envVar="AISERVICE_TENANT_SCHEDULING_CONFIG_FILE")
+
     @logMethodCall
     def _setMinioStorageDefaults(self) -> None:
         """
@@ -1496,7 +1513,8 @@ def chooseInstallFlavour(self) -> None:
             " - Enable optional Real Estate and Facilities configurations",
             " - Customize Db2 node affinity and tolerations, memory, cpu, and storage settings (when using the IBM Db2 Universal Operator)",
             " - Choose alternative Apache Kafka providers (default to Strimzi)",
-            " - Customize Grafana storage settings"
+            " - Customize Grafana storage settings",
+            " - Customize Scheduling configuration for AI workloads(Training pipeline & Inference services) for AI Service tenant"
         ])
         self.showAdvancedOptions = self.yesOrNo("Show advanced installation options")
 
@@ -1505,6 +1523,9 @@ def interactiveMode(self, simplified: bool, advanced: bool) -> None:
         # Interactive mode
         self.isInteractiveMode = True
 
+        # Initialize attributes that may be used later
+        self.aiserviceTenantSchedulingConfigFileLocal = None
+
         if simplified:
             self.showAdvancedOptions = False
         elif advanced:
@@ -1580,6 +1601,7 @@ def nonInteractiveMode(self) -> None:
         self.db2SetTolerations = False
         self.installAIService = False
         self.slsLicenseFileLocal = None
+        self.aiserviceTenantSchedulingConfigFileLocal = None
 
         self.approvals: Dict[str, Dict[str, Any]] = {
             "approval_core": {"id": "suite-verify"},  # After Core Platform verification has completed
@@ -1842,6 +1864,11 @@ def nonInteractiveMode(self) -> None:
                 if len(value) == 0 or len(value) > 4:
                     self.fatalError(f"Unsupported value for --s3-bucket-prefix(Must be 1-4 characters long): {value}")
 
+            elif key == "tenant_scheduling_config_file":
+                # No need to perform validation if file exist here, as it has been already validated by argParser type check.
+                if value is not None and value != "":
+                    self.aiserviceTenantSchedulingConfigFileLocal = value
+
             # Fail if there's any arguments we don't know how to handle
             else:
                 print(f"Unknown option: {key} {value}")
@@ -1989,6 +2016,7 @@ def install(self, argv):
         self.podTemplates()
         self.slsLicenseFile()
         self.manualCertificates()
+        self.aiserviceConfig()
 
         # Show a summary of the installation configuration
         self.printH1("Non-Interactive Install Command")
@@ -2157,6 +2185,7 @@ def install(self, argv):
                     additionalConfigs=self.additionalConfigsSecret,
                     podTemplates=self.podTemplatesSecret,
                     certs=self.certsSecret,
+                    aiserviceConfig=self.aiserviceConfigSecret,
                     slack_token=self.getParam("slack_token"),
                     slack_channel=self.getParam("slack_channel")
                 )

python/src/mas/cli/install/argBuilder.py

@@ -411,6 +411,8 @@ def buildCommand(self) -> str:
                 command += f"  --tenant-entitlement-start-date \"{self.getParam('tenant_entitlement_start_date')}\"{newline}"
             if self.getParam('tenant_entitlement_end_date') != "":
                 command += f"  --tenant-entitlement-end-date \"{self.getParam('tenant_entitlement_end_date')}\"{newline}"
+            if self.aiserviceTenantSchedulingConfigFileLocal:
+                command += f"  --tenant-scheduling-config-file \"{self.aiserviceTenantSchedulingConfigFileLocal}\"{newline}"
 
             if self.getParam('rsl_url') != "":
                 command += f"  --rsl-url \"{self.getParam('rsl_url')}\"{newline}"

python/src/mas/cli/install/argParser.py

@@ -1103,6 +1103,13 @@ def isValidFile(parser: argparse.ArgumentParser, arg: str) -> str:
     required=False,
     help="Provide the name of the Issuer to configure AI Service to issue certificates",
 )
+aiServiceArgGroup.add_argument(
+    "--tenant-scheduling-config-file",
+    dest="tenant_scheduling_config_file",
+    required=False,
+    help="Path to the YAML file that contains the scheduling configuration for tenant",
+    type=lambda x: isValidFile(installArgParser, x)
+)
 
 # IBM Cloud Pak for Data
 # -----------------------------------------------------------------------------

python/src/mas/cli/install/settings/additionalConfigs.py

@@ -34,10 +34,12 @@ class AdditionalConfigsMixin():
         slsLicenseFileLocal: str | None
         manualCertsDir: str | None
         showAdvancedOptions: bool
+        aiserviceTenantSchedulingConfigFileLocal: str | None
         additionalConfigsSecret: Dict[str, Any] | None
         podTemplatesSecret: Dict[str, Any] | None
         slsLicenseFileSecret: Dict[str, Any] | None
         certsSecret: Dict[str, Any] | None
+        aiserviceConfigSecret: Dict[str, Any] | None
 
         # Methods from BaseApp
         def setParam(self, param: str, value: str) -> None:
@@ -272,6 +274,21 @@ def slsLicenseFile(self) -> None:
             self.setParam("sls_entitlement_file", f"/workspace/entitlement/{path.basename(self.slsLicenseFileLocal)}")
             self.slsLicenseFileSecret = self.addFilesToSecret(slsLicenseFileSecret, self.slsLicenseFileLocal, '')
 
+    def aiserviceConfig(self) -> None:
+        self.aiserviceConfigSecret = None
+
+        if self.aiserviceTenantSchedulingConfigFileLocal:
+            aiserviceConfigSecret: dict[str, Any] = {
+                "apiVersion": "v1",
+                "kind": "Secret",
+                "type": "Opaque",
+                "metadata": {
+                    "name": "pipeline-aiservice-config"
+                }
+            }
+            self.setParam("tenant_scheduling_config_file", f"/workspace/aiservice/{path.basename(self.aiserviceTenantSchedulingConfigFileLocal)}")
+            self.aiserviceConfigSecret = self.addFilesToSecret(aiserviceConfigSecret, self.aiserviceTenantSchedulingConfigFileLocal, 'yaml')
+
     def addFilesToSecret(self, secretDict: dict, configPath: str, extension: str, keyPrefix: str = '') -> dict:
         """
         Add file (or files) to pipeline-additional-configs

python/src/mas/cli/install/summarizer.py

@@ -257,10 +257,12 @@ def aiServiceSummary(self) -> None:
             if "aiservice_certificate_issuer" in self.params:
                 self.printParamSummary("Certificate Issuer", "aiservice_certificate_issuer")
 
-            self.printH2("AI Service Tenant Entitlement")
+            self.printH2("AI Service Tenant Configuration")
             self.printParamSummary("Entitlement Type", "tenant_entitlement_type")
             self.printParamSummary("Start Date", "tenant_entitlement_start_date")
             self.printParamSummary("End Date", "tenant_entitlement_end_date")
+            if self.aiserviceTenantSchedulingConfigFileLocal:
+                self.printSummary("Scheduling configuration file", self.aiserviceTenantSchedulingConfigFileLocal)
 
             self.printH2("S3 Configuration")
             # self.printParamSummary("Storage provider", "aiservice_s3_provider")