Move node bootstrap files from /tmp to /opt/parallelcluster/tmp

CHANGELOG.md

@@ -7,6 +7,8 @@ CHANGELOG
 **ENHANCEMENTS**
 - Improve cluster update resiliency on login nodes by reusing the head-node-driven orchestration already in place on compute nodes,
   removing the dependency on cfn-hup and cfn-init.
+- Move all ParallelCluster-managed bootstrap files off `/tmp` into a dedicated `/opt/parallelcluster/tmp`
+  directory. Therefore, Image builds, cluster creations and updates work on custom AMIs that mount `/tmp` with `noexec`.
 
 **CHANGES**
 - The validator `ClusterNameValidator` now enforces cluster names to be limited to 40 characters when using `ExternalSlurmdbd`, 

cli/src/pcluster/constants.py

@@ -245,6 +245,12 @@
 
 DEFAULT_EPHEMERAL_DIR = "/scratch"
 
+# Dedicated ParallelCluster-managed directory used to stage bootstrap files (dna.json, extra.json,
+# wait_condition_handle.txt, bootstrap.sh) instead of /tmp, so bootstrap works when /tmp is mounted
+# with noexec. Defined once here and referenced by cluster_stack.py (cfn-init) and the user_data
+# templates (via the PclusterTmpDir Fn::Sub variable).
+PCLUSTER_TMP_DIR = "/opt/parallelcluster/tmp"
+
 LAMBDA_VPC_ACCESS_MANAGED_POLICY = "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole"
 
 IAM_NAME_PREFIX_LENGTH_LIMIT = 30

cli/src/pcluster/resources/compute_node/user_data.sh

@@ -51,17 +51,17 @@ datasource_list: [ Ec2, None ]
 output:
   all: "| tee -a /var/log/cloud-init-output.log | logger -t user-data -s 2>/dev/ttyS0"
 write_files:
-  - path: /tmp/dna.json
+  - path: ${PclusterTmpDir}/dna.json
     permissions: '0644'
     owner: root:root
     content: |
       ${DnaJson}
-  - path: /tmp/extra.json
+  - path: ${PclusterTmpDir}/extra.json
     permissions: '0644'
     owner: root:root
     content: |
       ${ExtraJson}
-  - path: /tmp/bootstrap.sh
+  - path: ${PclusterTmpDir}/bootstrap.sh
     permissions: '0744'
     owner: root:root
     content: |
@@ -145,12 +145,12 @@ write_files:
         done
         vendor_cookbook
       fi
-      cd /tmp
+      cd ${PclusterTmpDir}
       mkdir -p /etc/chef/ohai/hints
       touch /etc/chef/ohai/hints/ec2.json
 
       start=$(date +%s)
-      jq -s ".[0] * .[1]" /tmp/dna.json /tmp/extra.json > /etc/chef/dna.json || ( echo "jq not installed"; cp /tmp/dna.json /etc/chef/dna.json )
+      jq -s ".[0] * .[1]" ${PclusterTmpDir}/dna.json ${PclusterTmpDir}/extra.json > /etc/chef/dna.json || ( echo "jq not installed"; cp ${PclusterTmpDir}/dna.json /etc/chef/dna.json )
       {
         CINC_CMD="cinc-client --local-mode --config /etc/chef/client.rb --log_level info --logfile /var/log/chef-client.log --force-formatter --no-color --chef-zero-port 8889 --json-attributes /etc/chef/dna.json --override-runlist"
         FR_CMD="/opt/parallelcluster/scripts/fetch_and_run"
@@ -192,8 +192,8 @@ function error_exit
 }
 
 if [ "${Timeout}" == "NONE" ]; then
-  /tmp/bootstrap.sh
+  ${PclusterTmpDir}/bootstrap.sh
 else
-  timeout ${Timeout} /tmp/bootstrap.sh || error_exit
+  timeout ${Timeout} ${PclusterTmpDir}/bootstrap.sh || error_exit
 fi
 --==BOUNDARY==

cli/src/pcluster/resources/head_node/user_data.sh

@@ -72,7 +72,7 @@ function error_exit
   sleep 10
   # trim the error message because there is a size limit of 4096 bytes for cfn-signal
   # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/cloudformation-limits.html
-  cutoff=$(expr 4096 - $(stat --printf="%s" /tmp/wait_condition_handle.txt))
+  cutoff=$(expr 4096 - $(stat --printf="%s" ${PclusterTmpDir}/wait_condition_handle.txt))
   reason=$(head --bytes=${!cutoff} /var/log/parallelcluster/bootstrap_error_msg 2>/dev/null) || reason="$1"
   $CFN_BOOTSTRAP_VIRTUALENV_PATH/cfn-signal --exit-code=1 --reason="${!reason}" "${!wait_condition_handle_presigned_url}" --region ${AWS::Region} --url ${CloudFormationUrl}
   exit 1
@@ -91,9 +91,9 @@ export PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/opt/aw
 # Load ParallelCluster environment variables
 [ -f /etc/parallelcluster/pcluster_cookbook_environment.sh ] && . /etc/parallelcluster/pcluster_cookbook_environment.sh
 
-cd /tmp
+cd ${PclusterTmpDir}
 $CFN_BOOTSTRAP_VIRTUALENV_PATH/cfn-init -s ${AWS::StackName} -v -c deployFiles -r HeadNodeLaunchTemplate --region ${AWS::Region} --url ${CloudFormationUrl}
-wait_condition_handle_presigned_url=$(cat /tmp/wait_condition_handle.txt)
+wait_condition_handle_presigned_url=$(cat ${PclusterTmpDir}/wait_condition_handle.txt)
 
 custom_cookbook=${CustomChefCookbook}
 export _region=${AWS::Region}

cli/src/pcluster/resources/login_node/user_data.sh

@@ -48,17 +48,17 @@ datasource_list: [ Ec2, None ]
 output:
   all: "| tee -a /var/log/cloud-init-output.log | logger -t user-data -s 2>/dev/ttyS0"
 write_files:
-  - path: /tmp/dna.json
+  - path: ${PclusterTmpDir}/dna.json
     permissions: '0644'
     owner: root:root
     content: |
       ${DnaJson}
-  - path: /tmp/extra.json
+  - path: ${PclusterTmpDir}/extra.json
     permissions: '0644'
     owner: root:root
     content: |
       ${ExtraJson}
-  - path: /tmp/bootstrap.sh
+  - path: ${PclusterTmpDir}/bootstrap.sh
     permissions: '0744'
     owner: root:root
     content: |
@@ -136,11 +136,11 @@ write_files:
         done
         vendor_cookbook
       fi
-      cd /tmp
+      cd ${PclusterTmpDir}
       mkdir -p /etc/chef/ohai/hints
       touch /etc/chef/ohai/hints/ec2.json
 
-      jq -s ".[0] * .[1]" /tmp/dna.json /tmp/extra.json > /etc/chef/dna.json || ( echo "jq not installed"; cp /tmp/dna.json /etc/chef/dna.json )
+      jq -s ".[0] * .[1]" ${PclusterTmpDir}/dna.json ${PclusterTmpDir}/extra.json > /etc/chef/dna.json || ( echo "jq not installed"; cp ${PclusterTmpDir}/dna.json /etc/chef/dna.json )
       {
         CINC_CMD="cinc-client --local-mode --config /etc/chef/client.rb --log_level info --logfile /var/log/chef-client.log --force-formatter --no-color --chef-zero-port 8889 --json-attributes /etc/chef/dna.json"
         pushd /etc/chef &&
@@ -175,9 +175,9 @@ function error_exit
 }
 
 if [ "${Timeout}" == "NONE" ]; then
-  /tmp/bootstrap.sh
+  ${PclusterTmpDir}/bootstrap.sh
 else
-  timeout ${Timeout} /tmp/bootstrap.sh || error_exit
+  timeout ${Timeout} ${PclusterTmpDir}/bootstrap.sh || error_exit
 fi
 
 # Notify the AutoScalingGroup about the successful bootstrap

cli/src/pcluster/templates/cdk_builder_utils.py

@@ -43,6 +43,7 @@
     PCLUSTER_CLUSTER_NAME_TAG,
     PCLUSTER_DYNAMODB_PREFIX,
     PCLUSTER_NODE_TYPE_TAG,
+    PCLUSTER_TMP_DIR,
     ULTRASERVER_INSTANCE_PREFIX_LIST,
 )
 from pcluster.launch_template_utils import _LaunchTemplateBuilder
@@ -97,6 +98,7 @@ def get_common_user_data_env(node: Union[HeadNode, SlurmQueue, LoginNodesPool],
         "ParallelClusterVersion": COOKBOOK_PACKAGES_VERSIONS["parallelcluster"],
         "CookbookVersion": COOKBOOK_PACKAGES_VERSIONS["cookbook"],
         "ChefVersion": COOKBOOK_PACKAGES_VERSIONS["chef"],
+        "PclusterTmpDir": PCLUSTER_TMP_DIR,
     }
 
 

cli/src/pcluster/templates/cluster_stack.py

@@ -76,6 +76,7 @@
     P6E_GB200,
     PCLUSTER_DYNAMODB_PREFIX,
     PCLUSTER_S3_ARTIFACTS_DICT,
+    PCLUSTER_TMP_DIR,
     SLURM_PORTS_RANGE,
 )
 from pcluster.models.s3_bucket import S3Bucket
@@ -1430,29 +1431,20 @@ def _add_head_node(self):
             },
             "deployConfigFiles": {
                 "files": {
-                    # A nosec comment is appended to the following line in order to disable the B108 check.
-                    # The file is needed by the product
-                    # [B108:hardcoded_tmp_directory] Probable insecure usage of temp file/directory.
-                    "/tmp/dna.json": {  # nosec B108
+                    f"{PCLUSTER_TMP_DIR}/dna.json": {
                         "content": dna_json,
                         "mode": "000644",
                         "owner": "root",
                         "group": "root",
                         "encoding": "plain",
                     },
-                    # A nosec comment is appended to the following line in order to disable the B108 check.
-                    # The file is needed by the product
-                    # [B108:hardcoded_tmp_directory] Probable insecure usage of temp file/directory.
-                    "/tmp/extra.json": {  # nosec B108
+                    f"{PCLUSTER_TMP_DIR}/extra.json": {
                         "mode": "000644",
                         "owner": "root",
                         "group": "root",
                         "content": self.config.extra_chef_attributes,
                     },
-                    # A nosec comment is appended to the following line in order to disable the B108 check.
-                    # The file is needed by the product
-                    # [B108:hardcoded_tmp_directory] Probable insecure usage of temp file/directory.
-                    "/tmp/wait_condition_handle.txt": {  # nosec B108
+                    f"{PCLUSTER_TMP_DIR}/wait_condition_handle.txt": {
                         "mode": "000600",
                         "owner": "root",
                         "group": "root",
@@ -1464,8 +1456,9 @@ def _add_head_node(self):
                     "touch": {"command": "touch /etc/chef/ohai/hints/ec2.json"},
                     "jq": {
                         "command": (
-                            'jq -s ".[0] * .[1]" /tmp/dna.json /tmp/extra.json > /etc/chef/dna.json '
-                            '|| ( echo "jq not installed"; cp /tmp/dna.json /etc/chef/dna.json )'
+                            f'jq -s ".[0] * .[1]" {PCLUSTER_TMP_DIR}/dna.json '
+                            f"{PCLUSTER_TMP_DIR}/extra.json > /etc/chef/dna.json "
+                            f'|| ( echo "jq not installed"; cp {PCLUSTER_TMP_DIR}/dna.json /etc/chef/dna.json )'
                         )
                     },
                 },

cli/tests/pcluster/templates/test_cluster_stack.py

@@ -1063,7 +1063,11 @@ def test_head_node_dna_json(mocker, test_datadir, config_file_name, expected_hea
     )
 
     generated_head_node_dna_json = json.loads(
-        _get_cfn_init_file_content(template=generated_template, resource="HeadNodeLaunchTemplate", file="/tmp/dna.json")
+        _get_cfn_init_file_content(
+            template=generated_template,
+            resource="HeadNodeLaunchTemplate",
+            file="/opt/parallelcluster/tmp/dna.json",
+        )
     )
     slurm_specific_settings = {
         "ddb_table": "{'Ref': 'DynamoDBTable'}",

cli/tests/pcluster/templates/test_login_nodes_stack.py

@@ -65,9 +65,11 @@ def test_login_nodes_dna_json(
     expected_owner = "root:root"
     expected_permissions = "0644"
 
-    _assert_write_files_directive(login_node_user_data_template, "/tmp/dna.json", expected_owner, expected_permissions)
     _assert_write_files_directive(
-        login_node_user_data_template, "/tmp/extra.json", expected_owner, expected_permissions
+        login_node_user_data_template, "${PclusterTmpDir}/dna.json", expected_owner, expected_permissions
+    )
+    _assert_write_files_directive(
+        login_node_user_data_template, "${PclusterTmpDir}/extra.json", expected_owner, expected_permissions
     )
 
 

cli/tests/pcluster/templates/test_shared_storage.py

@@ -222,7 +222,7 @@ def test_unmanaged_shared_storage_fsx(mocker, test_datadir, config_file_name):
         .get("AWS::CloudFormation::Init")
         .get("deployConfigFiles")
         .get("files")
-        .get("/tmp/dna.json")
+        .get("/opt/parallelcluster/tmp/dna.json")
         .get("content")
         .get("Fn::Join")[1][4]
     )