diff --git a/CHANGELOG.md b/CHANGELOG.md index 34ddec5bbb..3528b4aa1c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ CHANGELOG **ENHANCEMENTS** - Improve cluster update resiliency on login nodes by reusing the head-node-driven orchestration already in place on compute nodes, removing the dependency on cfn-hup and cfn-init. +- Move all ParallelCluster-managed bootstrap files off `/tmp` into a dedicated `/opt/parallelcluster/tmp` + directory. Therefore, Image builds, cluster creations and updates work on custom AMIs that mount `/tmp` with `noexec`. **CHANGES** - The validator `ClusterNameValidator` now enforces cluster names to be limited to 40 characters when using `ExternalSlurmdbd`, diff --git a/cli/src/pcluster/constants.py b/cli/src/pcluster/constants.py index e4a0d774ba..39e95851be 100644 --- a/cli/src/pcluster/constants.py +++ b/cli/src/pcluster/constants.py @@ -245,6 +245,12 @@ DEFAULT_EPHEMERAL_DIR = "/scratch" +# Dedicated ParallelCluster-managed directory used to stage bootstrap files (dna.json, extra.json, +# wait_condition_handle.txt, bootstrap.sh) instead of /tmp, so bootstrap works when /tmp is mounted +# with noexec. Defined once here and referenced by cluster_stack.py (cfn-init) and the user_data +# templates (via the PclusterTmpDir Fn::Sub variable). +PCLUSTER_TMP_DIR = "/opt/parallelcluster/tmp" + LAMBDA_VPC_ACCESS_MANAGED_POLICY = "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole" IAM_NAME_PREFIX_LENGTH_LIMIT = 30 diff --git a/cli/src/pcluster/resources/compute_node/user_data.sh b/cli/src/pcluster/resources/compute_node/user_data.sh index a3ec97c5c5..79601a6ce8 100644 --- a/cli/src/pcluster/resources/compute_node/user_data.sh +++ b/cli/src/pcluster/resources/compute_node/user_data.sh @@ -51,17 +51,17 @@ datasource_list: [ Ec2, None ] output: all: "| tee -a /var/log/cloud-init-output.log | logger -t user-data -s 2>/dev/ttyS0" write_files: - - path: /tmp/dna.json + - path: ${PclusterTmpDir}/dna.json permissions: '0644' owner: root:root content: | ${DnaJson} - - path: /tmp/extra.json + - path: ${PclusterTmpDir}/extra.json permissions: '0644' owner: root:root content: | ${ExtraJson} - - path: /tmp/bootstrap.sh + - path: ${PclusterTmpDir}/bootstrap.sh permissions: '0744' owner: root:root content: | @@ -145,12 +145,12 @@ write_files: done vendor_cookbook fi - cd /tmp + cd ${PclusterTmpDir} mkdir -p /etc/chef/ohai/hints touch /etc/chef/ohai/hints/ec2.json start=$(date +%s) - jq -s ".[0] * .[1]" /tmp/dna.json /tmp/extra.json > /etc/chef/dna.json || ( echo "jq not installed"; cp /tmp/dna.json /etc/chef/dna.json ) + jq -s ".[0] * .[1]" ${PclusterTmpDir}/dna.json ${PclusterTmpDir}/extra.json > /etc/chef/dna.json || ( echo "jq not installed"; cp ${PclusterTmpDir}/dna.json /etc/chef/dna.json ) { CINC_CMD="cinc-client --local-mode --config /etc/chef/client.rb --log_level info --logfile /var/log/chef-client.log --force-formatter --no-color --chef-zero-port 8889 --json-attributes /etc/chef/dna.json --override-runlist" FR_CMD="/opt/parallelcluster/scripts/fetch_and_run" @@ -192,8 +192,8 @@ function error_exit } if [ "${Timeout}" == "NONE" ]; then - /tmp/bootstrap.sh + ${PclusterTmpDir}/bootstrap.sh else - timeout ${Timeout} /tmp/bootstrap.sh || error_exit + timeout ${Timeout} ${PclusterTmpDir}/bootstrap.sh || error_exit fi --==BOUNDARY== diff --git a/cli/src/pcluster/resources/head_node/user_data.sh b/cli/src/pcluster/resources/head_node/user_data.sh index 9deaa8be25..cd0ce263c0 100644 --- a/cli/src/pcluster/resources/head_node/user_data.sh +++ b/cli/src/pcluster/resources/head_node/user_data.sh @@ -72,7 +72,7 @@ function error_exit sleep 10 # trim the error message because there is a size limit of 4096 bytes for cfn-signal # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/cloudformation-limits.html - cutoff=$(expr 4096 - $(stat --printf="%s" /tmp/wait_condition_handle.txt)) + cutoff=$(expr 4096 - $(stat --printf="%s" ${PclusterTmpDir}/wait_condition_handle.txt)) reason=$(head --bytes=${!cutoff} /var/log/parallelcluster/bootstrap_error_msg 2>/dev/null) || reason="$1" $CFN_BOOTSTRAP_VIRTUALENV_PATH/cfn-signal --exit-code=1 --reason="${!reason}" "${!wait_condition_handle_presigned_url}" --region ${AWS::Region} --url ${CloudFormationUrl} exit 1 @@ -91,9 +91,9 @@ export PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/opt/aw # Load ParallelCluster environment variables [ -f /etc/parallelcluster/pcluster_cookbook_environment.sh ] && . /etc/parallelcluster/pcluster_cookbook_environment.sh -cd /tmp +cd ${PclusterTmpDir} $CFN_BOOTSTRAP_VIRTUALENV_PATH/cfn-init -s ${AWS::StackName} -v -c deployFiles -r HeadNodeLaunchTemplate --region ${AWS::Region} --url ${CloudFormationUrl} -wait_condition_handle_presigned_url=$(cat /tmp/wait_condition_handle.txt) +wait_condition_handle_presigned_url=$(cat ${PclusterTmpDir}/wait_condition_handle.txt) custom_cookbook=${CustomChefCookbook} export _region=${AWS::Region} diff --git a/cli/src/pcluster/resources/login_node/user_data.sh b/cli/src/pcluster/resources/login_node/user_data.sh index d02c8f0af2..417a365aaa 100644 --- a/cli/src/pcluster/resources/login_node/user_data.sh +++ b/cli/src/pcluster/resources/login_node/user_data.sh @@ -48,17 +48,17 @@ datasource_list: [ Ec2, None ] output: all: "| tee -a /var/log/cloud-init-output.log | logger -t user-data -s 2>/dev/ttyS0" write_files: - - path: /tmp/dna.json + - path: ${PclusterTmpDir}/dna.json permissions: '0644' owner: root:root content: | ${DnaJson} - - path: /tmp/extra.json + - path: ${PclusterTmpDir}/extra.json permissions: '0644' owner: root:root content: | ${ExtraJson} - - path: /tmp/bootstrap.sh + - path: ${PclusterTmpDir}/bootstrap.sh permissions: '0744' owner: root:root content: | @@ -136,11 +136,11 @@ write_files: done vendor_cookbook fi - cd /tmp + cd ${PclusterTmpDir} mkdir -p /etc/chef/ohai/hints touch /etc/chef/ohai/hints/ec2.json - jq -s ".[0] * .[1]" /tmp/dna.json /tmp/extra.json > /etc/chef/dna.json || ( echo "jq not installed"; cp /tmp/dna.json /etc/chef/dna.json ) + jq -s ".[0] * .[1]" ${PclusterTmpDir}/dna.json ${PclusterTmpDir}/extra.json > /etc/chef/dna.json || ( echo "jq not installed"; cp ${PclusterTmpDir}/dna.json /etc/chef/dna.json ) { CINC_CMD="cinc-client --local-mode --config /etc/chef/client.rb --log_level info --logfile /var/log/chef-client.log --force-formatter --no-color --chef-zero-port 8889 --json-attributes /etc/chef/dna.json" pushd /etc/chef && @@ -175,9 +175,9 @@ function error_exit } if [ "${Timeout}" == "NONE" ]; then - /tmp/bootstrap.sh + ${PclusterTmpDir}/bootstrap.sh else - timeout ${Timeout} /tmp/bootstrap.sh || error_exit + timeout ${Timeout} ${PclusterTmpDir}/bootstrap.sh || error_exit fi # Notify the AutoScalingGroup about the successful bootstrap diff --git a/cli/src/pcluster/templates/cdk_builder_utils.py b/cli/src/pcluster/templates/cdk_builder_utils.py index b5707dab83..25b64dd8a6 100644 --- a/cli/src/pcluster/templates/cdk_builder_utils.py +++ b/cli/src/pcluster/templates/cdk_builder_utils.py @@ -43,6 +43,7 @@ PCLUSTER_CLUSTER_NAME_TAG, PCLUSTER_DYNAMODB_PREFIX, PCLUSTER_NODE_TYPE_TAG, + PCLUSTER_TMP_DIR, ULTRASERVER_INSTANCE_PREFIX_LIST, ) from pcluster.launch_template_utils import _LaunchTemplateBuilder @@ -97,6 +98,7 @@ def get_common_user_data_env(node: Union[HeadNode, SlurmQueue, LoginNodesPool], "ParallelClusterVersion": COOKBOOK_PACKAGES_VERSIONS["parallelcluster"], "CookbookVersion": COOKBOOK_PACKAGES_VERSIONS["cookbook"], "ChefVersion": COOKBOOK_PACKAGES_VERSIONS["chef"], + "PclusterTmpDir": PCLUSTER_TMP_DIR, } diff --git a/cli/src/pcluster/templates/cluster_stack.py b/cli/src/pcluster/templates/cluster_stack.py index d1d4337f38..f3b87f434b 100644 --- a/cli/src/pcluster/templates/cluster_stack.py +++ b/cli/src/pcluster/templates/cluster_stack.py @@ -76,6 +76,7 @@ P6E_GB200, PCLUSTER_DYNAMODB_PREFIX, PCLUSTER_S3_ARTIFACTS_DICT, + PCLUSTER_TMP_DIR, SLURM_PORTS_RANGE, ) from pcluster.models.s3_bucket import S3Bucket @@ -1430,29 +1431,20 @@ def _add_head_node(self): }, "deployConfigFiles": { "files": { - # A nosec comment is appended to the following line in order to disable the B108 check. - # The file is needed by the product - # [B108:hardcoded_tmp_directory] Probable insecure usage of temp file/directory. - "/tmp/dna.json": { # nosec B108 + f"{PCLUSTER_TMP_DIR}/dna.json": { "content": dna_json, "mode": "000644", "owner": "root", "group": "root", "encoding": "plain", }, - # A nosec comment is appended to the following line in order to disable the B108 check. - # The file is needed by the product - # [B108:hardcoded_tmp_directory] Probable insecure usage of temp file/directory. - "/tmp/extra.json": { # nosec B108 + f"{PCLUSTER_TMP_DIR}/extra.json": { "mode": "000644", "owner": "root", "group": "root", "content": self.config.extra_chef_attributes, }, - # A nosec comment is appended to the following line in order to disable the B108 check. - # The file is needed by the product - # [B108:hardcoded_tmp_directory] Probable insecure usage of temp file/directory. - "/tmp/wait_condition_handle.txt": { # nosec B108 + f"{PCLUSTER_TMP_DIR}/wait_condition_handle.txt": { "mode": "000600", "owner": "root", "group": "root", @@ -1464,8 +1456,9 @@ def _add_head_node(self): "touch": {"command": "touch /etc/chef/ohai/hints/ec2.json"}, "jq": { "command": ( - 'jq -s ".[0] * .[1]" /tmp/dna.json /tmp/extra.json > /etc/chef/dna.json ' - '|| ( echo "jq not installed"; cp /tmp/dna.json /etc/chef/dna.json )' + f'jq -s ".[0] * .[1]" {PCLUSTER_TMP_DIR}/dna.json ' + f"{PCLUSTER_TMP_DIR}/extra.json > /etc/chef/dna.json " + f'|| ( echo "jq not installed"; cp {PCLUSTER_TMP_DIR}/dna.json /etc/chef/dna.json )' ) }, }, diff --git a/cli/tests/pcluster/templates/test_cluster_stack.py b/cli/tests/pcluster/templates/test_cluster_stack.py index 1c7e5bf7fb..90c05e4da2 100644 --- a/cli/tests/pcluster/templates/test_cluster_stack.py +++ b/cli/tests/pcluster/templates/test_cluster_stack.py @@ -1063,7 +1063,11 @@ def test_head_node_dna_json(mocker, test_datadir, config_file_name, expected_hea ) generated_head_node_dna_json = json.loads( - _get_cfn_init_file_content(template=generated_template, resource="HeadNodeLaunchTemplate", file="/tmp/dna.json") + _get_cfn_init_file_content( + template=generated_template, + resource="HeadNodeLaunchTemplate", + file="/opt/parallelcluster/tmp/dna.json", + ) ) slurm_specific_settings = { "ddb_table": "{'Ref': 'DynamoDBTable'}", diff --git a/cli/tests/pcluster/templates/test_login_nodes_stack.py b/cli/tests/pcluster/templates/test_login_nodes_stack.py index bb831e1c2a..51af5c3f8f 100644 --- a/cli/tests/pcluster/templates/test_login_nodes_stack.py +++ b/cli/tests/pcluster/templates/test_login_nodes_stack.py @@ -65,9 +65,11 @@ def test_login_nodes_dna_json( expected_owner = "root:root" expected_permissions = "0644" - _assert_write_files_directive(login_node_user_data_template, "/tmp/dna.json", expected_owner, expected_permissions) _assert_write_files_directive( - login_node_user_data_template, "/tmp/extra.json", expected_owner, expected_permissions + login_node_user_data_template, "${PclusterTmpDir}/dna.json", expected_owner, expected_permissions + ) + _assert_write_files_directive( + login_node_user_data_template, "${PclusterTmpDir}/extra.json", expected_owner, expected_permissions ) diff --git a/cli/tests/pcluster/templates/test_shared_storage.py b/cli/tests/pcluster/templates/test_shared_storage.py index 5edfcd1ced..d2f5500593 100644 --- a/cli/tests/pcluster/templates/test_shared_storage.py +++ b/cli/tests/pcluster/templates/test_shared_storage.py @@ -222,7 +222,7 @@ def test_unmanaged_shared_storage_fsx(mocker, test_datadir, config_file_name): .get("AWS::CloudFormation::Init") .get("deployConfigFiles") .get("files") - .get("/tmp/dna.json") + .get("/opt/parallelcluster/tmp/dna.json") .get("content") .get("Fn::Join")[1][4] )