From 4dcb495c369f9d8f2b2b51adb8ad40e6c43eff3f Mon Sep 17 00:00:00 2001 From: Roger Zhang Date: Fri, 31 Jul 2026 15:19:48 -0700 Subject: [PATCH 1/5] tests: resolve sam init menu positions dynamically in schemas tests test_init_interactive_with_event_bridge_app_aws_schemas_python has been failing (see run 30657529336). It answered the runtime prompt with "8" and its comment said "8: Python 3.9", but the prompt offers 15 runtimes for the Infrastructure event management use case and position 8 is now java8.al2023. python3.9 has drifted to position 10. The test therefore selected a Java runtime and fed Python-flow answers into a Java prompt sequence: every later answer landed on the wrong question, and it finally asserted on hello_world_function/schema, a Python layout a Java project never produces. The positions moved when java8.al2023/java11.al2023/java17.al2023 were added (#9125). Nothing about the runtime list is stable, so hardcoding an index means any future runtime addition silently repoints these tests at a different runtime -- a failure that looks like a product bug rather than a stale fixture. Adds _get_runtime_position and _get_use_case_position, which look positions up from the same source the prompt uses (InitTemplates(). get_preprocessed_manifest plus get_sorted_runtimes), mirroring the existing _get_registry_position helper. All 8 tests in the file now resolve the template/use-case/runtime answers dynamically; the 4 Java tests and the go test were relying on the same brittle indexing and only passed by luck. Resolved positions are unchanged for Java (4) and go (1), and corrected from 8 to 10 for python3.9. Stale position comments updated to name the runtime rather than a number. Testing: make pr passes (9347 passed, 25 skipped, coverage 94.09%). Verified the helpers resolve to use case 8, java17.al2023 4, python3.9 10, go (provided.al2) 1, and rendered each test's user_input to confirm the answer sequences. --- .../schemas/test_init_with_schemas_command.py | 104 ++++++++++++------ 1 file changed, 69 insertions(+), 35 deletions(-) diff --git a/tests/integration/init/schemas/test_init_with_schemas_command.py b/tests/integration/init/schemas/test_init_with_schemas_command.py index 5ef11f028f9..9f3ff69b01f 100644 --- a/tests/integration/init/schemas/test_init_with_schemas_command.py +++ b/tests/integration/init/schemas/test_init_with_schemas_command.py @@ -8,12 +8,16 @@ from click.testing import CliRunner from samcli.commands.init import cli as init_cmd +from samcli.commands.init.init_templates import InitTemplates +from samcli.commands.init.interactive_init_flow import get_sorted_runtimes from tests.integration.init.schemas.schemas_test_data_setup import SchemaTestDataSetup from tests.testing_utils import RUNNING_ON_CI, RUNNING_TEST_FOR_MASTER_ON_CI, RUN_BY_CANARY # Schemas tests require credentials. This is to skip running the test where credentials are not available. SKIP_SCHEMA_TESTS = RUNNING_ON_CI and RUNNING_TEST_FOR_MASTER_ON_CI and not RUN_BY_CANARY +EVENT_BRIDGE_USE_CASE = "Infrastructure event management" + def _get_registry_position(registry_name): """Query EventBridge Schema registries and return the 1-based menu position for the given registry name. @@ -34,6 +38,36 @@ def _get_registry_position(registry_name): raise ValueError(f"Registry '{registry_name}' not found. Available: {registries}") +def _get_runtime_position(runtime_name): + """Return the 1-based menu position of a runtime in the `sam init` runtime prompt. + + The prompt lists the runtimes the app-templates manifest offers for + EVENT_BRIDGE_USE_CASE, ordered by `get_sorted_runtimes`. That ordering shifts + whenever a runtime is added or removed, so hardcoding a position silently + starts selecting a different runtime -- every answer after it then lands on + the wrong question. Resolve it the same way `_get_registry_position` does. + """ + runtime_options = InitTemplates().get_preprocessed_manifest(None, None, None, None)[EVENT_BRIDGE_USE_CASE] + runtimes = get_sorted_runtimes(list(runtime_options.keys())) + for i, name in enumerate(runtimes, 1): + if name == runtime_name: + return i + raise ValueError(f"Runtime '{runtime_name}' not found. Available: {runtimes}") + + +def _get_use_case_position(use_case_name): + """Return the 1-based menu position of a use case in the `sam init` template prompt. + + Same rationale as `_get_runtime_position`: the manifest grows over time, so + the position of a use case is not stable. + """ + use_cases = list(InitTemplates().get_preprocessed_manifest(None, None, None, None).keys()) + for i, name in enumerate(use_cases, 1): + if name == use_case_name: + return i + raise ValueError(f"Use case '{use_case_name}' not found. Available: {use_cases}") + + @skipIf(SKIP_SCHEMA_TESTS, "Skip schema test") @pytest.mark.xdist_group(name="sam_init") class TestBasicInitWithEventBridgeCommand(SchemaTestDataSetup): @@ -41,8 +75,8 @@ class TestBasicInitWithEventBridgeCommand(SchemaTestDataSetup): def test_init_interactive_with_event_bridge_app_aws_registry(self): # WHEN the user follows interactive init prompts # 1: AWS Quick Start Templates - # 8: Infrastructure event management - Use case - # 4: Java Runtime + # {use_case_pos}: Infrastructure event management - Use case (dynamic position) + # {java_runtime_pos}: java17.al2023 (dynamic position) # 2: Maven # 2: select event-bridge app from scratch # N: disable adding xray tracing @@ -56,8 +90,8 @@ def test_init_interactive_with_event_bridge_app_aws_registry(self): aws_registry_pos = _get_registry_position("aws.events") user_input = f""" 1 -8 -4 +{_get_use_case_position(EVENT_BRIDGE_USE_CASE)} +{_get_runtime_position("java17.al2023")} 2 2 N @@ -85,8 +119,8 @@ def test_init_interactive_with_event_bridge_app_aws_registry(self): def test_init_interactive_with_event_bridge_app_partner_registry(self): # WHEN the user follows interactive init prompts # 1: AWS Quick Start Templates - # 8: Infrastructure event management - Use case - # 4: Java Runtime + # {use_case_pos}: Infrastructure event management - Use case (dynamic position) + # {java_runtime_pos}: java17.al2023 (dynamic position) # 2: Maven # 2: select event-bridge app from scratch # N: disable adding xray tracing @@ -99,8 +133,8 @@ def test_init_interactive_with_event_bridge_app_partner_registry(self): partner_registry_pos = _get_registry_position("partner-registry") user_input = f""" 1 -8 -4 +{_get_use_case_position(EVENT_BRIDGE_USE_CASE)} +{_get_runtime_position("java17.al2023")} 2 2 N @@ -139,8 +173,8 @@ def test_init_interactive_with_event_bridge_app_partner_registry(self): def test_init_interactive_with_event_bridge_app_pagination(self): # WHEN the user follows interactive init prompts # 1: AWS Quick Start Templates - # 8: Infrastructure event management - Use case - # 4: Java Runtime + # {use_case_pos}: Infrastructure event management - Use case (dynamic position) + # {java_runtime_pos}: java17.al2023 (dynamic position) # 2: Maven # 2: select event-bridge app from scratch # N: disable adding xray tracing @@ -155,8 +189,8 @@ def test_init_interactive_with_event_bridge_app_pagination(self): pagination_registry_pos = _get_registry_position("test-pagination") user_input = f""" 1 -8 -4 +{_get_use_case_position(EVENT_BRIDGE_USE_CASE)} +{_get_runtime_position("java17.al2023")} 2 2 N @@ -186,8 +220,8 @@ def test_init_interactive_with_event_bridge_app_pagination(self): def test_init_interactive_with_event_bridge_app_customer_registry(self): # WHEN the user follows interactive init prompts # 1: AWS Quick Start Templates - # 8: Infrastructure event management - Use case - # 4: Java Runtime + # {use_case_pos}: Infrastructure event management - Use case (dynamic position) + # {java_runtime_pos}: java17.al2023 (dynamic position) # 2: Maven # 2: select event-bridge app from scratch # N: disable adding xray tracing @@ -200,8 +234,8 @@ def test_init_interactive_with_event_bridge_app_customer_registry(self): other_schema_pos = _get_registry_position("other-schema") user_input = f""" 1 -8 -4 +{_get_use_case_position(EVENT_BRIDGE_USE_CASE)} +{_get_runtime_position("java17.al2023")} 2 2 N @@ -240,8 +274,8 @@ def test_init_interactive_with_event_bridge_app_customer_registry(self): def test_init_interactive_with_event_bridge_app_aws_schemas_python(self): # WHEN the user follows interactive init prompts # 1: AWS Quick Start Templates - # 8: Infrastructure event management - Use case - # 8: Python 3.9 + # {use_case_pos}: Infrastructure event management - Use case (dynamic position) + # {python_runtime_pos}: python3.9 (dynamic position) # 2: select event-bridge app from scratch # N: disable adding xray tracing # N: disable cloudwatch insights @@ -254,8 +288,8 @@ def test_init_interactive_with_event_bridge_app_aws_schemas_python(self): aws_registry_pos = _get_registry_position("aws.events") user_input = f""" 1 -8 -8 +{_get_use_case_position(EVENT_BRIDGE_USE_CASE)} +{_get_runtime_position("python3.9")} 2 N N @@ -280,8 +314,8 @@ def test_init_interactive_with_event_bridge_app_aws_schemas_python(self): def test_init_interactive_with_event_bridge_app_aws_schemas_go(self): # WHEN the user follows interactive init prompts # 1: AWS Quick Start Templates - # 8: Infrastructure event management - Use case - # 1: Go 1.x + # {use_case_pos}: Infrastructure event management - Use case (dynamic position) + # {go_runtime_pos}: go (provided.al2) (dynamic position) # 2: select event-bridge app from scratch # N: disable adding xray tracing # N: disable cloudwatch insights @@ -291,10 +325,10 @@ def test_init_interactive_with_event_bridge_app_aws_schemas_go(self): # 4: select aws.events as registries # 1: select aws schema - user_input = """ -1 -8 + user_input = f""" 1 +{_get_use_case_position(EVENT_BRIDGE_USE_CASE)} +{_get_runtime_position("go (provided.al2)")} 2 N N @@ -319,8 +353,8 @@ def test_init_interactive_with_event_bridge_app_non_default_profile_selection(se self._init_custom_config("mynewprofile", "us-west-2") # WHEN the user follows interactive init prompts # 1: AWS Quick Start Templates - # 8: Infrastructure event management - Use case - # 8: Python 3.9 + # {use_case_pos}: Infrastructure event management - Use case (dynamic position) + # {python_runtime_pos}: python3.9 (dynamic position) # 2: select event-bridge app from scratch # N: disable adding xray tracing # N: disable cloudwatch insights @@ -332,10 +366,10 @@ def test_init_interactive_with_event_bridge_app_non_default_profile_selection(se # 1: select aws.events as registries # 1: select aws schema - user_input = """ + user_input = f""" 1 -8 -8 +{_get_use_case_position(EVENT_BRIDGE_USE_CASE)} +{_get_runtime_position("python3.9")} 2 N N @@ -363,8 +397,8 @@ def test_init_interactive_with_event_bridge_app_non_supported_schemas_region(sel self._init_custom_config("default", "cn-north-1") # WHEN the user follows interactive init prompts # 1: AWS Quick Start Templates - # 8: Infrastructure event management - Use case - # 7: Python 3.9 + # {use_case_pos}: Infrastructure event management - Use case (dynamic position) + # {python_runtime_pos}: python3.9 (dynamic position) # 2: select event-bridge app from scratch # N: disable adding xray tracing # N: disable cloudwatch insights @@ -374,10 +408,10 @@ def test_init_interactive_with_event_bridge_app_non_supported_schemas_region(sel # 1: select aws.events as registries # 1: select aws schema - user_input = """ + user_input = f""" 1 -8 -8 +{_get_use_case_position(EVENT_BRIDGE_USE_CASE)} +{_get_runtime_position("python3.9")} 2 N N From b88580c8a12a087ca59782973f5ce05c988fbff7 Mon Sep 17 00:00:00 2001 From: Roger Zhang Date: Fri, 14 Aug 2026 14:54:41 -0700 Subject: [PATCH 2/5] tests: address review feedback on schemas init tests Review feedback on #9151. 1. Name the runtimes as constants (valerena) JAVA_RUNTIME_FOR_INIT, PYTHON_RUNTIME_FOR_INIT, GO_RUNTIME_FOR_INIT. None of these tests is about a specific runtime version -- they exercise the schemas flows (registry choice, pagination, profile selection) -- so the version is incidental and was repeated 4/3/1 times. python3.9 is already deprecated, so a one-line change rather than a hunt through the file is not hypothetical. The per-test comments now name the constant instead of the literal version, so changing a constant cannot leave a stale comment behind -- the same drift this PR exists to fix. Scoped to this file on purpose. The runtimes in test_init_command.py are passed as explicit `--runtime` CLI arguments, which are position-independent and fail loudly if removed; each of those tests deliberately pairs a runtime with a dependency manager (java21+maven, nodejs18.x+npm), so hoisting them into shared constants would hide what those tests are actually exercising. 2. Fetch the manifest once per process (review bot) Both helpers called InitTemplates().get_preprocessed_manifest() independently, and InitTemplates._get_manifest does requests.get(MANIFEST_URL, timeout=10) on every call, falling back to cloning the templates repo. Every test resolves both a use case and a runtime, so that was two fetches per test. It was also a correctness trap: resolved independently, one call could succeed against MANIFEST_URL while the other fell back to the bundled local manifest, giving a use-case position and a runtime position from two different snapshots -- the exact answer misalignment these helpers exist to prevent. Now behind lru_cache(maxsize=1). 3. Registry answers (review bot) The go test said "# 4: select aws.events as registries" but answered 1, while every sibling resolves it with _get_registry_position("aws.events"). Fixed to resolve it. Left ..._non_default_profile_selection hardcoded, and said why: it deliberately drives a non-default profile and an explicit us-east-1, so the prompt lists that profile/region's registries, while _get_registry_position resolves against the default Session(). Converting it would look up the wrong account and region. Testing: 8 tests collect. Resolved against the live manifest -- use case 8, java17.al2023 -> 4, python3.9 -> 10, go (provided.al2) -> 1 (unchanged except python3.9, which is the bug this PR fixes). Cache confirmed at 1 miss / 3 hits for four resolutions, i.e. one fetch instead of four. A removed runtime still raises ValueError listing what is available. black clean. --- .../schemas/test_init_with_schemas_command.py | 80 ++++++++++++++----- 1 file changed, 59 insertions(+), 21 deletions(-) diff --git a/tests/integration/init/schemas/test_init_with_schemas_command.py b/tests/integration/init/schemas/test_init_with_schemas_command.py index 9f3ff69b01f..f9feaf4ff0f 100644 --- a/tests/integration/init/schemas/test_init_with_schemas_command.py +++ b/tests/integration/init/schemas/test_init_with_schemas_command.py @@ -1,6 +1,7 @@ import os import tempfile import pytest +from functools import lru_cache from pathlib import Path from unittest import skipIf @@ -18,6 +19,19 @@ EVENT_BRIDGE_USE_CASE = "Infrastructure event management" +# Runtimes these tests init against. None of these tests is about a specific runtime +# version -- they exercise the schemas flows (registry choice, pagination, profiles) -- +# so the version is incidental and repeated across tests. Naming it here means a runtime +# leaving the manifest is a one-line change instead of a hunt through the file. +# `python3.9` is already deprecated, so that is not hypothetical. +# +# These are the labels the *prompt* displays, which are not always the runtime id: +# the go entry shows as "go (provided.al2)". `_get_runtime_position` matches them +# against the manifest, and raises with the available list if one disappears. +JAVA_RUNTIME_FOR_INIT = "java17.al2023" +PYTHON_RUNTIME_FOR_INIT = "python3.9" +GO_RUNTIME_FOR_INIT = "go (provided.al2)" + def _get_registry_position(registry_name): """Query EventBridge Schema registries and return the 1-based menu position for the given registry name. @@ -38,6 +52,24 @@ def _get_registry_position(registry_name): raise ValueError(f"Registry '{registry_name}' not found. Available: {registries}") +@lru_cache(maxsize=1) +def _get_manifest(): + """Fetch the preprocessed app-templates manifest once per test process. + + `get_preprocessed_manifest` is not cheap or deterministic per call: + `InitTemplates._get_manifest` does `requests.get(MANIFEST_URL, timeout=10)` and, on + timeout / connection error / non-200, falls back to cloning the templates repo or + reading the bundled local manifest. Every test below resolves both a use case and a + runtime, so without caching that is two fetches per test. + + Caching also removes a correctness trap. Resolved independently, one call could + succeed against MANIFEST_URL while the other fell back to the local manifest -- + giving a use-case position and a runtime position from two different snapshots, + which is the same answer misalignment these helpers exist to prevent. + """ + return InitTemplates().get_preprocessed_manifest(None, None, None, None) + + def _get_runtime_position(runtime_name): """Return the 1-based menu position of a runtime in the `sam init` runtime prompt. @@ -47,7 +79,7 @@ def _get_runtime_position(runtime_name): starts selecting a different runtime -- every answer after it then lands on the wrong question. Resolve it the same way `_get_registry_position` does. """ - runtime_options = InitTemplates().get_preprocessed_manifest(None, None, None, None)[EVENT_BRIDGE_USE_CASE] + runtime_options = _get_manifest()[EVENT_BRIDGE_USE_CASE] runtimes = get_sorted_runtimes(list(runtime_options.keys())) for i, name in enumerate(runtimes, 1): if name == runtime_name: @@ -61,7 +93,7 @@ def _get_use_case_position(use_case_name): Same rationale as `_get_runtime_position`: the manifest grows over time, so the position of a use case is not stable. """ - use_cases = list(InitTemplates().get_preprocessed_manifest(None, None, None, None).keys()) + use_cases = list(_get_manifest().keys()) for i, name in enumerate(use_cases, 1): if name == use_case_name: return i @@ -76,7 +108,7 @@ def test_init_interactive_with_event_bridge_app_aws_registry(self): # WHEN the user follows interactive init prompts # 1: AWS Quick Start Templates # {use_case_pos}: Infrastructure event management - Use case (dynamic position) - # {java_runtime_pos}: java17.al2023 (dynamic position) + # {java_runtime_pos}: JAVA_RUNTIME_FOR_INIT (dynamic position) # 2: Maven # 2: select event-bridge app from scratch # N: disable adding xray tracing @@ -91,7 +123,7 @@ def test_init_interactive_with_event_bridge_app_aws_registry(self): user_input = f""" 1 {_get_use_case_position(EVENT_BRIDGE_USE_CASE)} -{_get_runtime_position("java17.al2023")} +{_get_runtime_position(JAVA_RUNTIME_FOR_INIT)} 2 2 N @@ -120,7 +152,7 @@ def test_init_interactive_with_event_bridge_app_partner_registry(self): # WHEN the user follows interactive init prompts # 1: AWS Quick Start Templates # {use_case_pos}: Infrastructure event management - Use case (dynamic position) - # {java_runtime_pos}: java17.al2023 (dynamic position) + # {java_runtime_pos}: JAVA_RUNTIME_FOR_INIT (dynamic position) # 2: Maven # 2: select event-bridge app from scratch # N: disable adding xray tracing @@ -134,7 +166,7 @@ def test_init_interactive_with_event_bridge_app_partner_registry(self): user_input = f""" 1 {_get_use_case_position(EVENT_BRIDGE_USE_CASE)} -{_get_runtime_position("java17.al2023")} +{_get_runtime_position(JAVA_RUNTIME_FOR_INIT)} 2 2 N @@ -174,7 +206,7 @@ def test_init_interactive_with_event_bridge_app_pagination(self): # WHEN the user follows interactive init prompts # 1: AWS Quick Start Templates # {use_case_pos}: Infrastructure event management - Use case (dynamic position) - # {java_runtime_pos}: java17.al2023 (dynamic position) + # {java_runtime_pos}: JAVA_RUNTIME_FOR_INIT (dynamic position) # 2: Maven # 2: select event-bridge app from scratch # N: disable adding xray tracing @@ -190,7 +222,7 @@ def test_init_interactive_with_event_bridge_app_pagination(self): user_input = f""" 1 {_get_use_case_position(EVENT_BRIDGE_USE_CASE)} -{_get_runtime_position("java17.al2023")} +{_get_runtime_position(JAVA_RUNTIME_FOR_INIT)} 2 2 N @@ -221,7 +253,7 @@ def test_init_interactive_with_event_bridge_app_customer_registry(self): # WHEN the user follows interactive init prompts # 1: AWS Quick Start Templates # {use_case_pos}: Infrastructure event management - Use case (dynamic position) - # {java_runtime_pos}: java17.al2023 (dynamic position) + # {java_runtime_pos}: JAVA_RUNTIME_FOR_INIT (dynamic position) # 2: Maven # 2: select event-bridge app from scratch # N: disable adding xray tracing @@ -235,7 +267,7 @@ def test_init_interactive_with_event_bridge_app_customer_registry(self): user_input = f""" 1 {_get_use_case_position(EVENT_BRIDGE_USE_CASE)} -{_get_runtime_position("java17.al2023")} +{_get_runtime_position(JAVA_RUNTIME_FOR_INIT)} 2 2 N @@ -275,7 +307,7 @@ def test_init_interactive_with_event_bridge_app_aws_schemas_python(self): # WHEN the user follows interactive init prompts # 1: AWS Quick Start Templates # {use_case_pos}: Infrastructure event management - Use case (dynamic position) - # {python_runtime_pos}: python3.9 (dynamic position) + # {python_runtime_pos}: PYTHON_RUNTIME_FOR_INIT (dynamic position) # 2: select event-bridge app from scratch # N: disable adding xray tracing # N: disable cloudwatch insights @@ -289,7 +321,7 @@ def test_init_interactive_with_event_bridge_app_aws_schemas_python(self): user_input = f""" 1 {_get_use_case_position(EVENT_BRIDGE_USE_CASE)} -{_get_runtime_position("python3.9")} +{_get_runtime_position(PYTHON_RUNTIME_FOR_INIT)} 2 N N @@ -315,27 +347,27 @@ def test_init_interactive_with_event_bridge_app_aws_schemas_go(self): # WHEN the user follows interactive init prompts # 1: AWS Quick Start Templates # {use_case_pos}: Infrastructure event management - Use case (dynamic position) - # {go_runtime_pos}: go (provided.al2) (dynamic position) + # {go_runtime_pos}: GO_RUNTIME_FOR_INIT (dynamic position) # 2: select event-bridge app from scratch # N: disable adding xray tracing # N: disable cloudwatch insights # N: disable structured logging # eb-app-go: response to name # Y: Use default aws configuration - # 4: select aws.events as registries + # {aws_registry_pos}: select aws.events as registries (dynamic position) # 1: select aws schema - + aws_registry_pos = _get_registry_position("aws.events") user_input = f""" 1 {_get_use_case_position(EVENT_BRIDGE_USE_CASE)} -{_get_runtime_position("go (provided.al2)")} +{_get_runtime_position(GO_RUNTIME_FOR_INIT)} 2 N N N eb-app-go Y -1 +{aws_registry_pos} 1 """ with tempfile.TemporaryDirectory() as temp: @@ -354,7 +386,7 @@ def test_init_interactive_with_event_bridge_app_non_default_profile_selection(se # WHEN the user follows interactive init prompts # 1: AWS Quick Start Templates # {use_case_pos}: Infrastructure event management - Use case (dynamic position) - # {python_runtime_pos}: python3.9 (dynamic position) + # {python_runtime_pos}: PYTHON_RUNTIME_FOR_INIT (dynamic position) # 2: select event-bridge app from scratch # N: disable adding xray tracing # N: disable cloudwatch insights @@ -365,11 +397,17 @@ def test_init_interactive_with_event_bridge_app_non_default_profile_selection(se # schemas aws region us-east-1 # 1: select aws.events as registries # 1: select aws schema + # + # The registry answer stays hardcoded here, unlike the sibling tests. This test + # deliberately drives a non-default profile and an explicit us-east-1, so the + # registries the prompt lists are those of *that* profile/region -- + # `_get_registry_position` resolves against the default `Session()`, so using it + # here would look up the wrong account and region. user_input = f""" 1 {_get_use_case_position(EVENT_BRIDGE_USE_CASE)} -{_get_runtime_position("python3.9")} +{_get_runtime_position(PYTHON_RUNTIME_FOR_INIT)} 2 N N @@ -398,7 +436,7 @@ def test_init_interactive_with_event_bridge_app_non_supported_schemas_region(sel # WHEN the user follows interactive init prompts # 1: AWS Quick Start Templates # {use_case_pos}: Infrastructure event management - Use case (dynamic position) - # {python_runtime_pos}: python3.9 (dynamic position) + # {python_runtime_pos}: PYTHON_RUNTIME_FOR_INIT (dynamic position) # 2: select event-bridge app from scratch # N: disable adding xray tracing # N: disable cloudwatch insights @@ -411,7 +449,7 @@ def test_init_interactive_with_event_bridge_app_non_supported_schemas_region(sel user_input = f""" 1 {_get_use_case_position(EVENT_BRIDGE_USE_CASE)} -{_get_runtime_position("python3.9")} +{_get_runtime_position(PYTHON_RUNTIME_FOR_INIT)} 2 N N From 7b4d0005e8ae02051b743ffc8b8741674e59a324 Mon Sep 17 00:00:00 2001 From: Roger Zhang Date: Fri, 14 Aug 2026 16:08:23 -0700 Subject: [PATCH 3/5] tests: drop a stray answer from the non-default-profile schemas test Review feedback on #9151. The input for test_init_interactive_with_event_bridge_app_non_default_profile_selection carried an extra `3` after the project name, giving 14 answers against 13 documented prompts. The sibling test_init_interactive_with_event_bridge_app_aws_schemas_python is identical up to that point -- same runtime, use case, starter template and N/N/N -- and answers `Y` immediately after the project name, so the next prompt there is the AWS-config confirm, not a numbered choice. The `3` was absorbed rather than acted on. Confirmed against click directly: Use default AWS profile? [y/N]: 3 Error: invalid input Use default AWS profile? [y/N]: N <- re-prompt consumes the next line so the confirm rejected `3`, re-prompted, and ate the `N` intended for it. The test passed only because click's re-prompt loop swallowed the extra line -- the same "answers are one position off but something still absorbs them" fragility this PR removes, and it made the input contradict its own comment block. Testing: input now has 13 answers matching the 13 documented prompts. 8 tests collect; black clean. --- tests/integration/init/schemas/test_init_with_schemas_command.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/init/schemas/test_init_with_schemas_command.py b/tests/integration/init/schemas/test_init_with_schemas_command.py index f9feaf4ff0f..1f68fef26d3 100644 --- a/tests/integration/init/schemas/test_init_with_schemas_command.py +++ b/tests/integration/init/schemas/test_init_with_schemas_command.py @@ -413,7 +413,6 @@ def test_init_interactive_with_event_bridge_app_non_default_profile_selection(se N N eb-app-python39 -3 N 2 us-east-1 From 9e5e432534266da35b784c270b8c36a83923dea8 Mon Sep 17 00:00:00 2001 From: Roger Zhang Date: Fri, 14 Aug 2026 16:59:04 -0700 Subject: [PATCH 4/5] tests: remove a phantom answer that shifted the schemas registry choice Review feedback on #9151. Two tests fed one more answer than the schemas flow has prompts, so the registry position this PR resolves was not landing on the registry prompt at all. After the project name the flow asks, in order: 1. get_aws_configuration_choice() -- one click.confirm on the Y path 2. _get_registry_cli_choice -- one click.prompt per page render 3. _get_schema_cli_choice -- one click.prompt per page render do_paginate_cli issues exactly one prompt per render, returning either an item choice or N/P to page. Nothing sits between the confirm and the registry list, so the comment line "# 1: select schema from cli_paginator" did not correspond to a prompt. The other six tests agree on the shape. Four use `Y -> {registry_pos} -> `, and the pagination test uses `Y -> {registry_pos} -> N -> P -> 2`, both putting the resolved registry position directly after the confirm. Only these two wedged a literal 1 in between, which meant the literal answered the registry prompt and the resolved position answered the *schema* prompt, with the last line never consumed. It is masked today because aws.events sorts first among the registries these tests use (aws.events, discovered-schemas, other-schema, partner-registry, test-pagination), so _get_registry_position returns 1 and both readings pick the same items. Registry sorting is plain ASCII list.sort(), so any registry sorting before aws.events -- an uppercase-named one, for instance -- shifts the position, and the hardcoded 1 would then select the wrong registry while the resolved position was read as a schema index. That is the same drift this PR exists to remove, and the assertions here only check that a schema directory exists, so it would not be caught. Testing: all 8 tests now have an answer count matching their documented prompt list. 8 tests collect; black clean. --- .../init/schemas/test_init_with_schemas_command.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/integration/init/schemas/test_init_with_schemas_command.py b/tests/integration/init/schemas/test_init_with_schemas_command.py index 1f68fef26d3..22f5186d065 100644 --- a/tests/integration/init/schemas/test_init_with_schemas_command.py +++ b/tests/integration/init/schemas/test_init_with_schemas_command.py @@ -116,7 +116,6 @@ def test_init_interactive_with_event_bridge_app_aws_registry(self): # N: disable structured logging # eb-app-maven: response to name # Y: Use default aws configuration - # 1: select schema from cli_paginator # {aws_registry_pos}: select aws.events as registries (dynamic position) # 9: select schema AWSAPICallViaCloudTrail aws_registry_pos = _get_registry_position("aws.events") @@ -131,7 +130,6 @@ def test_init_interactive_with_event_bridge_app_aws_registry(self): N eb-app-maven Y -1 {aws_registry_pos} 9 """ @@ -314,7 +312,6 @@ def test_init_interactive_with_event_bridge_app_aws_schemas_python(self): # N: disable structured logging # eb-app-python39: response to name # Y: Use default aws configuration - # 1: select schema from cli_paginator # {aws_registry_pos}: select aws.events as registries (dynamic position) # 1: select aws schema aws_registry_pos = _get_registry_position("aws.events") @@ -328,7 +325,6 @@ def test_init_interactive_with_event_bridge_app_aws_schemas_python(self): N eb-app-python39 Y -1 {aws_registry_pos} 1 """ From 0a28a29b41b746a4359d4afbab3482a9d7f7737f Mon Sep 17 00:00:00 2001 From: Roger Zhang Date: Mon, 17 Aug 2026 12:23:01 -0700 Subject: [PATCH 5/5] tests: trim comments in the schemas init tests Review feedback: keep comments to one line and docstrings short. The reasoning that was inline now lives in the commits and PR, which is where someone digging will look anyway. - constants: 9 comment lines -> 1 - _get_manifest, _get_runtime_position, _get_use_case_position: multi-line docstrings -> one line each - the non-default-profile registry note: 6 lines -> 1 Kept the per-test prompt answer keys. Those are not prose -- they map each input line to the prompt it answers, which is what makes an off-by-one visible, and the last two review rounds were about their accuracy. Left _get_registry_position's docstring alone; it predates this PR and rewriting it would widen the diff for no functional reason. Testing: 8 tests collect. Helpers still resolve against the live manifest -- use case 8, java17.al2023 -> 4, python3.9 -> 10, go -> 1, cache 3 hits. black clean. --- .../schemas/test_init_with_schemas_command.py | 46 ++----------------- 1 file changed, 5 insertions(+), 41 deletions(-) diff --git a/tests/integration/init/schemas/test_init_with_schemas_command.py b/tests/integration/init/schemas/test_init_with_schemas_command.py index 22f5186d065..b436f5e30bb 100644 --- a/tests/integration/init/schemas/test_init_with_schemas_command.py +++ b/tests/integration/init/schemas/test_init_with_schemas_command.py @@ -19,15 +19,7 @@ EVENT_BRIDGE_USE_CASE = "Infrastructure event management" -# Runtimes these tests init against. None of these tests is about a specific runtime -# version -- they exercise the schemas flows (registry choice, pagination, profiles) -- -# so the version is incidental and repeated across tests. Naming it here means a runtime -# leaving the manifest is a one-line change instead of a hunt through the file. -# `python3.9` is already deprecated, so that is not hypothetical. -# -# These are the labels the *prompt* displays, which are not always the runtime id: -# the go entry shows as "go (provided.al2)". `_get_runtime_position` matches them -# against the manifest, and raises with the available list if one disappears. +# Prompt labels, not runtime ids -- the go entry displays as "go (provided.al2)". JAVA_RUNTIME_FOR_INIT = "java17.al2023" PYTHON_RUNTIME_FOR_INIT = "python3.9" GO_RUNTIME_FOR_INIT = "go (provided.al2)" @@ -54,31 +46,12 @@ def _get_registry_position(registry_name): @lru_cache(maxsize=1) def _get_manifest(): - """Fetch the preprocessed app-templates manifest once per test process. - - `get_preprocessed_manifest` is not cheap or deterministic per call: - `InitTemplates._get_manifest` does `requests.get(MANIFEST_URL, timeout=10)` and, on - timeout / connection error / non-200, falls back to cloning the templates repo or - reading the bundled local manifest. Every test below resolves both a use case and a - runtime, so without caching that is two fetches per test. - - Caching also removes a correctness trap. Resolved independently, one call could - succeed against MANIFEST_URL while the other fell back to the local manifest -- - giving a use-case position and a runtime position from two different snapshots, - which is the same answer misalignment these helpers exist to prevent. - """ + """Fetch once per process: each call re-fetches MANIFEST_URL, and two calls can differ.""" return InitTemplates().get_preprocessed_manifest(None, None, None, None) def _get_runtime_position(runtime_name): - """Return the 1-based menu position of a runtime in the `sam init` runtime prompt. - - The prompt lists the runtimes the app-templates manifest offers for - EVENT_BRIDGE_USE_CASE, ordered by `get_sorted_runtimes`. That ordering shifts - whenever a runtime is added or removed, so hardcoding a position silently - starts selecting a different runtime -- every answer after it then lands on - the wrong question. Resolve it the same way `_get_registry_position` does. - """ + """Resolve a runtime's 1-based position; the order shifts as runtimes are added/removed.""" runtime_options = _get_manifest()[EVENT_BRIDGE_USE_CASE] runtimes = get_sorted_runtimes(list(runtime_options.keys())) for i, name in enumerate(runtimes, 1): @@ -88,11 +61,7 @@ def _get_runtime_position(runtime_name): def _get_use_case_position(use_case_name): - """Return the 1-based menu position of a use case in the `sam init` template prompt. - - Same rationale as `_get_runtime_position`: the manifest grows over time, so - the position of a use case is not stable. - """ + """Resolve a use case's 1-based position; the manifest grows over time.""" use_cases = list(_get_manifest().keys()) for i, name in enumerate(use_cases, 1): if name == use_case_name: @@ -393,12 +362,7 @@ def test_init_interactive_with_event_bridge_app_non_default_profile_selection(se # schemas aws region us-east-1 # 1: select aws.events as registries # 1: select aws schema - # - # The registry answer stays hardcoded here, unlike the sibling tests. This test - # deliberately drives a non-default profile and an explicit us-east-1, so the - # registries the prompt lists are those of *that* profile/region -- - # `_get_registry_position` resolves against the default `Session()`, so using it - # here would look up the wrong account and region. + # Registry hardcoded: non-default profile/region resolves a different account. user_input = f""" 1