Added APIC OOB connectivity checks for CSCwu91693

aci-preupgrade-validation-script.py

@@ -6702,6 +6702,85 @@ def stale_dbgacEpgSummaryTask_check(tversion, **kwargs):
     return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url)
 
 
+@check_wrapper(check_title="APIC OOB Connectivity")
+def apic_oob_connectivity_check(cversion, tversion, **kwargs):
+    result = PASS
+    headers = ["Node ID", "OOB IP", "Port", "Status"]
+    recommended_action = "Restore OOB management connectivity between all APICs and ensure the required HTTPS ports are reachable across the OOB network."
+    doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#apic-oob-connectivity'
+
+    def get_apic_oob_connectivity(topSystems, port):
+        data = []
+        has_error = False
+
+        for topSystem in topSystems:
+            attrs = topSystem['topSystem']['attributes']
+            node_id = attrs.get('id', '')
+
+            if attrs.get('oobMgmtAddr', '0.0.0.0') != '0.0.0.0':
+                ip = attrs.get('oobMgmtAddr')
+            elif attrs.get('oobMgmtAddr6', '::') not in ('', '::', '0:0:0:0:0:0:0:0'):
+                ip = attrs.get('oobMgmtAddr6')
+            else:
+                continue
+
+            try:
+                if subprocess.call(
+                    'curl --max-time 5 -k -s -o /dev/null https://{}:{} 2>/dev/null'.format(ip, port),
+                    shell=True
+                ) != 0:
+                    data.append([node_id, ip, port, "Unreachable"])
+            except Exception as e:
+                log.error("Exception checking OOB connectivity for node %s: %s", node_id, e)
+                data.append([node_id, ip, port, "Error"])
+                has_error = True
+                continue
+
+        return data, has_error
+
+    if not tversion:
+        return Result(result=MANUAL, msg=TVER_MISSING)
+
+    if tversion.older_than("6.0(2a)"):
+        return Result(result=NA, msg=VER_NOT_AFFECTED)
+
+    topSystems = icurl('class', 'topSystem.json?query-target-filter=eq(topSystem.role,"controller")')
+    if not topSystems:
+        return Result(result=NA, msg="No APIC controller nodes found.")
+
+    data = []
+    has_error = False
+
+    # Default port check: APIC bootx uses port 443 from 6.0(2)
+    default_data, default_error = get_apic_oob_connectivity(topSystems, 443)
+    data.extend(default_data)
+    if default_error:
+        has_error = True
+
+    # Custom HTTPS port check: upgrade fanout uses commHttps port from 6.2(1)
+    if not cversion.older_than("6.2(1g)"):
+        port = 443
+        commHttps = icurl('class', 'commHttps.json')
+        if commHttps:
+            try:
+                port = int(commHttps[0]['commHttps']['attributes'].get('port', 443))
+            except (ValueError, KeyError):
+                log.warning("Could not read commHttps port")
+                return Result(result=ERROR, msg="Could not read commHttps HTTPS port from commHttps MO.")
+
+        if port != 443:  # Port 443 already covered by default port check above
+            custom_data, custom_error = get_apic_oob_connectivity(topSystems, port)
+            data.extend(custom_data)
+            if custom_error:
+                has_error = True
+
+    if has_error:
+        result = ERROR
+    elif data:
+        result = FAIL_UF
+    return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url)
+
+
 # ---- Script Execution ----
 
 
@@ -6877,6 +6956,7 @@ class CheckManager:
         wred_affected_model_check,
         n9k_c93180yc_fx3_switch_memory_check,
         stale_dbgacEpgSummaryTask_check,
+        apic_oob_connectivity_check,
 
     ]
     ssh_checks = [

docs/docs/validations.md

@@ -206,6 +206,7 @@ Items                                           | Defect       | This Script
 [WRED with Affected FM Models][d35]             | CSCwt50713   | :white_check_mark: | :no_entry_sign:
 [N9K-C93180YC-FX3 Switch Memory Less Than 32GB][d36] | CSCwm42741   | :white_check_mark: | :no_entry_sign:
 [Stale dbgacEpgSummaryTask Objects][d37]         | CSCwt69100   | :white_check_mark: | :no_entry_sign:
+[APIC OOB Connectivity][d38]                      | CSCwu91693   | :white_check_mark: | :white_check_mark: 6.2(2)
 
 [d1]: #ep-announce-compatibility
 [d2]: #eventmgr-db-size-defect-susceptibility
@@ -244,6 +245,7 @@ Items                                           | Defect       | This Script
 [d35]: #wred-with-affected-fm-models
 [d36]: #n9k-c93180yc-fx3-switch-memory-less-than-32gb
 [d37]: #stale-dbgacepgsummarytask-objects
+[d38]: #apic-oob-connectivity
 
 ## General Check Details
 
@@ -2846,6 +2848,15 @@ Affected versions: 6.1(5e) and below, or 6.2(1g).
 Contact Cisco TAC for next steps. For more details, refer to the workaround in [CSCwt69100][75].
 
 
+### APIC OOB Connectivity
+
+Starting from 6.0(2), APIC firmware upgrades are triggered via an HTTPS POST request (bootx) sent to each peer APIC over its out-of-band (OOB) management interface. Due to [CSCwu91693][77], if OOB connectivity to a peer APIC is unavailable at the time this trigger is sent, that APIC does not receive it and silently fails to start the upgrade, while the remaining reachable APICs proceed normally. This results in a partially upgraded cluster with no explicit error raised at the time of failure.
+
+This check verifies OOB reachability between APICs on the port(s) actually used for the upgrade trigger. The default port 443 is validated on all versions from 6.0(2) onward, since it is always used unless a custom HTTPS port is configured. From 6.2(1) onward, the upgrade trigger also honors a custom HTTPS port if one is configured via the `commHttps` policy; this custom port is validated only when the current version is 6.2(1) or later, and only when the configured port differs from 443, which is already covered by the default check.
+
+For each applicable port, the script queries `topSystem` filtered to `role=controller` to collect the OOB management IP of every APIC in the cluster, then attempts an HTTPS connection to each peer on that port with a 5-second timeout. If any APIC is found unreachable on a port used for the upgrade trigger, the check fails.
+
+
 [0]: https://github.com/datacenter/ACI-Pre-Upgrade-Validation-Script
 [1]: https://www.cisco.com/c/dam/en/us/td/docs/Website/datacenter/apicmatrix/index.html
 [2]: https://www.cisco.com/c/en/us/support/switches/nexus-9000-series-switches/products-release-notes-list.html
@@ -2923,4 +2934,5 @@ Contact Cisco TAC for next steps. For more details, refer to the workaround in [
 [74]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwm42741
 [75]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwt69100
 [76]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwt38698
+[77]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwu91693
 

tests/checks/apic_oob_connectivity_check/commHttps_custom_port.json

@@ -0,0 +1,9 @@
+[
+    {
+        "commHttps": {
+            "attributes": {
+                "port": "8443"
+            }
+        }
+    }
+]

tests/checks/apic_oob_connectivity_check/commHttps_default_port.json

@@ -0,0 +1,9 @@
+[
+    {
+        "commHttps": {
+            "attributes": {
+                "port": "443"
+            }
+        }
+    }
+]

tests/checks/apic_oob_connectivity_check/commHttps_invalid_port.json

@@ -0,0 +1 @@
+[{"commHttps": {"attributes": {"port": "invalid"}}}]

tests/checks/apic_oob_connectivity_check/test_apic_oob_connectivity_check.py

@@ -0,0 +1,159 @@
+import os
+import pytest
+import logging
+import importlib
+from helpers.utils import read_data
+
+script = importlib.import_module("aci-preupgrade-validation-script")
+
+log = logging.getLogger(__name__)
+dir = os.path.dirname(os.path.abspath(__file__))
+
+test_function = "apic_oob_connectivity_check"
+
+# icurl queries
+topSystem = 'topSystem.json?query-target-filter=eq(topSystem.role,"controller")'
+commHttps = "commHttps.json"
+
+
+@pytest.mark.parametrize(
+    "icurl_outputs, cversion, tversion, curl_exit_codes, expected_result",
+    [
+        # tversion not provided -> MANUAL
+        (
+            {topSystem: [], commHttps: []},
+            "6.0(2a)",
+            None,
+            [],
+            script.MANUAL,
+        ),
+        # tversion < 6.0(2a) -> NA (version not affected)
+        (
+            {topSystem: [], commHttps: []},
+            "5.2(7f)",
+            "5.2(7f)",
+            [],
+            script.NA,
+        ),
+        # tversion >= 6.0(2a), no controller nodes found -> NA
+        (
+            {topSystem: [], commHttps: []},
+            "6.0(2a)",
+            "6.0(3a)",
+            [],
+            script.NA,
+        ),
+        # tversion >= 6.0(2a), all APICs have no OOB configured -> PASS
+        (
+            {topSystem: read_data(dir, "topSystem_no_oob.json"), commHttps: []},
+            "6.0(2a)",
+            "6.0(3a)",
+            [],
+            script.PASS,
+        ),
+        # tversion >= 6.0(2a), all APICs reachable on port 443 -> PASS
+        (
+            {topSystem: read_data(dir, "topSystem_3apics_oob.json"), commHttps: []},
+            "6.0(2a)",
+            "6.0(3a)",
+            [0, 0, 0],
+            script.PASS,
+        ),
+        # cversion >= 6.2(1g), default port 443, all APICs reachable -> PASS
+        # (custom port check skipped since commHttps port == 443)
+        (
+            {
+                topSystem: read_data(dir, "topSystem_3apics_oob.json"),
+                commHttps: read_data(dir, "commHttps_default_port.json"),
+            },
+            "6.2(1g)",
+            "6.2(2a)",
+            [0, 0, 0],
+            script.PASS,
+        ),
+        # cversion >= 6.2(1g), custom port 8443, all APICs reachable on both ports -> PASS
+        # (default port 443: [0,0,0], custom port 8443: [0,0,0])
+        (
+            {
+                topSystem: read_data(dir, "topSystem_3apics_oob.json"),
+                commHttps: read_data(dir, "commHttps_custom_port.json"),
+            },
+            "6.2(1g)",
+            "6.2(2a)",
+            [0, 0, 0, 0, 0, 0],
+            script.PASS,
+        ),
+        # tversion >= 6.0(2a), one APIC unreachable on port 443 (exit 28) -> FAIL_UF
+        (
+            {topSystem: read_data(dir, "topSystem_3apics_oob.json"), commHttps: []},
+            "6.0(2a)",
+            "6.0(3a)",
+            [0, 28, 0],
+            script.FAIL_UF,
+        ),
+        # cversion >= 6.2(1g), one APIC unreachable on default port 443 (exit 7) -> FAIL_UF
+        (
+            {
+                topSystem: read_data(dir, "topSystem_3apics_oob.json"),
+                commHttps: read_data(dir, "commHttps_default_port.json"),
+            },
+            "6.2(1g)",
+            "6.2(2a)",
+            [0, 7, 0],
+            script.FAIL_UF,
+        ),
+        # cversion >= 6.2(1g), custom port 8443, all unreachable on custom port -> FAIL_UF
+        # (default port 443 all pass: [0,0,0], custom port 8443 all fail: [28,28,28])
+        (
+            {
+                topSystem: read_data(dir, "topSystem_3apics_oob.json"),
+                commHttps: read_data(dir, "commHttps_custom_port.json"),
+            },
+            "6.2(1g)",
+            "6.2(2a)",
+            [0, 0, 0, 28, 28, 28],
+            script.FAIL_UF,
+        ),
+        # cversion >= 6.2(1g), commHttps returns invalid port value -> ERROR
+        (
+            {
+                topSystem: read_data(dir, "topSystem_3apics_oob.json"),
+                commHttps: read_data(dir, "commHttps_invalid_port.json"),
+            },
+            "6.2(1g)",
+            "6.2(2a)",
+            [0, 0, 0],
+            script.ERROR,
+        ),
+        # IPv6 OOB, all APICs reachable -> PASS
+        (
+            {topSystem: read_data(dir, "topSystem_3apics_oob_ipv6.json"), commHttps: []},
+            "6.0(2a)",
+            "6.0(3a)",
+            [0, 0, 0],
+            script.PASS,
+        ),
+        # IPv6 OOB, one APIC unreachable (exit 28) -> FAIL_UF
+        (
+            {topSystem: read_data(dir, "topSystem_3apics_oob_ipv6.json"), commHttps: []},
+            "6.0(2a)",
+            "6.0(3a)",
+            [0, 28, 0],
+            script.FAIL_UF,
+        ),
+    ],
+)
+def test_logic(run_check, mock_icurl, monkeypatch, cversion, tversion, curl_exit_codes, expected_result):
+    idx = [0]
+
+    def mock_subprocess_call(cmd, shell=False):
+        code = curl_exit_codes[idx[0]] if idx[0] < len(curl_exit_codes) else 0
+        idx[0] += 1
+        return code
+
+    monkeypatch.setattr(script.subprocess, "call", mock_subprocess_call)
+    result = run_check(
+        cversion=script.AciVersion(cversion),
+        tversion=script.AciVersion(tversion) if tversion else None,
+    )
+    assert result.result == expected_result

tests/checks/apic_oob_connectivity_check/topSystem_3apics_oob.json

@@ -0,0 +1,32 @@
+[
+    {
+        "topSystem": {
+            "attributes": {
+                "id": "1",
+                "role": "controller",
+                "oobMgmtAddr": "10.30.10.189",
+                "oobMgmtAddr6": "::"
+            }
+        }
+    },
+    {
+        "topSystem": {
+            "attributes": {
+                "id": "2",
+                "role": "controller",
+                "oobMgmtAddr": "10.30.10.191",
+                "oobMgmtAddr6": "::"
+            }
+        }
+    },
+    {
+        "topSystem": {
+            "attributes": {
+                "id": "3",
+                "role": "controller",
+                "oobMgmtAddr": "10.30.10.193",
+                "oobMgmtAddr6": "::"
+            }
+        }
+    }
+]

tests/checks/apic_oob_connectivity_check/topSystem_3apics_oob_ipv6.json

@@ -0,0 +1,32 @@
+[
+    {
+        "topSystem": {
+            "attributes": {
+                "id": "1",
+                "role": "controller",
+                "oobMgmtAddr": "0.0.0.0",
+                "oobMgmtAddr6": "2001:db8::1"
+            }
+        }
+    },
+    {
+        "topSystem": {
+            "attributes": {
+                "id": "2",
+                "role": "controller",
+                "oobMgmtAddr": "0.0.0.0",
+                "oobMgmtAddr6": "2001:db8::2"
+            }
+        }
+    },
+    {
+        "topSystem": {
+            "attributes": {
+                "id": "3",
+                "role": "controller",
+                "oobMgmtAddr": "0.0.0.0",
+                "oobMgmtAddr6": "2001:db8::3"
+            }
+        }
+    }
+]

tests/checks/apic_oob_connectivity_check/topSystem_no_oob.json

@@ -0,0 +1,12 @@
+[
+    {
+        "topSystem": {
+            "attributes": {
+                "id": "1",
+                "role": "controller",
+                "oobMgmtAddr": "0.0.0.0",
+                "oobMgmtAddr6": "::"
+            }
+        }
+    }
+]