ENT-14128: Made autolinking fail loudly, added script to build locally, and fixed broken links and other build issues

.gitignore

@@ -1,6 +1,9 @@
 # Build artifacts
 cfdoc_log.markdown
 
+/tmp/
+/output/
+
 # emacs
 
 *~

build-locally.sh

@@ -0,0 +1,141 @@
+#!/usr/bin/env bash
+#
+# Build the CFEngine documentation locally in a Docker container.
+#
+# Self-contained: clones the sibling repos that the build expects
+# (core, nova, enterprise, masterfiles, nt-docs) into ./tmp/ and runs
+# the existing Docker-based pipeline against them, without requiring
+# anything outside this directory.
+#
+# Override via env vars if needed:
+#   BRANCH                  branch name to build for (default: master)
+#   PACKAGE_JOB             cf-remote or a buildcache job (default: cf-remote)
+#   PACKAGE_UPLOAD_DIRECTORY  (default: n/a — unused with cf-remote)
+#   PACKAGE_BUILD             (default: n/a — unused with cf-remote)
+#   LTS_VERSION               (default: empty)
+#   DOCKER                  docker binary to use (default: docker)
+#   IMAGE_NAME              tag for the build image (default: cfengine-docs-hugo)
+#   SKIP_PUBLISH=1          skip the _publish.sh step (just build)
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+cd "$SCRIPT_DIR"
+
+TMP_DIR="$SCRIPT_DIR/tmp"
+CACHE_DIR="$TMP_DIR/cache"   # persistent clones with .git
+WORK_DIR="$TMP_DIR/work"     # clean working copies (.git stripped) — what we mount
+DOC_WORK="$WORK_DIR/documentation"
+mkdir -p "$CACHE_DIR" "$WORK_DIR"
+
+BRANCH="${BRANCH:-master}"
+PACKAGE_JOB="${PACKAGE_JOB:-cf-remote}"
+PACKAGE_UPLOAD_DIRECTORY="${PACKAGE_UPLOAD_DIRECTORY:-n/a}"
+PACKAGE_BUILD="${PACKAGE_BUILD:-n/a}"
+LTS_VERSION="${LTS_VERSION:-}"
+DOCKER="${DOCKER:-docker}"
+IMAGE_NAME="${IMAGE_NAME:-cfengine-docs-hugo}"
+
+# repo_name url default_branch
+REPOS=(
+    "core            git@github.com:cfengine/core.git              master"
+    "nova            git@github.com:cfengine/nova.git              master"
+    "enterprise      git@github.com:cfengine/enterprise.git        master"
+    "masterfiles     git@github.com:cfengine/masterfiles.git       master"
+    "nt-docs         git@github.com:northerntechhq/nt-docs.git     main"
+)
+
+# 1. Clone (or update) the sibling repos under tmp/cache/, then export a
+#    clean working copy (no .git) to tmp/work/. We mount the .git-free
+#    copy because the container does `chmod -R` over each repo, and on
+#    macOS Docker bind mounts can't chmod git pack files written by the
+#    host user.
+echo "==> Preparing sibling repos under $TMP_DIR"
+for entry in "${REPOS[@]}"; do
+    # shellcheck disable=SC2086
+    set -- $entry
+    name="$1"; url="$2"; default_branch="$3"
+    cache="$CACHE_DIR/$name"
+    work="$WORK_DIR/$name"
+
+    if [ -d "$cache/.git" ]; then
+        echo "  - $name: fetching latest"
+        git -C "$cache" fetch --quiet --tags origin
+    else
+        echo "  - $name: cloning $url"
+        git clone --quiet "$url" "$cache"
+    fi
+
+    if git -C "$cache" rev-parse --verify --quiet "origin/$BRANCH" >/dev/null; then
+        git -C "$cache" checkout --quiet -B "$BRANCH" "origin/$BRANCH"
+    else
+        echo "    branch '$BRANCH' not found in $name; using '$default_branch'"
+        git -C "$cache" checkout --quiet -B "$default_branch" "origin/$default_branch"
+    fi
+
+    # Export a clean snapshot for the container. Using `git archive` so
+    # we get exactly what's tracked, without .git or untracked junk.
+    rm -rf "$work"
+    mkdir -p "$work"
+    git -C "$cache" archive --format=tar HEAD | tar -x -C "$work"
+done
+
+# 1b. Sync the documentation source itself into tmp/work/documentation.
+# The build mutates files in place (sed on config.toml, cfdoc_preprocess.py
+# rewriting markdown, etc.), so we must NOT bind-mount the user's checkout
+# directly. Use rsync with --delete to keep the copy in sync (including
+# uncommitted/untracked changes) without dragging tmp/ or .git into it.
+# cfdoc_log.markdown is a build artifact written into content/ by cfdoc_qa.py
+# (it's gitignored). If a previous run left it behind, the link checker
+# re-parses its log entries — which themselves contain literal
+# [foo#foo][foo#foo] markdown — and reports hundreds of bogus "unresolved
+# reference" errors. We must clear it from both sides: --exclude keeps the
+# host's copy from being synced in, and the explicit rm removes any copy a
+# previous in-container build wrote into the work tree (rsync --delete will
+# NOT remove an --exclude'd path, so the exclude alone is not enough).
+echo "==> Syncing documentation source to $DOC_WORK"
+mkdir -p "$DOC_WORK"
+rsync -a --delete \
+    --exclude='/tmp/' \
+    --exclude='/.git/' \
+    --exclude='/content/cfdoc_log.markdown' \
+    "$SCRIPT_DIR/" "$DOC_WORK/"
+rm -f "$DOC_WORK/content/cfdoc_log.markdown"
+
+# 2. Build the docker image (only if it's not already built).
+if ! "$DOCKER" image inspect "$IMAGE_NAME" >/dev/null 2>&1; then
+    echo "==> Building docker image $IMAGE_NAME"
+    "$DOCKER" build --tag "$IMAGE_NAME" "$SCRIPT_DIR/generator/build"
+else
+    echo "==> Reusing docker image $IMAGE_NAME (delete it to rebuild)"
+fi
+
+# 3. Run the build inside the container.
+# main.sh expects /nt/{documentation,core,nova,enterprise,masterfiles,nt-docs}.
+# We bind-mount this checkout as /nt/documentation and each tmp/<repo> as
+# its sibling, so nothing outside this directory is touched.
+echo "==> Running documentation build in container"
+RUN_FLAGS=(
+    --rm
+    -v "$DOC_WORK:/nt/documentation"
+)
+for entry in "${REPOS[@]}"; do
+    # shellcheck disable=SC2086
+    set -- $entry
+    RUN_FLAGS+=(-v "$WORK_DIR/$1:/nt/$1")
+done
+
+"$DOCKER" run "${RUN_FLAGS[@]}" "$IMAGE_NAME" \
+    bash -x documentation/generator/build/main.sh \
+        "$BRANCH" "$PACKAGE_JOB" "$PACKAGE_UPLOAD_DIRECTORY" \
+        "$PACKAGE_BUILD" "$LTS_VERSION"
+
+# 4. Optionally package the result (mirrors the Jenkins pipeline).
+if [ -z "${SKIP_PUBLISH:-}" ]; then
+    echo "==> Packaging output"
+    "$DOCKER" run "${RUN_FLAGS[@]}" "$IMAGE_NAME" \
+        bash -x documentation/generator/_scripts/_publish.sh "$BRANCH"
+fi
+
+echo "==> Done. Generated site is in: $DOC_WORK/generator/_site"
+echo "    Tarballs (if packaged) are in: $DOC_WORK/output/"

content/examples/tutorials/promise-type-module-development.markdown

@@ -10,7 +10,7 @@ aliases:
 
 Promise type modules are easy to write.
 In this tutorial, we will focus on implementing a new promise type in Python, with the provided CFEngine library, since this is the easiest and recommended way.
-If you are interested in how modules are implemented, or how you could do it in another programming language, see the [complete documentation][custom].
+If you are interested in how modules are implemented, or how you could do it in another programming language, see the [complete documentation][promise-type-custom].
 
 In short, you need to implement 2 functions: `validate_promise()` and `evaluate_promise()`.
 _Validation_ should check that the correct attributes are used, and any other constraints you may want to enforce, to determine whether a promise is valid or invalid.

content/getting-started/01-installing-cfengine/general-installation/_index.markdown

@@ -119,7 +119,7 @@ Edit `/etc/hosts` and add an entry for the IP address and hostname of the server
 
 ### CFEngine Enterprise post-installation setup
 
-See: [What steps should I take after installing CFEngine Enterprise?][FAQ#What steps should I take after installing CFEngine Enterprise]
+See: [What steps should I take after installing CFEngine Enterprise?][Enterprise reporting database#What steps should I take after installing CFEngine Enterprise?]
 
 ## More detailed installation guides
 

content/getting-started/01-installing-cfengine/local-virtual-machine.markdown

@@ -121,4 +121,4 @@ vagrant destroy hub
 
 Now that you have a Linux VM ready, go back to the main tutorial to install CFEngine:
 
-[Installation][Installation]
+[Installation][Installing CFEngine]

content/overview/_index.markdown

@@ -6,7 +6,7 @@ aliases:
   - "/overview.html"
 ---
 
-CFEngine is a distributed system for managing and monitoring computers across an IT network. Machines on the network that have CFEngine installed, and have registered themselves with a policy server (see [Installation][Installation]), will each be running a set of CFEngine component applications that manage and interpret a textual representation of your desired state for the system, referred to as policy. Policy files themselves contain sets of instructions to ensure machines on the network are in full compliance with a defined state. At the atomic level are sets, or _bundles_, of what are known in the CFEngine world as [Promises][Promises]. _Promises_ are at the heart of Promise Theory, which is in turn what CFEngine is all about.
+CFEngine is a distributed system for managing and monitoring computers across an IT network. Machines on the network that have CFEngine installed, and have registered themselves with a policy server (see [Installation][Installing CFEngine]), will each be running a set of CFEngine component applications that manage and interpret a textual representation of your desired state for the system, referred to as policy. Policy files themselves contain sets of instructions to ensure machines on the network are in full compliance with a defined state. At the atomic level are sets, or _bundles_, of what are known in the CFEngine world as [Promises][Promises]. _Promises_ are at the heart of Promise Theory, which is in turn what CFEngine is all about.
 
 ## Policy language and compliance
 

content/reference/promise-types/services.markdown

@@ -249,7 +249,7 @@ bundle agent my_custom_service_method_deb( service_identifier, desired_service_s
 }
 ```
 
-**See also:** [generic standard_services][Services Bodies and Bundles#standard_services]
+**See also:** [generic standard_services][lib/services.cf]
 
 **History:**
 

content/resources/faq/bootstrap-failed.markdown

@@ -136,7 +136,7 @@ In order for a host to communicate it must be within an IP range that is allowed
 
 - `def.acl` in the Masterfiles Policy Framework is included in this list by default.
 
-See also: [`def.acl`][Masterfiles Policy Framework#acl], [`def.trustkeysfrom`][Masterfiles Policy Framework#trustkeysfrom]
+See also: [`def.acl`][Masterfiles Policy Framework#acl], [`def.trustkeysfrom`][Masterfiles Policy Framework#Automatic bootstrap - Trusting keys from new hosts with trustkeysfrom]
 
 ### `trustkeysfrom` in `body server control`
 
@@ -149,4 +149,4 @@ This defines networks from which a host will automatically trust hosts. If you d
 - `verbose: 192.168.56.4>    Trying old style '/var/cfengine/ppkeys/root-192.168.56.4.pub'`
 - `verbose: 192.168.56.4>    Received key 'SHA=85f8a23d6738599e03951e6930e661bcd9bb3ae12f32486c9795cc9baa7d5b4e' not found in ppkeys`
 
-See also: [`def.acl`][Masterfiles Policy Framework#acl], [`def.trustkeysfrom`][Masterfiles Policy Framework#trustkeysfrom]
+See also: [`def.acl`][Masterfiles Policy Framework#acl], [`def.trustkeysfrom`][Masterfiles Policy Framework#Automatic bootstrap - Trusting keys from new hosts with trustkeysfrom]

content/resources/faq/integrate-custom-policy.markdown

@@ -22,7 +22,7 @@ The _autorun_ feature in the Masterfiles Policy Framework automatically adds
 policy files found in `services/autorun` to inputs and executes bundles tagged
 with _autorun_ as methods type promises in lexical order.
 
-**See also:** [`services_autorun` in the Masterfiles Policy Framework][Masterfiles Policy Framework#services\_autorun]
+**See also:** [`services_autorun` in the Masterfiles Policy Framework][mpf-services-autorun]
 
 ## Using augments
 
@@ -51,7 +51,7 @@ To extend inputs in the update policy define `update_inputs`.
 }
 ```
 
-**See also:** [Augments][Augments], [Extend inputs for update policy in the Masterfiles Policy Framework][Masterfiles Policy Framework#Append to inputs used by update policy]
+**See also:** [Augments][Augments], [Extend inputs for update policy in the Masterfiles Policy Framework][Append to inputs used by update policy]
 
 ## Using body file control
 

content/resources/faq/why-are-files-not-being-distributed.markdown

@@ -13,4 +13,4 @@ However not all files are
 considered for update in the default update policy.
 
 The default update policy in the MPF only copies files matching a list of
-regular expressions defined in [update_def.input_name_patterns][Masterfiles Policy Framework#files considered for copy during policy updates]
+regular expressions defined in [update_def.input_name_patterns][Masterfiles Policy Framework#Extend files considered for copy during policy updates]

content/web-ui/hub_administration/adjusting-schedules.markdown

@@ -8,8 +8,8 @@ aliases:
 ## Set cf-execd agent execution schedule
 
 By default `cf-execd` is configured to run `cf-agent` every 5 minutes. This can
-be adjusted by tuning the [schedule][cf-execd#schedule] in `body executor
-control`. In the [Masterfiles Policy Framework][Masterfiles Policy Framework] body
+be adjusted by tuning the [schedule][cf-execd#schedule] in
+`body executor control`. In the [Masterfiles Policy Framework][Masterfiles Policy Framework] body
 executor control can be found in `controls/cf_execd.cf`
 
 ## Set cf-hub hub_schedule

generator/_no_links.md

@@ -0,0 +1,15 @@
+<!--
+  Names that look like CFEngine functions (`foo()`) but should NOT be
+  autolinked. Autolinking normally turns `foo()` in backticks into a link to
+  the documentation for that function, and fails the build if no target is
+  found (see cfdoc_references_resolver.py). List one name per line here to opt
+  it out of autolinking entirely — it is left as plain backtick text and does
+  not count as a broken link. Lines starting with `<!--`/blank lines are
+  ignored. The trailing `()` is optional.
+-->
+
+validate_promise()
+evaluate_promise()
+PolicyResolve()
+BodyToJson()
+BundleToJson()

generator/_references.md

@@ -39,6 +39,7 @@
 [sys.uqhost]: reference-special-variables-sys.html#sys-uqhost
 [sys.policy_hub]: reference-special-variables-sys.html#sys-policy_hub
 [seed_cp]: reference-masterfiles-policy-framework-lib-files.html#seed_cp
+[Masterfiles Policy Framework]: reference-masterfiles-policy-framework.html
 [Append to inputs used by main policy]: reference-masterfiles-policy-framework.html#append-to-inputs-used-by-main-policy
 [mpf_extra_autorun_inputs]: reference-masterfiles-policy-framework.html#additional-automatically-loaded-inputs
 [Append to inputs used by update policy]: reference-masterfiles-policy-framework.html#append-to-inputs-used-by-update-policy

generator/_scripts/cfdoc_linkresolver.py

@@ -237,12 +237,31 @@ def applyLinkMap(file_name, config):
     for markdown_line in markdown_lines:
         config["context_current_line_number"] += 1
         # we ignore everything in code blocks
-        if previous_empty or in_pre:
-            if markdown_line.lstrip()[:3] == "```":
-                in_pre = not in_pre
-            if markdown_line[:4] == "    ":
-                new_lines.append(markdown_line)
-                continue
+        # Track fenced code blocks the way CommonMark/Hugo do, so we don't
+        # autolink inside code and don't lose state on macro-injected examples:
+        #   * a line is a fence only if it starts with ``` and has no further
+        #     ``` on the same line (inline "```x```" is not a fence);
+        #   * an open block closes only on a *bare* ``` (no info string), so a
+        #     "```cf3" line appearing inside an open block counts as content.
+        # A naive toggle miscounts inner fences and flips the in/out-of-code
+        # state for the rest of the file.
+        stripped = markdown_line.lstrip()
+        is_fence = stripped.startswith("```") and "```" not in stripped[3:]
+        if is_fence and not in_pre:
+            in_pre = True
+            new_lines.append(markdown_line)
+            previous_empty = False
+            continue
+        if in_pre:
+            new_lines.append(markdown_line)
+            if is_fence and stripped.strip().strip("`") == "":
+                in_pre = False
+            previous_empty = markdown_line.lstrip() == ""
+            continue
+        if previous_empty and markdown_line[:4] == "    ":
+            new_lines.append(markdown_line)
+            previous_empty = False
+            continue
 
         # don't link to the current section
         if markdown_line.find("title:") == 0:

generator/_scripts/cfdoc_macros.py

@@ -620,11 +620,25 @@ def prune_include_lines(markdown_lines, filename):
         # unless included example starts with an explicit code block, start a (CFEngine-brushed) block
         if markdown_lines[0].find("```") != 0:
             markdown_lines.insert(0, "\n```%s\n" % brush)
-        # if example ended with documentation, prune trailing code, else terminate block
-        if markdown_lines[-1] != ("\n```%s\n" % brush):
-            markdown_lines.append("```\n")
-        else:
+        if markdown_lines[-1] == ("\n```%s\n" % brush):
+            # snippet ended by opening a fresh (empty) code block; drop it
             del markdown_lines[-1]
+        else:
+            # Close the block only if the snippet actually ends *inside* one.
+            # An example that ends with documentation (#@ ...) has already
+            # closed its last code block, so appending another ``` would leave
+            # an unmatched fence that swallows all the content following the
+            # include (e.g. hiding real reference links from the resolvers).
+            in_code = False
+            for ml in markdown_lines:
+                stripped = ml.lstrip()
+                if stripped.startswith("```") and "```" not in stripped[3:]:
+                    if not in_code:
+                        in_code = True
+                    elif stripped.strip().strip("`") == "":
+                        in_code = False
+            if in_code:
+                markdown_lines.append("```\n")
     return markdown_lines