From 90b32ef054c556d3e5bcfc65a85f696f398df18c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20M=C3=BCller?=
 <1005065+DeepDiver1975@users.noreply.github.com>
Date: Fri, 21 Aug 2026 17:00:06 +0200
Subject: [PATCH] fix(ui): publish robots.txt at the site root again
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The retired docs-ui bundle shipped src/ui.yml with a static_files list
that included robots.txt -- that descriptor, not the file's location, is
what made https://doc.owncloud.com/robots.txt serve. The move to this
Antora monorepo copied docs-ui's static assets into ui/supplemental/ but
not its ui.yml, and the stock antora-ui-default bundle ships none, so no
code path was left that could publish anything at the site root:
@antora/ui-loader classifies every unlisted UI file as an asset under
ui.output_dir (assets/). robots.txt therefore had nowhere to land and
the URL has 404'd since, leaving the sitemap that
antora-extensions/sitemap-cleanup.js still publishes unadvertised.

Add the missing ui.yml descriptor plus robots.txt itself, carrying over
the legacy content and making the allow-all explicit with an empty
Disallow (RFC 9309 4.2.1); the legacy file's bare `User-agent: *` group
carried no rule at all.

test/static-files.test.js guards the build output: robots.txt at the
root, its Sitemap line matching site.url from site.yml, the advertised
sitemap.xml actually present, and neither robots.txt nor ui.yml leaking
into assets/ -- a dropped descriptor would otherwise still build green.

Verified on a full `npm run antora` build served locally: /robots.txt
200 text/plain, /sitemap.xml 200, /assets/ui.yml 404.

Closes #100

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Signed-off-by: Thomas Müller <1005065+DeepDiver1975@users.noreply.github.com>
---
 test/static-files.test.js  | 64 ++++++++++++++++++++++++++++++++++++++
 ui/supplemental/robots.txt |  7 +++++
 ui/supplemental/ui.yml     | 11 +++++++
 3 files changed, 82 insertions(+)
 create mode 100644 test/static-files.test.js
 create mode 100644 ui/supplemental/robots.txt
 create mode 100644 ui/supplemental/ui.yml

diff --git a/test/static-files.test.js b/test/static-files.test.js
new file mode 100644
index 0000000..4ff4e7c
--- /dev/null
+++ b/test/static-files.test.js
@@ -0,0 +1,64 @@
+'use strict'
+
+// Build-output guards for the root-level static files (robots.txt). Antora only
+// publishes a UI file at the site ROOT when ui/supplemental/ui.yml lists it
+// under `static_files`; every other file in ui/supplemental/ is classified as an
+// asset and lands under `assets/` instead -- which is how robots.txt got lost
+// when the docs site moved from the docs-ui bundle into this monorepo. These
+// tests assert against the generated public/ tree and skip when the site has not
+// been built yet (run `npm run antora`).
+
+const test = require('node:test')
+const assert = require('node:assert/strict')
+const fs = require('node:fs')
+const path = require('node:path')
+
+const ROOT = path.join(__dirname, '..')
+const PUBLIC = path.join(ROOT, 'public')
+
+// site.url from site.yml -- the two-space indent scopes the match to the `site:`
+// block, so the content source's `- url: .` cannot match. Deriving it here keeps
+// the Sitemap line in robots.txt from silently outliving a domain change.
+const SITE_URL = (fs.readFileSync(path.join(ROOT, 'site.yml'), 'utf8').match(/^ {2}url: (\S+)/m) || [])[1]
+
+function builtOrSkip (t, rel) {
+  const p = path.join(PUBLIC, rel)
+  if (!fs.existsSync(path.join(PUBLIC, 'index.html'))) {
+    t.skip('public/ not built (run `npm run antora` to enable)')
+    return null
+  }
+  assert.ok(fs.existsSync(p), `${rel} was not published to the site root`)
+  return fs.readFileSync(p, 'utf8')
+}
+
+test('robots.txt is published at the site root and allows all crawlers', (t) => {
+  const robots = builtOrSkip(t, 'robots.txt')
+  if (robots == null) return
+  assert.match(robots, /^User-agent: \*$/m, 'robots.txt has no `User-agent: *` group')
+  // An empty Disallow is the explicit "crawl everything" rule (RFC 9309 4.2.1).
+  assert.match(robots, /^Disallow:\s*$/m, 'robots.txt does not explicitly allow all paths')
+})
+
+test('robots.txt points crawlers at the published sitemap', (t) => {
+  const robots = builtOrSkip(t, 'robots.txt')
+  if (robots == null) return
+  const sitemap = (robots.match(/^Sitemap: (\S+)$/m) || [])[1]
+  assert.equal(sitemap, `${SITE_URL}/sitemap.xml`, 'Sitemap line does not match site.url from site.yml')
+  // The advertised sitemap must be a file we actually publish, not a 404.
+  assert.ok(fs.existsSync(path.join(PUBLIC, 'sitemap.xml')), 'advertised sitemap.xml is not in the build output')
+})
+
+test('root static files are not published under the UI output dir', (t) => {
+  if (!fs.existsSync(path.join(PUBLIC, 'index.html'))) {
+    t.skip('public/ not built (run `npm run antora` to enable)')
+    return
+  }
+  // A missing/incomplete ui.yml sends the file to assets/ instead of the root,
+  // where no crawler looks -- and the site would still build green.
+  assert.ok(
+    !fs.existsSync(path.join(PUBLIC, 'assets', 'robots.txt')),
+    'robots.txt was published as a UI asset (is it listed in ui/supplemental/ui.yml static_files?)'
+  )
+  // ui.yml is consumed by the UI loader; it must never be published itself.
+  assert.ok(!fs.existsSync(path.join(PUBLIC, 'assets', 'ui.yml')), 'ui.yml leaked into the build output')
+})
diff --git a/ui/supplemental/robots.txt b/ui/supplemental/robots.txt
new file mode 100644
index 0000000..e6bd302
--- /dev/null
+++ b/ui/supplemental/robots.txt
@@ -0,0 +1,7 @@
+# Published at the site root via the static_files list in ui.yml.
+# Crawl everything, and point crawlers at the sitemap that
+# antora-extensions/sitemap-cleanup.js writes to the site root.
+User-agent: *
+Disallow:
+
+Sitemap: https://doc.owncloud.com/sitemap.xml
diff --git a/ui/supplemental/ui.yml b/ui/supplemental/ui.yml
new file mode 100644
index 0000000..9f6e159
--- /dev/null
+++ b/ui/supplemental/ui.yml
@@ -0,0 +1,11 @@
+# Antora UI descriptor. The UI loader picks this up from the merged UI file map
+# (bundle files overlaid with ui.supplemental_files) and publishes everything
+# listed under static_files at the SITE ROOT, ignoring `ui.output_dir` from
+# site.yml. Without it, ui/supplemental/robots.txt would be classified as a
+# regular UI asset and published as assets/robots.txt, where crawlers never look.
+# The stock antora-ui-default bundle ships no ui.yml, so this adds the descriptor
+# rather than overriding one -- if the bundle ever gains one, its static_files
+# entries must be merged in here, because this file replaces it wholesale.
+# see: https://docs.antora.org/antora-ui-default/static-files/
+static_files:
+- robots.txt