Sitemap files are never read, so URLs nothing links to are never found

fuzz/Makefile.am

@@ -2,7 +2,7 @@
 if FUZZERS
 noinst_PROGRAMS = fuzz-charset fuzz-meta fuzz-idna fuzz-entities \
 	fuzz-unescape fuzz-filters fuzz-url fuzz-header fuzz-cachendx \
-	fuzz-htsparse fuzz-singlefile
+	fuzz-htsparse fuzz-singlefile fuzz-sitemap
 endif
 
 AM_CPPFLAGS = \
@@ -28,6 +28,7 @@ fuzz_header_SOURCES = fuzz-header.c fuzz.h
 fuzz_cachendx_SOURCES = fuzz-cachendx.c fuzz.h
 fuzz_htsparse_SOURCES = fuzz-htsparse.c fuzz.h
 fuzz_singlefile_SOURCES = fuzz-singlefile.c fuzz.h
+fuzz_sitemap_SOURCES = fuzz-sitemap.c fuzz.h
 
 # List corpus files explicitly: automake does not expand EXTRA_DIST globs.
 EXTRA_DIST = README.md run-fuzzers.sh \
@@ -52,4 +53,6 @@ EXTRA_DIST = README.md run-fuzzers.sh \
 	corpus/singlefile/img-src.html corpus/singlefile/link-rel.html \
 	corpus/singlefile/style-block.html corpus/singlefile/style-attr.html \
 	corpus/singlefile/srcset.html corpus/singlefile/rawtext.html \
-	corpus/singlefile/malformed.html corpus/singlefile/many-attrs.html
+	corpus/singlefile/malformed.html corpus/singlefile/many-attrs.html \
+	corpus/sitemap/urlset.xml corpus/sitemap/sitemapindex.xml \
+	corpus/sitemap/truncated.xml corpus/sitemap/urlset.xml.gz

fuzz/corpus/sitemap/sitemapindex.xml

@@ -0,0 +1 @@
+<sitemapindex><sitemap><loc>http://h.test/s2.xml.gz</loc></sitemap></sitemapindex>

fuzz/corpus/sitemap/truncated.xml

@@ -0,0 +1 @@
+<urlset><loc>http://h.test/x

fuzz/corpus/sitemap/urlset.xml

@@ -0,0 +1 @@
+
<?xml version="1.0"?><urlset><url><loc>http://h.test/a.html</loc></url><url><loc>https://h.test/b?x=1&amp;y=2</loc></url></urlset>

fuzz/fuzz-sitemap.c

@@ -0,0 +1,60 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 2026 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Fuzz the sitemap <loc> scanner (htssitemap.c): raw XML, gzip-framed bodies
+   and truncated streams all arrive here straight off the network. */
+#include "fuzz.h"
+#include "htssitemap.h"
+
+static hts_boolean sm_count(void *arg, const char *url) {
+  int *const n = (int *) arg;
+
+  (void) url;
+  (*n)++;
+  return HTS_TRUE;
+}
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  static const int caps[] = {0, 1, 16, HTS_SITEMAP_MAX_URLS_DOC};
+  hts_boolean is_index;
+  char *body;
+  int n = 0, cap;
+
+  if (size == 0)
+    return 0;
+  cap = caps[data[0] % (sizeof(caps) / sizeof(caps[0]))];
+  data++, size--;
+  /* A heap copy of exactly `size` bytes: the scanner must never rely on a
+     terminator, and ASan turns any overread into a report. */
+  body = malloct(size != 0 ? size : 1);
+  memcpy(body, data, size);
+
+  (void) hts_sitemap_scan(body, size, cap, &is_index, sm_count, &n);
+
+  freet(body);
+  return 0;
+}

html/cmdguide.html

@@ -163,8 +163,26 @@ <h3 id="scope">2. Scope: how far the crawl reaches</h3>
   <tr><td><tt>--near (-n)</tt></td><td>Also fetch non-HTML files "near" a followed link, such as an image linked from a page you kept but hosted elsewhere.</td></tr>
   <tr><td><tt>--ext-depth (-%e)</tt></td><td>How many levels of external links to follow once the crawl leaves your scope (default 0).</td></tr>
   <tr><td><tt>--test (-t)</tt></td><td>Also HEAD-test links that fall outside the scope, which are normally refused, without downloading them: a way to see what scope is excluding.</td></tr>
+  <tr><td><tt>--sitemap (-%m), --sitemap-url URL (-%mu)</tt></td><td>Also take start URLs from the site's sitemap, for pages nothing links to. Off by default.</td></tr>
 </table>
 
+<p>Link-following only finds what something links to. Anything a site publishes
+solely in its sitemap is invisible to HTTrack unless you ask for it.
+<tt>--sitemap</tt> reads the start host's <tt>robots.txt</tt> for
+<tt>Sitemap:</tt> lines and falls back to <tt>/sitemap.xml</tt>;
+<tt>--sitemap-url</tt> names one directly. Nested <tt>sitemapindex</tt> files
+and gzipped <tt>.xml.gz</tt> sitemaps are followed. The URLs found become start
+URLs with the full depth budget, but they still go through your filters and
+scope rules, so a sitemap cannot widen a crawl you deliberately narrowed. It is
+off by default because a sitemap can list thousands of pages nothing links
+to.</p>
+
+<p>One surprise worth knowing: a sitemap you name with <tt>--sitemap-url</tt>,
+and one the site itself declares in <tt>robots.txt</tt>, are fetched even when
+<tt>robots.txt</tt> disallows that path, because naming or declaring a sitemap
+is an invitation to read it. Only the guessed <tt>/sitemap.xml</tt> obeys a
+<tt>Disallow</tt>. The URLs listed inside are gated normally either way.</p>
+
 <p>The single most common surprise is "only the home page came down." That is
 usually not a scope option at all: it is an off-host redirect. A start URL of
 <tt>http://example.com/</tt> that redirects to <tt>https://www.example.com/</tt>

html/server/option8.html

@@ -108,6 +108,7 @@ <h2 align="center"><em>${LANG_O2} - ${LANG_IOPT8}</em></h2>
 <input type="hidden" name="keepqueryorder" value="">
 <input type="hidden" name="toler" value="">
 <input type="hidden" name="http10" value="">
+<input type="hidden" name="sitemap" value="">
 
 <input type="checkbox" name="cookies" ${checked:cookies}
 			title='${html:LANG_I1b}' onMouseOver="info('${html:LANG_I1b}'); return true" onMouseOut="info('&nbsp;'); return true"
@@ -143,6 +144,17 @@ <h2 align="center"><em>${LANG_O2} - ${LANG_IOPT8}</em></h2>
 </select>
 <br><br>
 
+<input type="checkbox" name="sitemap" ${checked:sitemap}
+			title='${html:LANG_SITEMAPTIP}' onMouseOver="info('${html:LANG_SITEMAPTIP}'); return true" onMouseOut="info('&nbsp;'); return true"
+> ${LANG_SITEMAP}
+<br><br>
+
+${LANG_SITEMAPURL}
+<input name="sitemapurl" value="${sitemapurl}" size="40"
+			title='${html:LANG_SITEMAPURLTIP}' onMouseOver="info('${html:LANG_SITEMAPURLTIP}'); return true" onMouseOut="info('&nbsp;'); return true"
+>
+<br><br>
+
 <input type="checkbox" name="updhack" ${checked:updhack}
 			title='${html:LANG_I1k}' onMouseOver="info('${html:LANG_I1k}'); return true" onMouseOut="info('&nbsp;'); return true"
 > ${LANG_I62b}

html/server/step2.html

@@ -141,6 +141,8 @@ <h2 align="center"><em>${fexist:index.html:LANG_G42}</em></h2>
 ${do:copy:KeepQueryOrder:keepqueryorder}
 ${do:copy:StripQuery:stripquery}
 ${do:copy:StoreAllInCache:cache2}
+${do:copy:Sitemap:sitemap}
+${do:copy:SitemapUrl:sitemapurl}
 ${do:copy:Warc:warc}
 ${do:copy:WarcFile:warcfile}
 ${do:copy:Changes:changes}

html/server/step4.html

@@ -188,6 +188,8 @@ <h2 align="center"><em>${LANG_J9}</em></h2>
 	${test:toler:--tolerant}
 	${test:http10:--http-10}
 	${test:cache2:--store-all-in-cache}
+	${test:sitemap:--sitemap}
+	${test:sitemapurl:--sitemap-url "}${html:sitemapurl}${test:sitemapurl:"}
 	${test:warc:--warc}
 	${test:warcfile:--warc-file "}${arg:warcfile}${test:warcfile:"}
 	${test:changes:--changes}
@@ -243,6 +245,8 @@ <h2 align="center"><em>${LANG_J9}</em></h2>
 KeepQueryOrder=${ztest:keepqueryorder:0:1}
 StripQuery=${stripquery}
 StoreAllInCache=${ztest:cache2:0:1}
+Sitemap=${ztest:sitemap:0:1}
+SitemapUrl=${sitemapurl}
 Warc=${ztest:warc:0:1}
 WarcFile=${warcfile}
 Changes=${ztest:changes:0:1}

lang.def

@@ -1054,3 +1054,11 @@ LANG_SINGLEFILEMAX
 Largest inlined asset (bytes):
 LANG_SINGLEFILEMAXTIP
 An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
+LANG_SITEMAP
+Seed the crawl from the site's sitemap
+LANG_SITEMAPTIP
+Read the site's sitemap (robots.txt Sitemap: lines, then /sitemap.xml) and add every URL it lists as a start URL.
+LANG_SITEMAPURL
+Sitemap address:
+LANG_SITEMAPURLTIP
+Address of a sitemap to read instead of probing the site; leave blank to probe robots.txt then /sitemap.xml.

lang/Bulgarian.txt

@@ -976,3 +976,11 @@ Largest inlined asset (bytes):
 Íàé-ãîëÿì âãðàäåí ðåñóðñ (áàéòîâå):
 An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
 Ðåñóðñ íàä òîçè ðàçìåð çàïàçâà îáèêíîâåíà âðúçêà; îñòàâåòå ïðàçíî çà ñòîéíîñòòà ïî ïîäðàçáèðàíå îò 10485760 áàéòà.
+Seed the crawl from the site's sitemap
+Çàïî÷âàíå íà îáõîæäàíåòî îò êàðòàòà íà ñàéòà
+Read the site's sitemap (robots.txt Sitemap: lines, then /sitemap.xml) and add every URL it lists as a start URL.
+Ïðî÷èòàíå íà êàðòàòà íà ñàéòà (ðåäîâåòå Sitemap: â robots.txt, ñëåä òîâà /sitemap.xml) è äîáàâÿíå íà âñåêè ïîñî÷åí URL àäðåñ êàòî íà÷àëåí.
+Sitemap address:
+Àäðåñ íà êàðòàòà íà ñàéòà:
+Address of a sitemap to read instead of probing the site; leave blank to probe robots.txt then /sitemap.xml.
+Àäðåñ íà êàðòà íà ñàéòà, êîÿòî äà áúäå ïðî÷åòåíà âìåñòî ñîíäèðàíå íà ñàéòà; îñòàâåòå ïðàçíî, çà äà ñå ïðîâåðè robots.txt, ñëåä òîâà /sitemap.xml.

lang/Castellano.txt

@@ -976,3 +976,11 @@ Largest inlined asset (bytes):
 Tamaño máximo del recurso incrustado (bytes):
 An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
 Un recurso mayor que este tamaño conserva un enlace normal; déjelo vacío para el valor predeterminado de 10485760 bytes.
+Seed the crawl from the site's sitemap
+Iniciar el rastreo desde el mapa del sitio
+Read the site's sitemap (robots.txt Sitemap: lines, then /sitemap.xml) and add every URL it lists as a start URL.
+Leer el mapa del sitio (líneas Sitemap: de robots.txt, luego /sitemap.xml) y añadir como dirección inicial cada URL que incluya.
+Sitemap address:
+Dirección del mapa del sitio:
+Address of a sitemap to read instead of probing the site; leave blank to probe robots.txt then /sitemap.xml.
+Dirección de un mapa del sitio que leer en lugar de sondear el sitio; déjelo vacío para sondear robots.txt y luego /sitemap.xml.

lang/Cesky.txt

@@ -976,3 +976,11 @@ Largest inlined asset (bytes):
 Nejvìtší vložený zdroj (bajty):
 An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
 Zdroj vìtší než tato velikost si ponechá bìžný odkaz; ponechte prázdné pro výchozí hodnotu 10485760 bajtù.
+Seed the crawl from the site's sitemap
+Zahájit procházení z mapy webu
+Read the site's sitemap (robots.txt Sitemap: lines, then /sitemap.xml) and add every URL it lists as a start URL.
+Naèíst mapu webu (øádky Sitemap: v souboru robots.txt, poté /sitemap.xml) a pøidat každou uvedenou adresu URL jako výchozí.
+Sitemap address:
+Adresa mapy webu:
+Address of a sitemap to read instead of probing the site; leave blank to probe robots.txt then /sitemap.xml.
+Adresa mapy webu, která se má naèíst místo zjišování na webu; ponechte prázdné pro zjištìní z robots.txt a poté /sitemap.xml.

lang/Chinese-BIG5.txt

@@ -976,3 +976,11 @@ Largest inlined asset (bytes):
 ¤º´O¸ê·½¤j¤p¤W­­¡]¦ì¤¸²Õ¡^¡G
 An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
 ¶W¹L¦¹¤j¤pªº¸ê·½·|«O¯d¤@¯ë³sµ²¡F¯dªÅ«h¨Ï¥Î¹w³]ªº 10485760 ¦ì¤¸²Õ¡C
+Seed the crawl from the site's sitemap
+±qºô¯¸ªº Sitemap ¶}©lÂ^¨ú
+Read the site's sitemap (robots.txt Sitemap: lines, then /sitemap.xml) and add every URL it lists as a start URL.
+Ū¨úºô¯¸ªº Sitemap¡]robots.txt ¤¤ªº Sitemap: ¦æ¡AµM«á /sitemap.xml¡^¡A¨Ã±N¨ä¤¤¦C¥Xªº¨C­Óºô§}¥[¤J¬°°_©lºô§}¡C
+Sitemap address:
+Sitemap ¦ì§}¡G
+Address of a sitemap to read instead of probing the site; leave blank to probe robots.txt then /sitemap.xml.
+­nŪ¨úªº Sitemap ¦ì§}¡A¥Î¨Ó¨ú¥N¦Û°Ê±´´ú¡F¯dªÅ«h¥ý±´´ú robots.txt ¦A±´´ú /sitemap.xml¡C

lang/Chinese-Simplified.txt

@@ -976,3 +976,11 @@ Largest inlined asset (bytes):
 ÄÚǶ×ÊÔ´´óСÉÏÏÞ£¨×Ö½Ú£©£º
 An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
 ³¬¹ý´Ë´óСµÄ×ÊÔ´»á±£ÁôÆÕͨÁ´½Ó£»Áô¿ÕÔòʹÓÃĬÈ쵀 10485760 ×Ö½Ú¡£
+Seed the crawl from the site's sitemap
+´ÓÍøÕ¾µÄ Sitemap ¿ªÊ¼×¥È¡
+Read the site's sitemap (robots.txt Sitemap: lines, then /sitemap.xml) and add every URL it lists as a start URL.
+¶ÁÈ¡ÍøÕ¾µÄ Sitemap£¨robots.txt ÖÐµÄ Sitemap: ÐУ¬È»ºó /sitemap.xml£©£¬²¢½«ÆäÖÐÁгöµÄÿ¸öÍøÖ·Ìí¼ÓΪÆðʼÍøÖ·¡£
+Sitemap address:
+Sitemap µØÖ·£º
+Address of a sitemap to read instead of probing the site; leave blank to probe robots.txt then /sitemap.xml.
+Òª¶ÁÈ¡µÄ Sitemap µØÖ·£¬ÓÃÀ´´úÌæ×Ô¶¯Ì½²â£»Áô¿ÕÔòÏÈ̽²â robots.txt ÔÙ̽²â /sitemap.xml¡£

lang/Croatian.txt

@@ -978,3 +978,11 @@ Largest inlined asset (bytes):
 Najveæi ugraðeni resurs (bajtovi):
 An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
 Resurs veæi od ove velièine zadr¾ava obiènu poveznicu; ostavite prazno za zadanih 10485760 bajtova.
+Seed the crawl from the site's sitemap
+Pokreni pretra¾ivanje iz karte web-mjesta
+Read the site's sitemap (robots.txt Sitemap: lines, then /sitemap.xml) and add every URL it lists as a start URL.
+Proèitaj kartu web-mjesta (retke Sitemap: iz robots.txt, zatim /sitemap.xml) i dodaj svaki navedeni URL kao poèetnu adresu.
+Sitemap address:
+Adresa karte web-mjesta:
+Address of a sitemap to read instead of probing the site; leave blank to probe robots.txt then /sitemap.xml.
+Adresa karte web-mjesta koju treba proèitati umjesto ispitivanja web-mjesta; ostavite prazno za ispitivanje robots.txt pa /sitemap.xml.

lang/Dansk.txt

@@ -1024,3 +1024,11 @@ Largest inlined asset (bytes):
 Største indlejrede ressource (byte):
 An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
 En ressource over denne størrelse beholder et almindeligt link; lad feltet stå tomt for standardværdien på 10485760 byte.
+Seed the crawl from the site's sitemap
+Start gennemgangen fra webstedets sitemap
+Read the site's sitemap (robots.txt Sitemap: lines, then /sitemap.xml) and add every URL it lists as a start URL.
+Læs webstedets sitemap (Sitemap:-linjer i robots.txt, derefter /sitemap.xml) og tilføj hver angivet URL som startadresse.
+Sitemap address:
+Sitemap-adresse:
+Address of a sitemap to read instead of probing the site; leave blank to probe robots.txt then /sitemap.xml.
+Adressen på et sitemap, der skal læses i stedet for at undersøge webstedet; lad feltet stå tomt for at undersøge robots.txt og derefter /sitemap.xml.

lang/Deutsch.txt

@@ -976,3 +976,11 @@ Largest inlined asset (bytes):
 Größte eingebettete Ressource (Bytes):
 An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
 Eine Ressource über dieser Größe behält einen gewöhnlichen Link; leer lassen für den Standardwert von 10485760 Bytes.
+Seed the crawl from the site's sitemap
+Erfassung mit der Sitemap der Website beginnen
+Read the site's sitemap (robots.txt Sitemap: lines, then /sitemap.xml) and add every URL it lists as a start URL.
+Die Sitemap der Website lesen (Sitemap:-Zeilen in robots.txt, dann /sitemap.xml) und jede dort aufgeführte URL als Startadresse hinzufügen.
+Sitemap address:
+Sitemap-Adresse:
+Address of a sitemap to read instead of probing the site; leave blank to probe robots.txt then /sitemap.xml.
+Adresse einer Sitemap, die anstelle der Suche auf der Website gelesen wird; leer lassen, um robots.txt und dann /sitemap.xml zu prüfen.

lang/Eesti.txt

@@ -976,3 +976,11 @@ Largest inlined asset (bytes):
 Suurim manustatud ressurss (baiti):
 An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
 Sellest suurem ressurss säilitab tavalise lingi; jäta tühjaks vaikeväärtuse 10485760 baiti jaoks.
+Seed the crawl from the site's sitemap
+Alusta kogumist saidi saidikaardist
+Read the site's sitemap (robots.txt Sitemap: lines, then /sitemap.xml) and add every URL it lists as a start URL.
+Loe saidi saidikaarti (robots.txt-i Sitemap:-read, seejärel /sitemap.xml) ja lisa iga seal loetletud URL alguslingina.
+Sitemap address:
+Saidikaardi aadress:
+Address of a sitemap to read instead of probing the site; leave blank to probe robots.txt then /sitemap.xml.
+Saidikaardi aadress, mida lugeda saidi sondeerimise asemel; jäta tühjaks, et kontrollida robots.txt-i ja seejärel /sitemap.xml-i.

lang/English.txt

@@ -1024,3 +1024,11 @@ Largest inlined asset (bytes):
 Largest inlined asset (bytes):
 An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
 An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
+Seed the crawl from the site's sitemap
+Seed the crawl from the site's sitemap
+Read the site's sitemap (robots.txt Sitemap: lines, then /sitemap.xml) and add every URL it lists as a start URL.
+Read the site's sitemap (robots.txt Sitemap: lines, then /sitemap.xml) and add every URL it lists as a start URL.
+Sitemap address:
+Sitemap address:
+Address of a sitemap to read instead of probing the site; leave blank to probe robots.txt then /sitemap.xml.
+Address of a sitemap to read instead of probing the site; leave blank to probe robots.txt then /sitemap.xml.

lang/Finnish.txt

@@ -978,3 +978,11 @@ Largest inlined asset (bytes):
 Suurin upotettu resurssi (tavua):
 An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
 Tätä suurempi resurssi säilyttää tavallisen linkin; jätä tyhjäksi, jolloin käytetään oletusarvoa 10485760 tavua.
+Seed the crawl from the site's sitemap
+Aloita haku sivuston sivukartasta
+Read the site's sitemap (robots.txt Sitemap: lines, then /sitemap.xml) and add every URL it lists as a start URL.
+Lue sivuston sivukartta (robots.txt-tiedoston Sitemap:-rivit, sitten /sitemap.xml) ja lisää jokainen siinä lueteltu URL-osoite aloitusosoitteeksi.
+Sitemap address:
+Sivukartan osoite:
+Address of a sitemap to read instead of probing the site; leave blank to probe robots.txt then /sitemap.xml.
+Luettavan sivukartan osoite sivuston luotaamisen sijaan; jätä tyhjäksi, jolloin tarkistetaan robots.txt ja sitten /sitemap.xml.

lang/Francais.txt

@@ -1024,3 +1024,11 @@ Largest inlined asset (bytes):
 Taille maximale d'une ressource intégrée (octets) :
 An asset above this size keeps an ordinary link; leave blank for the 10485760-byte default.
 Au-delà de cette taille, la ressource reste un lien ordinaire ; laissez vide pour la valeur par défaut de 10485760 octets.
+Seed the crawl from the site's sitemap
+Partir du plan de site (sitemap)
+Read the site's sitemap (robots.txt Sitemap: lines, then /sitemap.xml) and add every URL it lists as a start URL.
+Lire le plan de site (lignes Sitemap: de robots.txt, puis /sitemap.xml) et ajouter chaque URL listée comme adresse de départ.
+Sitemap address:
+Adresse du plan de site :
+Address of a sitemap to read instead of probing the site; leave blank to probe robots.txt then /sitemap.xml.
+Adresse d'un plan de site à lire au lieu de sonder le site ; laissez vide pour sonder robots.txt puis /sitemap.xml.