From a6cf940d376f0950fffa9aab0966c1c7480ba849 Mon Sep 17 00:00:00 2001 From: Xavier Roche Date: Mon, 27 Jul 2026 08:12:36 +0200 Subject: [PATCH 1/2] Surface the sitemap, single-file and change-report engine options MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The engine gained --sitemap, --single-file and --changes; bump the vendored snapshot to core master and put a checkbox on each of the tabs the reference GUI uses (Spider, Build, Log/Index/Cache). The three go out as their own argv token rather than packed into the compacted -%… string: -%m and -%Z each have a two-letter variant (-%mu, -%Zs), and the engine scans that string one character at a time, so a neighbouring flag's first letter would be read as the variant and swallow an argument. Android.mk lists engine sources explicitly, so the bump also has to register htssitemap.c, htssinglefile.c, htschanges.c and htscmdline.c. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_0163vUGoRjjxBmt67HhHQ7ps Signed-off-by: Xavier Roche --- .../com/httrack/android/OptionsActivity.java | 12 +++---- .../com/httrack/android/OptionsMapper.java | 33 +++++++++++++++++++ app/src/main/jni/Android.mk | 2 ++ app/src/main/jni/httrack | 2 +- .../res/layout/activity_options_build.xml | 6 ++++ .../layout/activity_options_logindexcache.xml | 6 ++++ .../res/layout/activity_options_spider.xml | 7 ++++ app/src/main/res/values/strings.xml | 3 ++ .../httrack/android/OptionsEmissionTest.java | 23 +++++++++++++ 9 files changed, 87 insertions(+), 7 deletions(-) diff --git a/app/src/main/java/com/httrack/android/OptionsActivity.java b/app/src/main/java/com/httrack/android/OptionsActivity.java index f4b53a7..0b4fe9c 100755 --- a/app/src/main/java/com/httrack/android/OptionsActivity.java +++ b/app/src/main/java/com/httrack/android/OptionsActivity.java @@ -170,8 +170,8 @@ public static class LinksTab extends Tab { @ActivityId(R.layout.activity_options_build) @Fields({ R.id.checkDosNames, R.id.checkIso9660, R.id.checkNoErrorPages, R.id.checkNoExternalPages, R.id.checkHidePasswords, - R.id.checkHideQueryStrings, R.id.checkDoNotPurge, R.id.radioBuild, - R.id.editCustomBuild }) + R.id.checkHideQueryStrings, R.id.checkDoNotPurge, R.id.checkSingleFile, + R.id.radioBuild, R.id.editCustomBuild }) public static class BuildTab extends Tab { } @@ -186,8 +186,8 @@ public static class BrowserId extends Tab { @ActivityId(R.layout.activity_options_spider) @Fields({ R.id.checkAcceptCookies, R.id.editCookiesFile, R.id.radioCheckDocumentType, R.id.checkParseJavaFiles, R.id.radioSpider, - R.id.checkUpdateHacks, R.id.checkUrlHacks, R.id.checkTolerentRequests, - R.id.checkForceHttp10 }) + R.id.checkSitemap, R.id.checkUpdateHacks, R.id.checkUrlHacks, + R.id.checkTolerentRequests, R.id.checkForceHttp10 }) public static class Spider extends Tab { } @@ -202,8 +202,8 @@ public static class Proxy extends Tab { @ActivityId(R.layout.activity_options_logindexcache) @Fields({ R.id.checkStoreAllFilesInCache, R.id.checkDoNotRedownloadLocallErasedFiles, R.id.checkWarc, - R.id.checkCreateLogFiles, R.id.radioVerbosity, R.id.checkUseIndex, - R.id.checkUseWordIndex, R.id.checkUseMailIndex }) + R.id.checkChanges, R.id.checkCreateLogFiles, R.id.radioVerbosity, + R.id.checkUseIndex, R.id.checkUseWordIndex, R.id.checkUseMailIndex }) public static class LogIndexCache extends Tab { } diff --git a/app/src/main/java/com/httrack/android/OptionsMapper.java b/app/src/main/java/com/httrack/android/OptionsMapper.java index 1a13e78..98993da 100755 --- a/app/src/main/java/com/httrack/android/OptionsMapper.java +++ b/app/src/main/java/com/httrack/android/OptionsMapper.java @@ -104,6 +104,7 @@ public class OptionsMapper { new Pair(R.id.checkHidePasswords, "NoPwdInPages"), new Pair(R.id.checkHideQueryStrings, "NoQueryStrings"), new Pair(R.id.checkDoNotPurge, "NoPurgeOldFiles"), + new Pair(R.id.checkSingleFile, "SingleFile"), new Pair(R.id.checkWarc, "Warc"), new Pair(R.id.radioBuild, "Build"), new Pair(R.id.editCustomBuild, "BuildString"), @@ -121,6 +122,7 @@ public class OptionsMapper { new Pair(R.id.radioCheckDocumentType, "CheckType"), new Pair(R.id.checkParseJavaFiles, "ParseJava"), new Pair(R.id.radioSpider, "FollowRobotsTxt"), + new Pair(R.id.checkSitemap, "Sitemap"), new Pair(R.id.checkUpdateHacks, "UpdateHack"), new Pair(R.id.checkUrlHacks, "URLHack"), new Pair(R.id.checkTolerentRequests, "TolerantRequests"), @@ -137,6 +139,7 @@ public class OptionsMapper { "StoreAllInCache"), new Pair(R.id.checkDoNotRedownloadLocallErasedFiles, "NoRecatch"), + new Pair(R.id.checkChanges, "Changes"), new Pair(R.id.checkCreateLogFiles, "Log"), /* FIXME with Log */ new Pair(R.id.radioVerbosity, "LogType"), @@ -200,6 +203,9 @@ public class OptionsMapper { new Pair("NoQueryStrings", "0"), new Pair("NoPurgeOldFiles", "0"), new Pair("Warc", "0"), + new Pair("Sitemap", "0"), + new Pair("SingleFile", "0"), + new Pair("Changes", "0"), new Pair("Cookies", "1"), new Pair("CheckType", "1"), new Pair("ParseJava", "1"), @@ -290,6 +296,10 @@ public class OptionsMapper { new Pair("NoPurgeOldFiles", new SimpleOptionFlag( "X0")), new Pair("Warc", new SimpleOptionFlag("%r")), + new Pair("Sitemap", new LongOptionFlag("--sitemap")), + new Pair("SingleFile", new LongOptionFlag( + "--single-file")), + new Pair("Changes", new LongOptionFlag("--changes")), new Pair("Build", buildHandler.getTypeMapper()), new Pair("BuildString", buildHandler.getCustomMapper()), @@ -1366,6 +1376,29 @@ public void emit(final StringBuilder flags, final List commandline, } } + /** + * Boolean option emitted as its own argument, in long form. + * + * Mandatory when the short form has a two-letter variant (-%m / -%mu, -%Z / + * -%Zs): packed in the compacted flag string, the next flag's first letter + * would be read as that variant and swallow an argument. + */ + public static class LongOptionFlag implements OptionMapper { + protected final String option; + + public LongOptionFlag(final String option) { + this.option = option; + } + + @Override + public void emit(final StringBuilder flags, final List commandline, + final String value) { + if ("1".equals(value)) { + commandline.add(option); + } + } + } + /** * Option without any value. */ diff --git a/app/src/main/jni/Android.mk b/app/src/main/jni/Android.mk index 06400da..c138de6 100755 --- a/app/src/main/jni/Android.mk +++ b/app/src/main/jni/Android.mk @@ -76,6 +76,8 @@ LOCAL_SRC_FILES := httrack/src/htscore.c httrack/src/htsparse.c \ httrack/src/htscache_selftest.c httrack/src/htsdns_selftest.c \ httrack/src/htscodec.c httrack/src/htsproxy.c \ httrack/src/htsurlport.c httrack/src/htswarc.c \ + httrack/src/htssitemap.c httrack/src/htssinglefile.c \ + httrack/src/htschanges.c httrack/src/htscmdline.c \ httrack/src/minizip/ioapi.c \ httrack/src/minizip/mztools.c httrack/src/minizip/unzip.c \ httrack/src/minizip/zip.c diff --git a/app/src/main/jni/httrack b/app/src/main/jni/httrack index 3265a5d..9e29c1e 160000 --- a/app/src/main/jni/httrack +++ b/app/src/main/jni/httrack @@ -1 +1 @@ -Subproject commit 3265a5de00d844828e484a467e7a9ceb61f73544 +Subproject commit 9e29c1e159c75142d6dcc554d8aa009bf15d1ba5 diff --git a/app/src/main/res/layout/activity_options_build.xml b/app/src/main/res/layout/activity_options_build.xml index 81bc5b4..9a284b1 100644 --- a/app/src/main/res/layout/activity_options_build.xml +++ b/app/src/main/res/layout/activity_options_build.xml @@ -56,6 +56,12 @@ android:layout_height="wrap_content" android:text="@string/do_not_purge_old_files" /> + + + + + + Hide passwords Hide query strings Do not purge old files + Inline assets as data: URIs (self-contained pages) Write WARC archive + Report what changed since the previous mirror + Seed the crawl from the site\'s sitemap Max simultaneous connections Persistent connections (Keep-Alive) File timeout diff --git a/app/src/test/java/com/httrack/android/OptionsEmissionTest.java b/app/src/test/java/com/httrack/android/OptionsEmissionTest.java index 0fade0b..0e61fb4 100644 --- a/app/src/test/java/com/httrack/android/OptionsEmissionTest.java +++ b/app/src/test/java/com/httrack/android/OptionsEmissionTest.java @@ -5,6 +5,7 @@ import static org.junit.Assert.assertTrue; import com.httrack.android.OptionsMapper.ArgumentOption; +import com.httrack.android.OptionsMapper.LongOptionFlag; import com.httrack.android.OptionsMapper.OptionMapper; import com.httrack.android.OptionsMapper.ProxyHandler; import com.httrack.android.OptionsMapper.SimpleOptionFlag; @@ -98,4 +99,26 @@ public void warcToggleEmitsNothingWhenUnchecked() { new SimpleOptionFlag("%r").emit(flags, new ArrayList(), "0"); assertFalse(flags.toString().contains("%r")); } + + /* sitemap/single-file/changes: own token, and never in the packed string. */ + @Test + public void longOptionEmitsItsOwnTokenWhenChecked() { + final StringBuilder flags = new StringBuilder(); + final List cmd = new ArrayList(); + new LongOptionFlag("--sitemap").emit(flags, cmd, "1"); + assertEquals(1, cmd.size()); + assertEquals("--sitemap", cmd.get(0)); + assertEquals("", flags.toString()); + } + + @Test + public void longOptionStaysSilentWhenUncheckedOrUnset() { + for (final String value : new String[] { "0", "", null }) { + final StringBuilder flags = new StringBuilder(); + final List cmd = new ArrayList(); + new LongOptionFlag("--single-file").emit(flags, cmd, value); + assertTrue(cmd.isEmpty()); + assertEquals("", flags.toString()); + } + } } From aaae592c20c35255e4ed8ab338991470b2dc289c Mon Sep 17 00:00:00 2001 From: Xavier Roche Date: Mon, 27 Jul 2026 08:20:50 +0200 Subject: [PATCH 2/2] review: make the emitter tests discriminate, trim the mapper comment The pair of tests passed against three broken emitters: one ignoring its constructor argument, one also packing into the flag string (guarded on a non-empty buffer, which the empty test fixture never tripped), and one emitting only on the first call. Assert every token, seed the buffer the way buildCommandline() does, and emit twice, since the mappers are static singletons. All three mutations now fail the suite. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_0163vUGoRjjxBmt67HhHQ7ps Signed-off-by: Xavier Roche --- .../com/httrack/android/OptionsMapper.java | 8 ++--- .../httrack/android/OptionsEmissionTest.java | 32 ++++++++++++++----- 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/app/src/main/java/com/httrack/android/OptionsMapper.java b/app/src/main/java/com/httrack/android/OptionsMapper.java index 98993da..c914f6b 100755 --- a/app/src/main/java/com/httrack/android/OptionsMapper.java +++ b/app/src/main/java/com/httrack/android/OptionsMapper.java @@ -1377,11 +1377,9 @@ public void emit(final StringBuilder flags, final List commandline, } /** - * Boolean option emitted as its own argument, in long form. - * - * Mandatory when the short form has a two-letter variant (-%m / -%mu, -%Z / - * -%Zs): packed in the compacted flag string, the next flag's first letter - * would be read as that variant and swallow an argument. + * Required for a short flag with a two-letter variant (-%m / -%mu, -%Z / + * -%Zs): packed in the compacted string, the next flag's first letter would + * be misread as that variant and swallow an argument. */ public static class LongOptionFlag implements OptionMapper { protected final String option; diff --git a/app/src/test/java/com/httrack/android/OptionsEmissionTest.java b/app/src/test/java/com/httrack/android/OptionsEmissionTest.java index 0e61fb4..cd2dee5 100644 --- a/app/src/test/java/com/httrack/android/OptionsEmissionTest.java +++ b/app/src/test/java/com/httrack/android/OptionsEmissionTest.java @@ -103,22 +103,38 @@ public void warcToggleEmitsNothingWhenUnchecked() { /* sitemap/single-file/changes: own token, and never in the packed string. */ @Test public void longOptionEmitsItsOwnTokenWhenChecked() { - final StringBuilder flags = new StringBuilder(); - final List cmd = new ArrayList(); - new LongOptionFlag("--sitemap").emit(flags, cmd, "1"); - assertEquals(1, cmd.size()); - assertEquals("--sitemap", cmd.get(0)); - assertEquals("", flags.toString()); + for (final String option : new String[] { "--sitemap", "--single-file", + "--changes" }) { + /* seeded as buildCommandline() does, so a guarded append cannot hide */ + final StringBuilder flags = new StringBuilder("-"); + final List cmd = new ArrayList(); + new LongOptionFlag(option).emit(flags, cmd, "1"); + assertEquals(1, cmd.size()); + assertEquals(option, cmd.get(0)); + assertEquals("-", flags.toString()); + } } @Test public void longOptionStaysSilentWhenUncheckedOrUnset() { for (final String value : new String[] { "0", "", null }) { - final StringBuilder flags = new StringBuilder(); + final StringBuilder flags = new StringBuilder("-"); final List cmd = new ArrayList(); new LongOptionFlag("--single-file").emit(flags, cmd, value); assertTrue(cmd.isEmpty()); - assertEquals("", flags.toString()); + assertEquals("-", flags.toString()); } } + + /* the mappers are static singletons: emitting again must not go quiet. */ + @Test + public void longOptionEmitsOnEveryBuild() { + final OptionMapper mapper = new LongOptionFlag("--changes"); + final List first = new ArrayList(); + final List second = new ArrayList(); + mapper.emit(new StringBuilder("-"), first, "1"); + mapper.emit(new StringBuilder("-"), second, "1"); + assertEquals(first, second); + assertEquals("--changes", second.get(0)); + } }