diff --git a/app/src/main/java/com/httrack/android/OptionsActivity.java b/app/src/main/java/com/httrack/android/OptionsActivity.java index f4b53a7..0b4fe9c 100755 --- a/app/src/main/java/com/httrack/android/OptionsActivity.java +++ b/app/src/main/java/com/httrack/android/OptionsActivity.java @@ -170,8 +170,8 @@ public static class LinksTab extends Tab { @ActivityId(R.layout.activity_options_build) @Fields({ R.id.checkDosNames, R.id.checkIso9660, R.id.checkNoErrorPages, R.id.checkNoExternalPages, R.id.checkHidePasswords, - R.id.checkHideQueryStrings, R.id.checkDoNotPurge, R.id.radioBuild, - R.id.editCustomBuild }) + R.id.checkHideQueryStrings, R.id.checkDoNotPurge, R.id.checkSingleFile, + R.id.radioBuild, R.id.editCustomBuild }) public static class BuildTab extends Tab { } @@ -186,8 +186,8 @@ public static class BrowserId extends Tab { @ActivityId(R.layout.activity_options_spider) @Fields({ R.id.checkAcceptCookies, R.id.editCookiesFile, R.id.radioCheckDocumentType, R.id.checkParseJavaFiles, R.id.radioSpider, - R.id.checkUpdateHacks, R.id.checkUrlHacks, R.id.checkTolerentRequests, - R.id.checkForceHttp10 }) + R.id.checkSitemap, R.id.checkUpdateHacks, R.id.checkUrlHacks, + R.id.checkTolerentRequests, R.id.checkForceHttp10 }) public static class Spider extends Tab { } @@ -202,8 +202,8 @@ public static class Proxy extends Tab { @ActivityId(R.layout.activity_options_logindexcache) @Fields({ R.id.checkStoreAllFilesInCache, R.id.checkDoNotRedownloadLocallErasedFiles, R.id.checkWarc, - R.id.checkCreateLogFiles, R.id.radioVerbosity, R.id.checkUseIndex, - R.id.checkUseWordIndex, R.id.checkUseMailIndex }) + R.id.checkChanges, R.id.checkCreateLogFiles, R.id.radioVerbosity, + R.id.checkUseIndex, R.id.checkUseWordIndex, R.id.checkUseMailIndex }) public static class LogIndexCache extends Tab { } diff --git a/app/src/main/java/com/httrack/android/OptionsMapper.java b/app/src/main/java/com/httrack/android/OptionsMapper.java index 1a13e78..c914f6b 100755 --- a/app/src/main/java/com/httrack/android/OptionsMapper.java +++ b/app/src/main/java/com/httrack/android/OptionsMapper.java @@ -104,6 +104,7 @@ public class OptionsMapper { new Pair(R.id.checkHidePasswords, "NoPwdInPages"), new Pair(R.id.checkHideQueryStrings, "NoQueryStrings"), new Pair(R.id.checkDoNotPurge, "NoPurgeOldFiles"), + new Pair(R.id.checkSingleFile, "SingleFile"), new Pair(R.id.checkWarc, "Warc"), new Pair(R.id.radioBuild, "Build"), new Pair(R.id.editCustomBuild, "BuildString"), @@ -121,6 +122,7 @@ public class OptionsMapper { new Pair(R.id.radioCheckDocumentType, "CheckType"), new Pair(R.id.checkParseJavaFiles, "ParseJava"), new Pair(R.id.radioSpider, "FollowRobotsTxt"), + new Pair(R.id.checkSitemap, "Sitemap"), new Pair(R.id.checkUpdateHacks, "UpdateHack"), new Pair(R.id.checkUrlHacks, "URLHack"), new Pair(R.id.checkTolerentRequests, "TolerantRequests"), @@ -137,6 +139,7 @@ public class OptionsMapper { "StoreAllInCache"), new Pair(R.id.checkDoNotRedownloadLocallErasedFiles, "NoRecatch"), + new Pair(R.id.checkChanges, "Changes"), new Pair(R.id.checkCreateLogFiles, "Log"), /* FIXME with Log */ new Pair(R.id.radioVerbosity, "LogType"), @@ -200,6 +203,9 @@ public class OptionsMapper { new Pair("NoQueryStrings", "0"), new Pair("NoPurgeOldFiles", "0"), new Pair("Warc", "0"), + new Pair("Sitemap", "0"), + new Pair("SingleFile", "0"), + new Pair("Changes", "0"), new Pair("Cookies", "1"), new Pair("CheckType", "1"), new Pair("ParseJava", "1"), @@ -290,6 +296,10 @@ public class OptionsMapper { new Pair("NoPurgeOldFiles", new SimpleOptionFlag( "X0")), new Pair("Warc", new SimpleOptionFlag("%r")), + new Pair("Sitemap", new LongOptionFlag("--sitemap")), + new Pair("SingleFile", new LongOptionFlag( + "--single-file")), + new Pair("Changes", new LongOptionFlag("--changes")), new Pair("Build", buildHandler.getTypeMapper()), new Pair("BuildString", buildHandler.getCustomMapper()), @@ -1366,6 +1376,27 @@ public void emit(final StringBuilder flags, final List commandline, } } + /** + * Required for a short flag with a two-letter variant (-%m / -%mu, -%Z / + * -%Zs): packed in the compacted string, the next flag's first letter would + * be misread as that variant and swallow an argument. + */ + public static class LongOptionFlag implements OptionMapper { + protected final String option; + + public LongOptionFlag(final String option) { + this.option = option; + } + + @Override + public void emit(final StringBuilder flags, final List commandline, + final String value) { + if ("1".equals(value)) { + commandline.add(option); + } + } + } + /** * Option without any value. */ diff --git a/app/src/main/jni/Android.mk b/app/src/main/jni/Android.mk index 06400da..c138de6 100755 --- a/app/src/main/jni/Android.mk +++ b/app/src/main/jni/Android.mk @@ -76,6 +76,8 @@ LOCAL_SRC_FILES := httrack/src/htscore.c httrack/src/htsparse.c \ httrack/src/htscache_selftest.c httrack/src/htsdns_selftest.c \ httrack/src/htscodec.c httrack/src/htsproxy.c \ httrack/src/htsurlport.c httrack/src/htswarc.c \ + httrack/src/htssitemap.c httrack/src/htssinglefile.c \ + httrack/src/htschanges.c httrack/src/htscmdline.c \ httrack/src/minizip/ioapi.c \ httrack/src/minizip/mztools.c httrack/src/minizip/unzip.c \ httrack/src/minizip/zip.c diff --git a/app/src/main/jni/httrack b/app/src/main/jni/httrack index 3265a5d..9e29c1e 160000 --- a/app/src/main/jni/httrack +++ b/app/src/main/jni/httrack @@ -1 +1 @@ -Subproject commit 3265a5de00d844828e484a467e7a9ceb61f73544 +Subproject commit 9e29c1e159c75142d6dcc554d8aa009bf15d1ba5 diff --git a/app/src/main/res/layout/activity_options_build.xml b/app/src/main/res/layout/activity_options_build.xml index 81bc5b4..9a284b1 100644 --- a/app/src/main/res/layout/activity_options_build.xml +++ b/app/src/main/res/layout/activity_options_build.xml @@ -56,6 +56,12 @@ android:layout_height="wrap_content" android:text="@string/do_not_purge_old_files" /> + + + + + + Hide passwords Hide query strings Do not purge old files + Inline assets as data: URIs (self-contained pages) Write WARC archive + Report what changed since the previous mirror + Seed the crawl from the site\'s sitemap Max simultaneous connections Persistent connections (Keep-Alive) File timeout diff --git a/app/src/test/java/com/httrack/android/OptionsEmissionTest.java b/app/src/test/java/com/httrack/android/OptionsEmissionTest.java index 0fade0b..cd2dee5 100644 --- a/app/src/test/java/com/httrack/android/OptionsEmissionTest.java +++ b/app/src/test/java/com/httrack/android/OptionsEmissionTest.java @@ -5,6 +5,7 @@ import static org.junit.Assert.assertTrue; import com.httrack.android.OptionsMapper.ArgumentOption; +import com.httrack.android.OptionsMapper.LongOptionFlag; import com.httrack.android.OptionsMapper.OptionMapper; import com.httrack.android.OptionsMapper.ProxyHandler; import com.httrack.android.OptionsMapper.SimpleOptionFlag; @@ -98,4 +99,42 @@ public void warcToggleEmitsNothingWhenUnchecked() { new SimpleOptionFlag("%r").emit(flags, new ArrayList(), "0"); assertFalse(flags.toString().contains("%r")); } + + /* sitemap/single-file/changes: own token, and never in the packed string. */ + @Test + public void longOptionEmitsItsOwnTokenWhenChecked() { + for (final String option : new String[] { "--sitemap", "--single-file", + "--changes" }) { + /* seeded as buildCommandline() does, so a guarded append cannot hide */ + final StringBuilder flags = new StringBuilder("-"); + final List cmd = new ArrayList(); + new LongOptionFlag(option).emit(flags, cmd, "1"); + assertEquals(1, cmd.size()); + assertEquals(option, cmd.get(0)); + assertEquals("-", flags.toString()); + } + } + + @Test + public void longOptionStaysSilentWhenUncheckedOrUnset() { + for (final String value : new String[] { "0", "", null }) { + final StringBuilder flags = new StringBuilder("-"); + final List cmd = new ArrayList(); + new LongOptionFlag("--single-file").emit(flags, cmd, value); + assertTrue(cmd.isEmpty()); + assertEquals("-", flags.toString()); + } + } + + /* the mappers are static singletons: emitting again must not go quiet. */ + @Test + public void longOptionEmitsOnEveryBuild() { + final OptionMapper mapper = new LongOptionFlag("--changes"); + final List first = new ArrayList(); + final List second = new ArrayList(); + mapper.emit(new StringBuilder("-"), first, "1"); + mapper.emit(new StringBuilder("-"), second, "1"); + assertEquals(first, second); + assertEquals("--changes", second.get(0)); + } }