Surface the sitemap, single-file and change-report options in the GUI

app/src/main/java/com/httrack/android/OptionsActivity.java

@@ -170,8 +170,8 @@ public static class LinksTab extends Tab {
   @ActivityId(R.layout.activity_options_build)
   @Fields({ R.id.checkDosNames, R.id.checkIso9660, R.id.checkNoErrorPages,
       R.id.checkNoExternalPages, R.id.checkHidePasswords,
-      R.id.checkHideQueryStrings, R.id.checkDoNotPurge, R.id.radioBuild,
-      R.id.editCustomBuild })
+      R.id.checkHideQueryStrings, R.id.checkDoNotPurge, R.id.checkSingleFile,
+      R.id.radioBuild, R.id.editCustomBuild })
   public static class BuildTab extends Tab {
   }
 
@@ -186,8 +186,8 @@ public static class BrowserId extends Tab {
   @ActivityId(R.layout.activity_options_spider)
   @Fields({ R.id.checkAcceptCookies, R.id.editCookiesFile,
       R.id.radioCheckDocumentType, R.id.checkParseJavaFiles, R.id.radioSpider,
-      R.id.checkUpdateHacks, R.id.checkUrlHacks, R.id.checkTolerentRequests,
-      R.id.checkForceHttp10 })
+      R.id.checkSitemap, R.id.checkUpdateHacks, R.id.checkUrlHacks,
+      R.id.checkTolerentRequests, R.id.checkForceHttp10 })
   public static class Spider extends Tab {
   }
 
@@ -202,8 +202,8 @@ public static class Proxy extends Tab {
   @ActivityId(R.layout.activity_options_logindexcache)
   @Fields({ R.id.checkStoreAllFilesInCache,
       R.id.checkDoNotRedownloadLocallErasedFiles, R.id.checkWarc,
-      R.id.checkCreateLogFiles, R.id.radioVerbosity, R.id.checkUseIndex,
-      R.id.checkUseWordIndex, R.id.checkUseMailIndex })
+      R.id.checkChanges, R.id.checkCreateLogFiles, R.id.radioVerbosity,
+      R.id.checkUseIndex, R.id.checkUseWordIndex, R.id.checkUseMailIndex })
   public static class LogIndexCache extends Tab {
   }
 

app/src/main/java/com/httrack/android/OptionsMapper.java

@@ -104,6 +104,7 @@ public class OptionsMapper {
       new Pair<Integer, String>(R.id.checkHidePasswords, "NoPwdInPages"),
       new Pair<Integer, String>(R.id.checkHideQueryStrings, "NoQueryStrings"),
       new Pair<Integer, String>(R.id.checkDoNotPurge, "NoPurgeOldFiles"),
+      new Pair<Integer, String>(R.id.checkSingleFile, "SingleFile"),
       new Pair<Integer, String>(R.id.checkWarc, "Warc"),
       new Pair<Integer, String>(R.id.radioBuild, "Build"),
       new Pair<Integer, String>(R.id.editCustomBuild, "BuildString"),
@@ -121,6 +122,7 @@ public class OptionsMapper {
       new Pair<Integer, String>(R.id.radioCheckDocumentType, "CheckType"),
       new Pair<Integer, String>(R.id.checkParseJavaFiles, "ParseJava"),
       new Pair<Integer, String>(R.id.radioSpider, "FollowRobotsTxt"),
+      new Pair<Integer, String>(R.id.checkSitemap, "Sitemap"),
       new Pair<Integer, String>(R.id.checkUpdateHacks, "UpdateHack"),
       new Pair<Integer, String>(R.id.checkUrlHacks, "URLHack"),
       new Pair<Integer, String>(R.id.checkTolerentRequests, "TolerantRequests"),
@@ -137,6 +139,7 @@ public class OptionsMapper {
           "StoreAllInCache"),
       new Pair<Integer, String>(R.id.checkDoNotRedownloadLocallErasedFiles,
           "NoRecatch"),
+      new Pair<Integer, String>(R.id.checkChanges, "Changes"),
       new Pair<Integer, String>(R.id.checkCreateLogFiles, "Log"),
       /* FIXME with Log */
       new Pair<Integer, String>(R.id.radioVerbosity, "LogType"),
@@ -200,6 +203,9 @@ public class OptionsMapper {
       new Pair<String, String>("NoQueryStrings", "0"),
       new Pair<String, String>("NoPurgeOldFiles", "0"),
       new Pair<String, String>("Warc", "0"),
+      new Pair<String, String>("Sitemap", "0"),
+      new Pair<String, String>("SingleFile", "0"),
+      new Pair<String, String>("Changes", "0"),
       new Pair<String, String>("Cookies", "1"),
       new Pair<String, String>("CheckType", "1"),
       new Pair<String, String>("ParseJava", "1"),
@@ -290,6 +296,10 @@ public class OptionsMapper {
       new Pair<String, OptionMapper>("NoPurgeOldFiles", new SimpleOptionFlag(
           "X0")),
       new Pair<String, OptionMapper>("Warc", new SimpleOptionFlag("%r")),
+      new Pair<String, OptionMapper>("Sitemap", new LongOptionFlag("--sitemap")),
+      new Pair<String, OptionMapper>("SingleFile", new LongOptionFlag(
+          "--single-file")),
+      new Pair<String, OptionMapper>("Changes", new LongOptionFlag("--changes")),
       new Pair<String, OptionMapper>("Build", buildHandler.getTypeMapper()),
       new Pair<String, OptionMapper>("BuildString",
           buildHandler.getCustomMapper()),
@@ -1366,6 +1376,27 @@ public void emit(final StringBuilder flags, final List<String> commandline,
     }
   }
 
+  /**
+   * Required for a short flag with a two-letter variant (-%m / -%mu, -%Z /
+   * -%Zs): packed in the compacted string, the next flag's first letter would
+   * be misread as that variant and swallow an argument.
+   */
+  public static class LongOptionFlag implements OptionMapper {
+    protected final String option;
+
+    public LongOptionFlag(final String option) {
+      this.option = option;
+    }
+
+    @Override
+    public void emit(final StringBuilder flags, final List<String> commandline,
+        final String value) {
+      if ("1".equals(value)) {
+        commandline.add(option);
+      }
+    }
+  }
+
   /**
    * Option without any value.
    */

app/src/main/jni/Android.mk

@@ -76,6 +76,8 @@ LOCAL_SRC_FILES := httrack/src/htscore.c httrack/src/htsparse.c 			\
 	httrack/src/htscache_selftest.c httrack/src/htsdns_selftest.c			\
 	httrack/src/htscodec.c httrack/src/htsproxy.c							\
 	httrack/src/htsurlport.c httrack/src/htswarc.c							\
+	httrack/src/htssitemap.c httrack/src/htssinglefile.c					\
+	httrack/src/htschanges.c httrack/src/htscmdline.c						\
 	httrack/src/minizip/ioapi.c												\
 	httrack/src/minizip/mztools.c httrack/src/minizip/unzip.c				\
 	httrack/src/minizip/zip.c

app/src/main/res/layout/activity_options_build.xml

@@ -56,6 +56,12 @@
             android:layout_height="wrap_content"
             android:text="@string/do_not_purge_old_files" />
 
+        <CheckBox
+            android:id="@+id/checkSingleFile"
+            android:layout_width="fill_parent"
+            android:layout_height="wrap_content"
+            android:text="@string/single_file" />
+
         <LinearLayout
             android:layout_width="fill_parent"
             android:layout_height="wrap_content"

app/src/main/res/layout/activity_options_logindexcache.xml

@@ -31,6 +31,12 @@
             android:layout_height="wrap_content"
             android:text="@string/write_warc_archive" />
 
+        <CheckBox
+            android:id="@+id/checkChanges"
+            android:layout_width="fill_parent"
+            android:layout_height="wrap_content"
+            android:text="@string/report_changes" />
+
         <LinearLayout
             android:layout_width="fill_parent"
             android:layout_height="wrap_content"

app/src/main/res/layout/activity_options_spider.xml

@@ -118,6 +118,13 @@
             </RadioGroup>
         </LinearLayout>
 
+        <CheckBox
+            android:id="@+id/checkSitemap"
+            android:layout_width="fill_parent"
+            android:layout_height="wrap_content"
+            android:layout_gravity="end"
+            android:text="@string/sitemap" />
+
         <CheckBox
             android:id="@+id/checkUpdateHacks"
             android:layout_width="fill_parent"

app/src/main/res/values/strings.xml

@@ -68,7 +68,10 @@
     <string name="hide_passwords">Hide passwords</string>
     <string name="hide_query_strings">Hide query strings</string>
     <string name="do_not_purge_old_files">Do not purge old files</string>
+    <string name="single_file">Inline assets as data: URIs (self-contained pages)</string>
     <string name="write_warc_archive">Write WARC archive</string>
+    <string name="report_changes">Report what changed since the previous mirror</string>
+    <string name="sitemap">Seed the crawl from the site\'s sitemap</string>
     <string name="number_of_connections">Max simultaneous connections</string>
     <string name="persistent_connections">Persistent connections (Keep-Alive)</string>
     <string name="timeout">File timeout</string>

app/src/test/java/com/httrack/android/OptionsEmissionTest.java

@@ -5,6 +5,7 @@
 import static org.junit.Assert.assertTrue;
 
 import com.httrack.android.OptionsMapper.ArgumentOption;
+import com.httrack.android.OptionsMapper.LongOptionFlag;
 import com.httrack.android.OptionsMapper.OptionMapper;
 import com.httrack.android.OptionsMapper.ProxyHandler;
 import com.httrack.android.OptionsMapper.SimpleOptionFlag;
@@ -98,4 +99,42 @@ public void warcToggleEmitsNothingWhenUnchecked() {
     new SimpleOptionFlag("%r").emit(flags, new ArrayList<String>(), "0");
     assertFalse(flags.toString().contains("%r"));
   }
+
+  /* sitemap/single-file/changes: own token, and never in the packed string. */
+  @Test
+  public void longOptionEmitsItsOwnTokenWhenChecked() {
+    for (final String option : new String[] { "--sitemap", "--single-file",
+        "--changes" }) {
+      /* seeded as buildCommandline() does, so a guarded append cannot hide */
+      final StringBuilder flags = new StringBuilder("-");
+      final List<String> cmd = new ArrayList<String>();
+      new LongOptionFlag(option).emit(flags, cmd, "1");
+      assertEquals(1, cmd.size());
+      assertEquals(option, cmd.get(0));
+      assertEquals("-", flags.toString());
+    }
+  }
+
+  @Test
+  public void longOptionStaysSilentWhenUncheckedOrUnset() {
+    for (final String value : new String[] { "0", "", null }) {
+      final StringBuilder flags = new StringBuilder("-");
+      final List<String> cmd = new ArrayList<String>();
+      new LongOptionFlag("--single-file").emit(flags, cmd, value);
+      assertTrue(cmd.isEmpty());
+      assertEquals("-", flags.toString());
+    }
+  }
+
+  /* the mappers are static singletons: emitting again must not go quiet. */
+  @Test
+  public void longOptionEmitsOnEveryBuild() {
+    final OptionMapper mapper = new LongOptionFlag("--changes");
+    final List<String> first = new ArrayList<String>();
+    final List<String> second = new ArrayList<String>();
+    mapper.emit(new StringBuilder("-"), first, "1");
+    mapper.emit(new StringBuilder("-"), second, "1");
+    assertEquals(first, second);
+    assertEquals("--changes", second.get(0));
+  }
 }