From 5a65bbdb25928ac1696f4f38a6dd633317f55ce1 Mon Sep 17 00:00:00 2001 From: ahadsuleymanli Date: Thu, 23 Jan 2020 13:07:36 +0300 Subject: [PATCH] let number of output directories be specified as an argument --- README.md | 6 ++++-- WikiExtractor.py | 28 ++++++++++++++++++++++------ 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 2de2fa9e..3361206d 100644 --- a/README.md +++ b/README.md @@ -40,8 +40,8 @@ Each file will contains several documents in this [document format](http://media [--filter_category path_of_categories_file] [--filter_disambig_pages] [-it abbr,b,big] [-de gallery,timeline,noinclude] [--keep_tables] - [--processes PROCESSES] [-q] [--debug] [-a] [-v] - [--log_file] + [--processes PROCESSES] [--dir_count DIR_COUNT] + [-q] [--debug] [-a] [-v] [--log_file] input Wikipedia Extractor: @@ -69,6 +69,8 @@ Each file will contains several documents in this [document format](http://media -h, --help show this help message and exit --processes PROCESSES Number of processes to use (default 1) + --dir_count DIR_COUNT + Force number of directories to distribute the files amongst (default is as many directories as needed with 100 files per directory) Output: -o OUTPUT, --output OUTPUT diff --git a/WikiExtractor.py b/WikiExtractor.py index 730b3bab..c0090e7b 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -216,6 +216,7 @@ def __eq__ (self, other): g_page_total = 0 g_page_articl_total=0 g_page_articl_used_total=0 +g_args = None # page filtering logic -- remove templates, undesired xml namespaces, and disambiguation pages def keepPage(ns, catSet, page): global g_page_articl_total,g_page_total,g_page_articl_used_total @@ -2667,23 +2668,36 @@ class NextFile(object): """ Synchronous generation of next available file name. """ - filesPerDir = 100 def __init__(self, path_name): + global g_args + self.forcedDirCount = g_args.dir_count self.path_name = path_name self.dir_index = -1 self.file_index = -1 + self.file_indexes = [-1]*self.forcedDirCount def __next__(self): - self.file_index = (self.file_index + 1) % NextFile.filesPerDir - if self.file_index == 0: - self.dir_index += 1 + if self.forcedDirCount > 0: + self.forced_next_helper() + else: + self.file_index = (self.file_index + 1) % NextFile.filesPerDir + if self.file_index == 0: + self.dir_index += 1 dirname = self._dirname() if not os.path.isdir(dirname): os.makedirs(dirname) return self._filepath() + def forced_next_helper(self): + ''' + returns path in the next dir in the forced directory list to accomodate the dir_count + ''' + self.dir_index = (self.dir_index + 1) % self.forcedDirCount + self.file_indexes[self.dir_index] += 1 + self.file_index = (self.file_indexes[self.dir_index]) + next = __next__ def _dirname(self): @@ -3108,7 +3122,7 @@ def reduce_process(opts, output_queue, spool_length, minFileSize = 200 * 1024 def main(): - + global g_args parser = argparse.ArgumentParser(prog=os.path.basename(sys.argv[0]), formatter_class=argparse.RawDescriptionHelpFormatter, description=__doc__) @@ -3156,6 +3170,8 @@ def main(): default_process_count = max(1, cpu_count() - 1) parser.add_argument("--processes", type=int, default=default_process_count, help="Number of processes to use (default %(default)s)") + parser.add_argument("--dir_count", type=int, default=-1, + help="Force number of directories to distribute the files amongst (default is as many directories as needed with 100 files per directory)") groupS = parser.add_argument_group('Special') groupS.add_argument("-q", "--quiet", action="store_true", @@ -3172,7 +3188,7 @@ def main(): groupP.add_argument("--filter_category", help="specify the file that listing the Categories you want to include or exclude. One line for" " one category. starting with: 1) '#' comment, ignored; 2) '^' exclude; Note: excluding has higher priority than including") - args = parser.parse_args() + g_args = args = parser.parse_args() options.keepLinks = args.links options.keepSections = args.sections