From 1ebc7701e302f85296ad44163921c61db084e1c3 Mon Sep 17 00:00:00 2001 From: Zae Myung Kim Date: Thu, 3 Oct 2019 15:34:29 +0900 Subject: [PATCH 1/3] extract page categories --- WikiExtractor.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index 730b3bab..4d029a17 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -556,7 +556,7 @@ def __init__(self, id, revid, title, lines): self.recursion_exceeded_3_errs = 0 # parameter recursion self.template_title_errs = 0 - def write_output(self, out, text): + def write_output(self, out, text, categories): """ :param out: a memory file :param text: the text of the page @@ -567,7 +567,8 @@ def write_output(self, out, text): 'id': self.id, 'url': url, 'title': self.title, - 'text': "\n".join(text) + 'text': "\n".join(text), + 'categories': categories } if options.print_revision: json_data['revid'] = self.revid @@ -579,10 +580,11 @@ def write_output(self, out, text): out.write(out_str) out.write('\n') else: + categories_line = ', '.join(categories) if options.print_revision: - header = '\n' % (self.id, self.revid, url, self.title) + header = '\n' % (self.id, self.revid, url, self.title, categories_line) else: - header = '\n' % (self.id, url, self.title) + header = '\n' % (self.id, url, self.title, categories_line) footer = "\n\n" if out == sys.stdout: # option -a or -o - header = header.encode('utf-8') @@ -644,7 +646,7 @@ def extract(self, out): # $text = $frame->expand( $dom ); # text = self.transform(text) - text = self.wiki2text(text) + text, categories = self.wiki2text(text) text = compact(self.clean(text)) # from zwChan text = [title_str] + text @@ -652,7 +654,7 @@ def extract(self, out): if sum(len(line) for line in text) < options.min_text_length: return - self.write_output(out, text) + self.write_output(out, text, categories) errs = (self.template_title_errs, self.recursion_exceeded_1_errs, @@ -725,6 +727,9 @@ def wiki2text(self, text): # residuals of unbalanced quotes text = text.replace("'''", '').replace("''", '"') + # extract page categories + categories = re.findall(r'\[\[Category:(.+?)\]\]', text) + # replace internal links text = replaceInternalLinks(text) @@ -743,7 +748,7 @@ def wiki2text(self, text): res += unescape(text[cur:m.start()]) + m.group(1) cur = m.end() text = res + unescape(text[cur:]) - return text + return text, categories def clean(self, text): From 511e7f13f117de07ad134db5342836805a8d3685 Mon Sep 17 00:00:00 2001 From: Zae Myung Kim Date: Thu, 3 Oct 2019 16:26:49 +0900 Subject: [PATCH 2/3] move category extraction, usr argparse, exclude sortkey --- WikiExtractor.py | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index 4d029a17..b87b3c03 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -143,6 +143,10 @@ def __eq__ (self, other): # Drop tables from the article keep_tables = False, + ## + # Extract and save page categories + extract_categories = False, + ## # Whether to preserve links in output keepLinks = False, @@ -556,7 +560,7 @@ def __init__(self, id, revid, title, lines): self.recursion_exceeded_3_errs = 0 # parameter recursion self.template_title_errs = 0 - def write_output(self, out, text, categories): + def write_output(self, out, text, categories=[]): """ :param out: a memory file :param text: the text of the page @@ -567,9 +571,11 @@ def write_output(self, out, text, categories): 'id': self.id, 'url': url, 'title': self.title, - 'text': "\n".join(text), - 'categories': categories + 'text': "\n".join(text) } + if options.extract_categories: + json_data['categories'] = categories + if options.print_revision: json_data['revid'] = self.revid # We don't use json.dump(data, out) because we want to be @@ -580,11 +586,14 @@ def write_output(self, out, text, categories): out.write(out_str) out.write('\n') else: - categories_line = ', '.join(categories) if options.print_revision: - header = '\n' % (self.id, self.revid, url, self.title, categories_line) + header = '\n' % (self.id, self.revid, url, self.title) else: - header = '\n' % (self.id, url, self.title, categories_line) + header = '\n' % (self.id, url, self.title) + if options.extract_categories: + categories_line = ', '.join(categories) + header = header[:-2] + ' categories="%s">\n' % (categories_line) + footer = "\n\n" if out == sys.stdout: # option -a or -o - header = header.encode('utf-8') @@ -646,7 +655,11 @@ def extract(self, out): # $text = $frame->expand( $dom ); # text = self.transform(text) - text, categories = self.wiki2text(text) + categories = [] + if options.extract_categories: + # Extract categories but not sortkeys + categories = re.findall(r'\[\[Category:([^|\]]+)(?:|[^\]]+)?\]\]', text) + text = self.wiki2text(text) text = compact(self.clean(text)) # from zwChan text = [title_str] + text @@ -727,9 +740,6 @@ def wiki2text(self, text): # residuals of unbalanced quotes text = text.replace("'''", '').replace("''", '"') - # extract page categories - categories = re.findall(r'\[\[Category:(.+?)\]\]', text) - # replace internal links text = replaceInternalLinks(text) @@ -748,7 +758,7 @@ def wiki2text(self, text): res += unescape(text[cur:m.start()]) + m.group(1) cur = m.end() text = res + unescape(text[cur:]) - return text, categories + return text def clean(self, text): @@ -3158,6 +3168,8 @@ def main(): help="comma separated list of elements that will be removed from the article text") groupP.add_argument("--keep_tables", action="store_true", default=options.keep_tables, help="Preserve tables in the output article text (default=%(default)s)") + groupP.add_argument("--extract_categories", action="store_true", default=options.extract_categories, + help="Extract and save page categories (default=%(default)s)") default_process_count = max(1, cpu_count() - 1) parser.add_argument("--processes", type=int, default=default_process_count, help="Number of processes to use (default %(default)s)") @@ -3192,6 +3204,7 @@ def main(): options.expand_templates = args.no_templates options.filter_disambig_pages = args.filter_disambig_pages options.keep_tables = args.keep_tables + options.extract_categories = args.extract_categories try: power = 'kmg'.find(args.bytes[-1].lower()) + 1 From 1305cf67468f7938f8facb74a0e4dfcd3bb068d4 Mon Sep 17 00:00:00 2001 From: Zae Myung Kim Date: Thu, 3 Oct 2019 22:40:27 +0900 Subject: [PATCH 3/3] add --category_surface --- WikiExtractor.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index b87b3c03..89f0a632 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -146,6 +146,7 @@ def __eq__ (self, other): ## # Extract and save page categories extract_categories = False, + category_surface = 'Category', ## # Whether to preserve links in output @@ -658,7 +659,8 @@ def extract(self, out): categories = [] if options.extract_categories: # Extract categories but not sortkeys - categories = re.findall(r'\[\[Category:([^|\]]+)(?:|[^\]]+)?\]\]', text) + rgx_category = r'\[\[%s:([^|\]]+)(?:|[^\]]+)?\]\]' % options.category_surface + categories = re.findall(rgx_category, text) text = self.wiki2text(text) text = compact(self.clean(text)) # from zwChan @@ -3169,7 +3171,9 @@ def main(): groupP.add_argument("--keep_tables", action="store_true", default=options.keep_tables, help="Preserve tables in the output article text (default=%(default)s)") groupP.add_argument("--extract_categories", action="store_true", default=options.extract_categories, - help="Extract and save page categories (default=%(default)s)") + help="Whether to extract page categories, when used, --category_surface is also required (default=%(default)s)") + groupP.add_argument("--category_surface", type=str, default=options.category_surface, + help="How `Category` is written in the wiki namespace. E.g. for English: `Category`, for French: `Catégorie`") default_process_count = max(1, cpu_count() - 1) parser.add_argument("--processes", type=int, default=default_process_count, help="Number of processes to use (default %(default)s)") @@ -3205,6 +3209,7 @@ def main(): options.filter_disambig_pages = args.filter_disambig_pages options.keep_tables = args.keep_tables options.extract_categories = args.extract_categories + options.category_surface = args.category_surface try: power = 'kmg'.find(args.bytes[-1].lower()) + 1