From e2d332eb238b2acdfd7fecff59309af2365578b0 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Fri, 13 Dec 2019 01:28:29 +0100 Subject: [PATCH 001/110] Compacted Headers # Whether to add an empty line after header noLineAfterHeader = False, --- WikiExtractor.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index 730b3bab..50f103a0 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -17,7 +17,7 @@ # orangain (orangain@gmail.com) # Seth Cleveland (scleveland@turnitin.com) # Bren Barn -# +# HjalmarrSv # ============================================================================= # Copyright (c) 2011-2017. Giuseppe Attardi (attardi@di.unipi.it). # ============================================================================= @@ -154,6 +154,10 @@ def __eq__ (self, other): ## # Whether to preserve lists keepLists = False, + + ## + # Whether to add an empty line after header + noLineAfterHeader = False, ## # Whether to output HTML instead of text @@ -604,7 +608,11 @@ def extract(self, out): if options.toHTML: title_str = '

' + self.title + '

' else: - title_str = self.title + '\n' + if options.noLineAfterHeader: + title_str = self.title + else: + title_str = self.title + '\n' + #Above added by HjalmarrSv # https://www.mediawiki.org/wiki/Help:Magic_words colon = self.title.find(':') if colon != -1: @@ -3133,6 +3141,8 @@ def main(): help="preserve links") groupP.add_argument("-s", "--sections", action="store_true", help="preserve sections") + groupP.add_argument("-s", "--noLineAfterHeader", action="store_true", + help="does not add line below heading") groupP.add_argument("--lists", action="store_true", help="preserve lists") groupP.add_argument("-ns", "--namespaces", default="", metavar="ns1,ns2", From 053fde39dd2b582a5118e29af2e3b6fcb9d69a3f Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Fri, 13 Dec 2019 01:47:19 +0100 Subject: [PATCH 002/110] Update WikiExtractor.py --- WikiExtractor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index 50f103a0..fb09f427 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -3141,7 +3141,7 @@ def main(): help="preserve links") groupP.add_argument("-s", "--sections", action="store_true", help="preserve sections") - groupP.add_argument("-s", "--noLineAfterHeader", action="store_true", + groupP.add_argument("--noLineAfterHeader", action="store_true", help="does not add line below heading") groupP.add_argument("--lists", action="store_true", help="preserve lists") From 59e597cdb5497a5feaabae2f6b3118968414456e Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Fri, 13 Dec 2019 02:12:46 +0100 Subject: [PATCH 003/110] Update WikiExtractor.py --- WikiExtractor.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index fb09f427..f2294b9f 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -149,7 +149,7 @@ def __eq__ (self, other): ## # Whether to preserve section titles - keepSections = True, + keepSections = False, ## # Whether to preserve lists @@ -607,11 +607,10 @@ def extract(self, out): # Separate header from text with a newline. if options.toHTML: title_str = '

' + self.title + '

' + elif options.noLineAfterHeader: + title_str = self.title else: - if options.noLineAfterHeader: - title_str = self.title - else: - title_str = self.title + '\n' + title_str = self.title + '\n' #Above added by HjalmarrSv # https://www.mediawiki.org/wiki/Help:Magic_words colon = self.title.find(':') From 58700406403b664f3060f599db0f8d709d4cafcd Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Fri, 13 Dec 2019 02:47:18 +0100 Subject: [PATCH 004/110] Update WikiExtractor.py --- WikiExtractor.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index f2294b9f..d142ba49 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -158,7 +158,11 @@ def __eq__ (self, other): ## # Whether to add an empty line after header noLineAfterHeader = False, - + + ## + # Whether to add aHeader and Footer + addHeaderFooterHeader = False, + ## # Whether to output HTML instead of text toHTML = False, @@ -582,7 +586,7 @@ def write_output(self, out, text): out_str = out_str.encode('utf-8') out.write(out_str) out.write('\n') - else: + elif addHeaderFooter: if options.print_revision: header = '\n' % (self.id, self.revid, url, self.title) else: @@ -597,6 +601,13 @@ def write_output(self, out, text): out.write(line) out.write('\n') out.write(footer) + else: + for line in text: + if out == sys.stdout: # option -a or -o - + line = line.encode('utf-8') + if line!="": + out.write(line) + out.write('\n') def extract(self, out): """ @@ -3140,6 +3151,8 @@ def main(): help="preserve links") groupP.add_argument("-s", "--sections", action="store_true", help="preserve sections") + groupP.add_argument("--addHeaderFooter", action="store_true", + help="adds header and footer to each article") groupP.add_argument("--noLineAfterHeader", action="store_true", help="does not add line below heading") groupP.add_argument("--lists", action="store_true", From d2e633097658cdabd4ca54eed5bc148828c12331 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Fri, 13 Dec 2019 11:40:00 +0100 Subject: [PATCH 005/110] fixed added errors --- WikiExtractor.py | 57 ++++++++++++++++++++++++++++++++++++------------ 1 file changed, 43 insertions(+), 14 deletions(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index d142ba49..0d7558cb 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -161,7 +161,15 @@ def __eq__ (self, other): ## # Whether to add aHeader and Footer - addHeaderFooterHeader = False, + headersfooters = False, + + ## + # Whether to add aHeader and Footer + spacefree = False, + + ## + # Whether to add aHeader and Footer + titlefree = False, ## # Whether to output HTML instead of text @@ -586,29 +594,39 @@ def write_output(self, out, text): out_str = out_str.encode('utf-8') out.write(out_str) out.write('\n') - elif addHeaderFooter: + elif headersfooters: if options.print_revision: header = '\n' % (self.id, self.revid, url, self.title) else: header = '\n' % (self.id, url, self.title) footer = "\n\n" - if out == sys.stdout: # option -a or -o - - header = header.encode('utf-8') - out.write(header) for line in text: if out == sys.stdout: # option -a or -o - line = line.encode('utf-8') - out.write(line) - out.write('\n') + if options.spacefree: + if line != "": + out.write(line) + out.write('\n') + else: + out.write(line) + out.write('\n') out.write(footer) + if options.spacefree: + out.write('\n') else: for line in text: if out == sys.stdout: # option -a or -o - line = line.encode('utf-8') - if line!="": + if options.spacefree: + if line != "": + out.write(line) + out.write('\n') + else: out.write(line) - out.write('\n') - + out.write('\n') + if options.spacefree: + out.write('\n') + def extract(self, out): """ :param out: a memory file. @@ -665,8 +683,8 @@ def extract(self, out): text = self.wiki2text(text) text = compact(self.clean(text)) # from zwChan - text = [title_str] + text - + if not options.titlefree: + text = [title_str] + text if sum(len(line) for line in text) < options.min_text_length: return @@ -3151,10 +3169,16 @@ def main(): help="preserve links") groupP.add_argument("-s", "--sections", action="store_true", help="preserve sections") - groupP.add_argument("--addHeaderFooter", action="store_true", + + groupP.add_argument("--headersfooters", action="store_true", help="adds header and footer to each article") groupP.add_argument("--noLineAfterHeader", action="store_true", help="does not add line below heading") + groupP.add_argument("--titlefree", action="store_true", + help="no titles on articles") + groupP.add_argument("--spacefree", action="store_true", + help="minimize empty lines") + groupP.add_argument("--lists", action="store_true", help="preserve lists") groupP.add_argument("-ns", "--namespaces", default="", metavar="ns1,ns2", @@ -3205,7 +3229,12 @@ def main(): options.min_text_length = args.min_text_length if args.html: options.keepLinks = True - + + options.noLineAfterHeader = args.noLineAfterHeader + options.headersfooters = args.headersfooters + options.titlefree = args.titlefree + options.spacefree = args.spacefree + options.expand_templates = args.no_templates options.filter_disambig_pages = args.filter_disambig_pages options.keep_tables = args.keep_tables From eeda9548ca71bb12a1b6201a58c56deee4acbe1b Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Fri, 13 Dec 2019 12:08:11 +0100 Subject: [PATCH 006/110] works but errors may exist as well as quirks ## # Whether to preserve lists keepLists = False, ## # Whether to add an empty line after header noLineAfterHeader = False, ## # Whether to add aHeader and Footer headersfooters = False, ## # Whether to add aHeader and Footer spacefree = False, ## # Whether to add aHeader and Footer titlefree = False, --- WikiExtractor.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index 0d7558cb..59251308 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -594,12 +594,15 @@ def write_output(self, out, text): out_str = out_str.encode('utf-8') out.write(out_str) out.write('\n') - elif headersfooters: + elif options.headersfooters: if options.print_revision: header = '\n' % (self.id, self.revid, url, self.title) else: header = '\n' % (self.id, url, self.title) footer = "\n\n" + out.write(header) + if options.titlefree and not options.noLineAfterHeader: + out.write('\n') for line in text: if out == sys.stdout: # option -a or -o - line = line.encode('utf-8') @@ -613,7 +616,7 @@ def write_output(self, out, text): out.write(footer) if options.spacefree: out.write('\n') - else: + else: for line in text: if out == sys.stdout: # option -a or -o - line = line.encode('utf-8') From 8d10750148b5b4ec72a114fdfd4da5e3ae4d1f39 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Fri, 13 Dec 2019 12:33:55 +0100 Subject: [PATCH 007/110] Added Example --- README.md | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 2de2fa9e..89827348 100644 --- a/README.md +++ b/README.md @@ -33,17 +33,21 @@ The script is invoked with a Wikipedia dump file as an argument. The output is stored in several files of similar size in a given directory. Each file will contains several documents in this [document format](http://medialab.di.unipi.it/wiki/Document_Format). - usage: WikiExtractor.py [-h] [-o OUTPUT] [-b n[KMG]] [-c] [--json] [--html] - [-l] [-s] [--lists] [-ns ns1,ns2] - [--templates TEMPLATES] [--no-templates] [-r] +usage: WikiExtractor.py + [-h] [-o OUTPUT] [-b n[KMG]] [-c] [--json] [--html] + [-l] [-s] [--headersfooters] [--noLineAfterHeader] + [--titlefree] [--spacefree] [--lists] [-ns ns1,ns2] + [--templates TEMPLATES] [--no_templates] [-r] [--min_text_length MIN_TEXT_LENGTH] - [--filter_category path_of_categories_file] [--filter_disambig_pages] [-it abbr,b,big] [-de gallery,timeline,noinclude] [--keep_tables] - [--processes PROCESSES] [-q] [--debug] [-a] [-v] - [--log_file] + [--processes PROCESSES] [-q] [--debug] [-a] + [--log_file LOG_FILE] [-v] + [--filter_category FILTER_CATEGORY] input +Example: python3 WikiExtractor.py -o wiki/test --headersfooters --titlefree --spacefree wiki/wiki-20191101-pages-articles.xml + Wikipedia Extractor: Extracts and cleans text from a Wikipedia database dump and stores output in a number of files of similar size in a given directory. @@ -116,6 +120,14 @@ Each file will contains several documents in this [document format](http://media from the article text --keep_tables Preserve tables in the output article text (default=False) + --headersfooters Adds header and footer to each article + (default=False) + --noLineAfterHeader Does not add line below title. Title is directly on article. + (default=False) + --titlefree No titles on articles + (default=False) + --spacefree Minimize empty lines, that is, only empty lines are before/after title. + (default=False) Special: -q, --quiet suppress reporting progress info From 3b6a710768c0a7496f7471f1ad5be6ab6a2b65e7 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Fri, 13 Dec 2019 12:58:38 +0100 Subject: [PATCH 008/110] Added 2 examples --- README.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 89827348..111eb155 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,15 @@ usage: WikiExtractor.py [--filter_category FILTER_CATEGORY] input -Example: python3 WikiExtractor.py -o wiki/test --headersfooters --titlefree --spacefree wiki/wiki-20191101-pages-articles.xml +Examples (tested for "correct" output): + +python3 WikiExtractor.py -o wiki/test --headersfooters --titlefree --spacefree wiki/wiki-20191101-pages-articles.xml + +python3 WikiExtractor.py -o wiki/test --titlefree --spacefree wiki/wiki-20191101-pages-articles.xml + +python3 WikiExtractor.py -o wiki/test --noLineAfterHeader --spacefree wiki/wiki-20191101-pages-articles.xml + + Wikipedia Extractor: Extracts and cleans text from a Wikipedia database dump and stores output in a From ef9c07a8cec699f7d33d6db3afd0d709c20f89d3 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Sat, 14 Dec 2019 09:07:46 +0100 Subject: [PATCH 009/110] -- spacefree now also removes line = " " -- spacefree now also removes line = " " in addition to line = "". Both empty lines and one space lines are frequent in the output otherwise. Clean output afterwards with sed or similar tool of choice, but carefully - it will quickly do _exactly_ as told. --- WikiExtractor.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index 59251308..2af6847f 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -607,21 +607,21 @@ def write_output(self, out, text): if out == sys.stdout: # option -a or -o - line = line.encode('utf-8') if options.spacefree: - if line != "": - out.write(line) - out.write('\n') + if line != "" and line != " ": #Remove empty lines and lines with one space that occur frequently. + out.write(line) #Maybe they should not be produced to begin with. + out.write('\n') #One space lines may come from both empty articles and empty titles. else: out.write(line) out.write('\n') out.write(footer) - if options.spacefree: + if options.spacefree: #For separating articles when all empty lines removed with --spacefree. out.write('\n') else: for line in text: if out == sys.stdout: # option -a or -o - line = line.encode('utf-8') if options.spacefree: - if line != "": + if line != "" and line != " ": out.write(line) out.write('\n') else: From 7b2abd40ce8d19112fc81e5c09972fb716a45041 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Fri, 20 Dec 2019 19:51:59 +0100 Subject: [PATCH 010/110] Changed --spacefree to --squeeze-blank (same as in cat --squeeze-blank ) + magic word Although the code uses options.squeeze_blank for uniformity Added one magic word, 'pagelanguage', based on diff below @see https://doc.wikimedia.org/mediawiki-core/master/php/MagicWord_8php_source.html https://doc.wikimedia.org/mediawiki-core/master/php/MagicWordFactory_8php_source.html --- WikiExtractor.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index 2af6847f..fc7bb62a 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -164,8 +164,8 @@ def __eq__ (self, other): headersfooters = False, ## - # Whether to add aHeader and Footer - spacefree = False, + # Suppress repeated empty output lines + squeeze_blank = False, ## # Whether to add aHeader and Footer @@ -606,7 +606,7 @@ def write_output(self, out, text): for line in text: if out == sys.stdout: # option -a or -o - line = line.encode('utf-8') - if options.spacefree: + if options.squeeze_blank: if line != "" and line != " ": #Remove empty lines and lines with one space that occur frequently. out.write(line) #Maybe they should not be produced to begin with. out.write('\n') #One space lines may come from both empty articles and empty titles. @@ -614,20 +614,20 @@ def write_output(self, out, text): out.write(line) out.write('\n') out.write(footer) - if options.spacefree: #For separating articles when all empty lines removed with --spacefree. + if options.squeeze_blank: #For separating articles when all empty lines removed with --squeeze-blank. out.write('\n') else: for line in text: if out == sys.stdout: # option -a or -o - line = line.encode('utf-8') - if options.spacefree: + if options.squeeze_blank: if line != "" and line != " ": out.write(line) out.write('\n') else: out.write(line) out.write('\n') - if options.spacefree: + if options.squeeze_blank: out.write('\n') def extract(self, out): @@ -1586,6 +1586,7 @@ class MagicWords(object): One copy in each Extractor. @see https://doc.wikimedia.org/mediawiki-core/master/php/MagicWord_8php_source.html + https://doc.wikimedia.org/mediawiki-core/master/php/MagicWordFactory_8php_source.html #added 191220 """ names = [ '!', @@ -1663,6 +1664,7 @@ class MagicWords(object): 'localtimestamp', 'directionmark', 'contentlanguage', + 'pagelanguage', #added 191220 'numberofadmins', 'cascadingsources', ] @@ -3179,8 +3181,8 @@ def main(): help="does not add line below heading") groupP.add_argument("--titlefree", action="store_true", help="no titles on articles") - groupP.add_argument("--spacefree", action="store_true", - help="minimize empty lines") + groupP.add_argument("--squeeze_blank", "--squeeze-blank", action="store_true", + help="suppress repeated empty output lines") groupP.add_argument("--lists", action="store_true", help="preserve lists") @@ -3236,7 +3238,7 @@ def main(): options.noLineAfterHeader = args.noLineAfterHeader options.headersfooters = args.headersfooters options.titlefree = args.titlefree - options.spacefree = args.spacefree + options.squeeze_blank = args.squeeze-blank options.expand_templates = args.no_templates options.filter_disambig_pages = args.filter_disambig_pages From de81a98a2d0c799da1750c64cc807b7792d0c32e Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Mon, 23 Dec 2019 10:14:19 +0100 Subject: [PATCH 011/110] Update updated to reflect change to --squeeze-blank --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 111eb155..886dd060 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ Each file will contains several documents in this [document format](http://media usage: WikiExtractor.py [-h] [-o OUTPUT] [-b n[KMG]] [-c] [--json] [--html] [-l] [-s] [--headersfooters] [--noLineAfterHeader] - [--titlefree] [--spacefree] [--lists] [-ns ns1,ns2] + [--titlefree] [--squeeze-blank] [--lists] [-ns ns1,ns2] [--templates TEMPLATES] [--no_templates] [-r] [--min_text_length MIN_TEXT_LENGTH] [--filter_disambig_pages] [-it abbr,b,big] @@ -48,11 +48,11 @@ usage: WikiExtractor.py Examples (tested for "correct" output): -python3 WikiExtractor.py -o wiki/test --headersfooters --titlefree --spacefree wiki/wiki-20191101-pages-articles.xml +python3 WikiExtractor.py -o wiki/test --headersfooters --titlefree --squeeze-blank wiki/wiki-20191101-pages-articles.xml -python3 WikiExtractor.py -o wiki/test --titlefree --spacefree wiki/wiki-20191101-pages-articles.xml +python3 WikiExtractor.py -o wiki/test --titlefree --squeeze-blank wiki/wiki-20191101-pages-articles.xml -python3 WikiExtractor.py -o wiki/test --noLineAfterHeader --spacefree wiki/wiki-20191101-pages-articles.xml +python3 WikiExtractor.py -o wiki/test --noLineAfterHeader --squeeze-blank wiki/wiki-20191101-pages-articles.xml From 866e073d53c42c0bfaf4b16fc623452e5443e9dd Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Mon, 23 Dec 2019 10:15:53 +0100 Subject: [PATCH 012/110] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 886dd060..1661e724 100644 --- a/README.md +++ b/README.md @@ -134,7 +134,7 @@ python3 WikiExtractor.py -o wiki/test --noLineAfterHeader --squeeze-blank wiki/w (default=False) --titlefree No titles on articles (default=False) - --spacefree Minimize empty lines, that is, only empty lines are before/after title. + --squeeze-blank Minimize empty lines, that is, only empty lines are before/after title. (default=False) Special: From 22da7ea967b7facfeb81ec0a581736766c3aa874 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Sat, 28 Dec 2019 22:24:03 +0100 Subject: [PATCH 013/110] --no-title as alias for --titlefree both should work the same --- WikiExtractor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index fc7bb62a..760b9b9f 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -3179,7 +3179,7 @@ def main(): help="adds header and footer to each article") groupP.add_argument("--noLineAfterHeader", action="store_true", help="does not add line below heading") - groupP.add_argument("--titlefree", action="store_true", + groupP.add_argument("--no-title", "--titlefree", action="store_true", help="no titles on articles") groupP.add_argument("--squeeze_blank", "--squeeze-blank", action="store_true", help="suppress repeated empty output lines") From 8378de6bdd58ef6068dbbd2d10a0b54ddf119539 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Sat, 28 Dec 2019 22:28:12 +0100 Subject: [PATCH 014/110] --no-title should work as --titlefree --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1661e724..e3fabf5d 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ Each file will contains several documents in this [document format](http://media usage: WikiExtractor.py [-h] [-o OUTPUT] [-b n[KMG]] [-c] [--json] [--html] [-l] [-s] [--headersfooters] [--noLineAfterHeader] - [--titlefree] [--squeeze-blank] [--lists] [-ns ns1,ns2] + [--titlefree | --no-title] [--squeeze-blank] [--lists] [-ns ns1,ns2] [--templates TEMPLATES] [--no_templates] [-r] [--min_text_length MIN_TEXT_LENGTH] [--filter_disambig_pages] [-it abbr,b,big] From dc56429a41de24c0a2b71f349075126ce41d2278 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Sat, 28 Dec 2019 22:43:15 +0100 Subject: [PATCH 015/110] --no-templates as alias for --no_templates as suggested by operator-name as: wants to merge 1 commit into attardi:master from operator-name:master --- WikiExtractor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index 760b9b9f..b9c1386e 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -3190,7 +3190,7 @@ def main(): help="accepted namespaces in links") groupP.add_argument("--templates", help="use or create file containing templates") - groupP.add_argument("--no_templates", action="store_false", + groupP.add_argument( "--no-templates", "--no_templates", action="store_false", help="Do not expand templates") groupP.add_argument("-r", "--revision", action="store_true", default=options.print_revision, help="Include the document revision id (default=%(default)s)") From 7198d0a736c749b376de47e6463ed65f2471374c Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Sun, 29 Dec 2019 00:57:55 +0100 Subject: [PATCH 016/110] temporarily change --no-title to -no-title for debugging --- WikiExtractor.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index b9c1386e..524efe50 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -168,7 +168,7 @@ def __eq__ (self, other): squeeze_blank = False, ## - # Whether to add aHeader and Footer + # Whether to have a title on articles titlefree = False, ## @@ -3179,7 +3179,7 @@ def main(): help="adds header and footer to each article") groupP.add_argument("--noLineAfterHeader", action="store_true", help="does not add line below heading") - groupP.add_argument("--no-title", "--titlefree", action="store_true", + groupP.add_argument("-no-title", "--titlefree", action="store_true", help="no titles on articles") groupP.add_argument("--squeeze_blank", "--squeeze-blank", action="store_true", help="suppress repeated empty output lines") @@ -3238,7 +3238,7 @@ def main(): options.noLineAfterHeader = args.noLineAfterHeader options.headersfooters = args.headersfooters options.titlefree = args.titlefree - options.squeeze_blank = args.squeeze-blank + options.squeeze_blank = args.squeeze_blank options.expand_templates = args.no_templates options.filter_disambig_pages = args.filter_disambig_pages From 67a0ce7a153f49a07750d190eeb9299e1212d48f Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Sun, 29 Dec 2019 10:41:32 +0100 Subject: [PATCH 017/110] Update WikiExtractor.py removed doublette of '__TOC__', added: '__NOEDITSECTION__', #added 191229 '__EXPECTUNUSEDCATEGORY__', #added 191229 as found in private $mDoubleUnderscoreIDs here: https://doc.wikimedia.org/mediawiki-core/master/php/MagicWordFactory_8php_source.html --- WikiExtractor.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index 524efe50..d5a3f655 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -1682,11 +1682,12 @@ def __setitem__(self, name, value): '__NOTOC__', '__FORCETOC__', '__TOC__', - '__TOC__', + '__NOEDITSECTION__', #added 191229 '__NEWSECTIONLINK__', '__NONEWSECTIONLINK__', '__NOGALLERY__', '__HIDDENCAT__', + '__EXPECTUNUSEDCATEGORY__', #added 191229 '__NOCONTENTCONVERT__', '__NOCC__', '__NOTITLECONVERT__', From 49ad98814940ab47d6b673b15058e6bebe6e7d55 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Sun, 29 Dec 2019 12:54:51 +0100 Subject: [PATCH 018/110] About cleaning --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index e3fabf5d..d6b4a836 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,15 @@ python3 WikiExtractor.py -o wiki/test --titlefree --squeeze-blank wiki/wiki-2019 python3 WikiExtractor.py -o wiki/test --noLineAfterHeader --squeeze-blank wiki/wiki-20191101-pages-articles.xml +After running the extractor there may be a need for cleaning the output. In linux you may use any of the following examples. Please copy all the files to a safe place first. ANY ERROR IN THE CODE WILL DESTROY YOUR TEXT. You can be sure your text will be destroyed many times before you find the right cleaning scripts. +left trim on one file: sed -i 's/^[ ]*//g' YOURTEXT +right trim on one file: sed -i 's/[ ]*$//g' YOURTEXT +If you want to work many files at a time use (do NOT have any othe files in the folder or subfolders): +left trim on all files in folder or subfolder: find wiki/* -type f -exec sed -i 's/^[ ]*//g' {} \; +right trim on all files in folder or subfolderfind wiki/* -type f -exec sed -i 's/[ ]*$//g' {} \; +remove a line that starts with < and ends with > on all files in folder or subfolderfind wiki/* -type f -exec sed -E -i '/^<[^<]*>$/d' {} \; +remove a line that starts with ( and ends with ) on all files in folder or subfolderfind wiki/* -type f -exec sed -E -i '/^[(][^(]*[)]$/d' {} \; +Search Internet for variations and how to use with other operating systems. Wikipedia Extractor: From bb68ed7360fcfba24c885685dd2f7111f41a98ee Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Sun, 29 Dec 2019 12:56:06 +0100 Subject: [PATCH 019/110] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d6b4a836..c0d63741 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ python3 WikiExtractor.py -o wiki/test --titlefree --squeeze-blank wiki/wiki-2019 python3 WikiExtractor.py -o wiki/test --noLineAfterHeader --squeeze-blank wiki/wiki-20191101-pages-articles.xml -After running the extractor there may be a need for cleaning the output. In linux you may use any of the following examples. Please copy all the files to a safe place first. ANY ERROR IN THE CODE WILL DESTROY YOUR TEXT. You can be sure your text will be destroyed many times before you find the right cleaning scripts. +After running the extractor there may be a need for cleaning the output. In linux you may use any of the following examples. Please copy all the files to a safe place first. ANY ERROR IN THE CODE WILL DESTROY YOUR TEXT. You can be sure your text will be destroyed many times before you find the right cleaning scripts.
left trim on one file: sed -i 's/^[ ]*//g' YOURTEXT right trim on one file: sed -i 's/[ ]*$//g' YOURTEXT If you want to work many files at a time use (do NOT have any othe files in the folder or subfolders): From 4f192edeb4afab11f8391c0111b0eef3a30fc3d1 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Sun, 29 Dec 2019 12:57:18 +0100 Subject: [PATCH 020/110] Update README.md --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index c0d63741..a3ad2ccb 100644 --- a/README.md +++ b/README.md @@ -55,13 +55,13 @@ python3 WikiExtractor.py -o wiki/test --titlefree --squeeze-blank wiki/wiki-2019 python3 WikiExtractor.py -o wiki/test --noLineAfterHeader --squeeze-blank wiki/wiki-20191101-pages-articles.xml After running the extractor there may be a need for cleaning the output. In linux you may use any of the following examples. Please copy all the files to a safe place first. ANY ERROR IN THE CODE WILL DESTROY YOUR TEXT. You can be sure your text will be destroyed many times before you find the right cleaning scripts.
-left trim on one file: sed -i 's/^[ ]*//g' YOURTEXT -right trim on one file: sed -i 's/[ ]*$//g' YOURTEXT -If you want to work many files at a time use (do NOT have any othe files in the folder or subfolders): -left trim on all files in folder or subfolder: find wiki/* -type f -exec sed -i 's/^[ ]*//g' {} \; -right trim on all files in folder or subfolderfind wiki/* -type f -exec sed -i 's/[ ]*$//g' {} \; -remove a line that starts with < and ends with > on all files in folder or subfolderfind wiki/* -type f -exec sed -E -i '/^<[^<]*>$/d' {} \; -remove a line that starts with ( and ends with ) on all files in folder or subfolderfind wiki/* -type f -exec sed -E -i '/^[(][^(]*[)]$/d' {} \; +left trim on one file: sed -i 's/^[ ]*//g' YOURTEXT
+right trim on one file: sed -i 's/[ ]*$//g' YOURTEXT
+If you want to work many files at a time use (do NOT have any othe files in the folder or subfolders):
+left trim on all files in folder or subfolder: find wiki/* -type f -exec sed -i 's/^[ ]*//g' {} \;
+right trim on all files in folder or subfolderfind wiki/* -type f -exec sed -i 's/[ ]*$//g' {} \;
+remove a line that starts with < and ends with > on all files in folder or subfolderfind wiki/* -type f -exec sed -E -i '/^<[^<]*>$/d' {} \;
+remove a line that starts with ( and ends with ) on all files in folder or subfolderfind wiki/* -type f -exec sed -E -i '/^[(][^(]*[)]$/d' {} \;
Search Internet for variations and how to use with other operating systems. From d2ce1a8587033698fa194e397c7c754ab0b33608 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Tue, 31 Dec 2019 01:50:51 +0100 Subject: [PATCH 021/110] spelling from total of page: %d, total of articl page: %d; total of used articl page: to total of pages: %d, total of article pages: %d; total of used article pages: --- WikiExtractor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index d5a3f655..5cccf083 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -3041,7 +3041,7 @@ def process_dump(input_file, template_file, out_file, file_size, file_compress, extract_rate = page_num / extract_duration logging.info("Finished %d-process extraction of %d articles in %.1fs (%.1f art/s)", process_count, page_num, extract_duration, extract_rate) - logging.info("total of page: %d, total of articl page: %d; total of used articl page: %d" % (g_page_total, g_page_articl_total,g_page_articl_used_total)) + logging.info("total of pages: %d, total of article pages: %d; total of used article pages: %d" % (g_page_total, g_page_articl_total,g_page_articl_used_total)) # ---------------------------------------------------------------------- From ea31c41d96426a9f917932a1cd8678b6bc47082a Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Tue, 31 Dec 2019 12:31:53 +0100 Subject: [PATCH 022/110] Added mediawiki links in comments --- WikiExtractor.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index 5cccf083..75c1a044 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -1585,8 +1585,9 @@ class MagicWords(object): """ One copy in each Extractor. - @see https://doc.wikimedia.org/mediawiki-core/master/php/MagicWord_8php_source.html - https://doc.wikimedia.org/mediawiki-core/master/php/MagicWordFactory_8php_source.html #added 191220 + @see https://doc.wikimedia.org/mediawiki-core/master/php/MagicWord_8php_source.html - invalid old link + https://doc.wikimedia.org/mediawiki-core/master/php/MagicWordFactory_8php_source.html + https://www.mediawiki.org/wiki/Manual:Magic_words """ names = [ '!', @@ -1677,7 +1678,7 @@ def __getitem__(self, name): def __setitem__(self, name, value): self.values[name] = value - + # https://www.mediawiki.org/wiki/Help:Magic_words#Behavior_switches switches = ( '__NOTOC__', '__FORCETOC__', From 67903abe5771174f03c5af994e06a646712f1272 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Sat, 4 Jan 2020 00:11:25 +0100 Subject: [PATCH 023/110] Implement josecannete/wikiextractorforBERT https://github.com/josecannete/wikiextractorforBERT --- WikiExtractor.py | 57 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 54 insertions(+), 3 deletions(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index 75c1a044..66fb4a12 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -17,7 +17,7 @@ # orangain (orangain@gmail.com) # Seth Cleveland (scleveland@turnitin.com) # Bren Barn -# HjalmarrSv +# HjalmarrSv [Thanks to: josecannete for wikiextractorforBERT] # ============================================================================= # Copyright (c) 2011-2017. Giuseppe Attardi (attardi@di.unipi.it). # ============================================================================= @@ -171,6 +171,14 @@ def __eq__ (self, other): # Whether to have a title on articles titlefree = False, + ## + ## Point separated + point_separated = False, + + ## + ## Remove tags like '<*>' + remove_html_tags = False, + ## # Whether to output HTML instead of text toHTML = False, @@ -298,6 +306,15 @@ def get_url(uid): placeholder_tags = {'math': 'formula', 'code': 'codice'} +tag_re = re.compile(r'<[^>]+>') + +special_token_re = re.compile(r'__\S+__') + +def remove_tags(text): + return tag_re.sub('', text) + +def remove_special_tokens(text): + return special_token_re.sub('', text) def normalizeTitle(title): """Normalize title""" @@ -688,6 +705,21 @@ def extract(self, out): # from zwChan if not options.titlefree: text = [title_str] + text + + if options.remove_html_tags: + text = [remove_tags(line) for line in text] + + if options.remove_special_tokens: + text = [remove_special_tokens(line) for line in text] + + if options.point_separated: + text = list(map(lambda t: t.replace('\n', '').replace('. ', '.\n').strip(), text)) + text = list(filter(lambda t: t is not '', text)) + text = list(filter(lambda t: t is not '\n', text)) + + if len(text) == 0: + return + if sum(len(line) for line in text) < options.min_text_length: return @@ -3185,7 +3217,15 @@ def main(): help="no titles on articles") groupP.add_argument("--squeeze_blank", "--squeeze-blank", action="store_true", help="suppress repeated empty output lines") - + groupP.add_argument("--for-bert", action="store_true", + help="ready for bert pre-training") + groupP.add_argument("--remove-special-tokens", action="store_true", + help="to remove every special token ie: between '__*__'") + groupP.add_argument("--remove-html-tags", action="store_true", + help="to remove every html tag ie: between '<*/>'") + groupP.add_argument("--point-separated", action="store_true", + help="every line is separated") + groupP.add_argument("--lists", action="store_true", help="preserve lists") groupP.add_argument("-ns", "--namespaces", default="", metavar="ns1,ns2", @@ -3241,11 +3281,22 @@ def main(): options.headersfooters = args.headersfooters options.titlefree = args.titlefree options.squeeze_blank = args.squeeze_blank - + options.point_separated = args.point_separated + options.remove_html_tags = args.remove_html_tags + options.remove_special_tokens = args.remove_special_tokens + options.expand_templates = args.no_templates options.filter_disambig_pages = args.filter_disambig_pages options.keep_tables = args.keep_tables + if args.for_bert: + options.headersfooters = False + options.titlefree = True + options.point_separated = True + # not needed as true for text: options.blank_line_between_docs = True + options.remove_html_tags = True + options.remove_special_tokens = True + try: power = 'kmg'.find(args.bytes[-1].lower()) + 1 file_size = int(args.bytes[:-1]) * 1024 ** power From a32e7b10c3139b9fb40f1b223c21c6fe88b7c286 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Sat, 4 Jan 2020 00:34:59 +0100 Subject: [PATCH 024/110] Fix: separate docs --- WikiExtractor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index 66fb4a12..8e50bbab 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -3293,7 +3293,7 @@ def main(): options.headersfooters = False options.titlefree = True options.point_separated = True - # not needed as true for text: options.blank_line_between_docs = True + options.squeeze_blank = True #Here it actually adds an empty line between articles. Otherwise it is not needed. options.remove_html_tags = True options.remove_special_tokens = True From 33251533f7e6685e2fbdc474eac99caf89b80455 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Sat, 4 Jan 2020 00:46:37 +0100 Subject: [PATCH 025/110] --for-bert New option. One sentence per line. Nothing but article text. One empty line between articles. Some cleaning. Adapted from work by josecannete on wikiextractorforBERT here on github Also some small edits. --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a3ad2ccb..01ee1b8c 100644 --- a/README.md +++ b/README.md @@ -54,15 +54,17 @@ python3 WikiExtractor.py -o wiki/test --titlefree --squeeze-blank wiki/wiki-2019 python3 WikiExtractor.py -o wiki/test --noLineAfterHeader --squeeze-blank wiki/wiki-20191101-pages-articles.xml +python3 WikiExtractor.py -o wiki/test --for-bert wiki/wiki-20191101-pages-articles.xml + After running the extractor there may be a need for cleaning the output. In linux you may use any of the following examples. Please copy all the files to a safe place first. ANY ERROR IN THE CODE WILL DESTROY YOUR TEXT. You can be sure your text will be destroyed many times before you find the right cleaning scripts.
left trim on one file: sed -i 's/^[ ]*//g' YOURTEXT
right trim on one file: sed -i 's/[ ]*$//g' YOURTEXT
If you want to work many files at a time use (do NOT have any othe files in the folder or subfolders):
left trim on all files in folder or subfolder: find wiki/* -type f -exec sed -i 's/^[ ]*//g' {} \;
-right trim on all files in folder or subfolderfind wiki/* -type f -exec sed -i 's/[ ]*$//g' {} \;
+right trim on all files in folder or subfolder: find wiki/* -type f -exec sed -i 's/[ ]*$//g' {} \;
remove a line that starts with < and ends with > on all files in folder or subfolderfind wiki/* -type f -exec sed -E -i '/^<[^<]*>$/d' {} \;
remove a line that starts with ( and ends with ) on all files in folder or subfolderfind wiki/* -type f -exec sed -E -i '/^[(][^(]*[)]$/d' {} \;
-Search Internet for variations and how to use with other operating systems. +Search Internet for variations and how to use with other operating systems. One variation would be to remove option "-i" and write changes to new files, instead of -i[nline] - although not very useful if you do more than one cleaning operation. Wikipedia Extractor: From 8bed74723e95033d7474b08cbdbbb23731601f90 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Sat, 4 Jan 2020 01:13:21 +0100 Subject: [PATCH 026/110] Added better example Use --min_text_length 100 for removing very short articles. --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 01ee1b8c..c59ddcfb 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,9 @@ python3 WikiExtractor.py -o wiki/test --noLineAfterHeader --squeeze-blank wiki/w python3 WikiExtractor.py -o wiki/test --for-bert wiki/wiki-20191101-pages-articles.xml +python3 WikiExtractor.py -o wiki/test --filter_disambig_pages --no_templates --for-bert --min_text_length 100 wiki/svwiki-20191101-pages-articles.xml + + After running the extractor there may be a need for cleaning the output. In linux you may use any of the following examples. Please copy all the files to a safe place first. ANY ERROR IN THE CODE WILL DESTROY YOUR TEXT. You can be sure your text will be destroyed many times before you find the right cleaning scripts.
left trim on one file: sed -i 's/^[ ]*//g' YOURTEXT
right trim on one file: sed -i 's/[ ]*$//g' YOURTEXT
From b4420f350faf4123a51360d850ea710bf34c197b Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Sat, 4 Jan 2020 01:14:17 +0100 Subject: [PATCH 027/110] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c59ddcfb..2183be68 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ python3 WikiExtractor.py -o wiki/test --noLineAfterHeader --squeeze-blank wiki/w python3 WikiExtractor.py -o wiki/test --for-bert wiki/wiki-20191101-pages-articles.xml -python3 WikiExtractor.py -o wiki/test --filter_disambig_pages --no_templates --for-bert --min_text_length 100 wiki/svwiki-20191101-pages-articles.xml +python3 WikiExtractor.py -o wiki/test --filter_disambig_pages --no_templates --for-bert --min_text_length 100 wiki/wiki-20191101-pages-articles.xml After running the extractor there may be a need for cleaning the output. In linux you may use any of the following examples. Please copy all the files to a safe place first. ANY ERROR IN THE CODE WILL DESTROY YOUR TEXT. You can be sure your text will be destroyed many times before you find the right cleaning scripts.
From 703249a7e17689100aef6f6c5acaf73c01da8941 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Sat, 4 Jan 2020 01:28:18 +0100 Subject: [PATCH 028/110] Update README.md --- README.md | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 2183be68..81c30e0f 100644 --- a/README.md +++ b/README.md @@ -33,18 +33,20 @@ The script is invoked with a Wikipedia dump file as an argument. The output is stored in several files of similar size in a given directory. Each file will contains several documents in this [document format](http://medialab.di.unipi.it/wiki/Document_Format). -usage: WikiExtractor.py - [-h] [-o OUTPUT] [-b n[KMG]] [-c] [--json] [--html] - [-l] [-s] [--headersfooters] [--noLineAfterHeader] - [--titlefree | --no-title] [--squeeze-blank] [--lists] [-ns ns1,ns2] - [--templates TEMPLATES] [--no_templates] [-r] - [--min_text_length MIN_TEXT_LENGTH] - [--filter_disambig_pages] [-it abbr,b,big] - [-de gallery,timeline,noinclude] [--keep_tables] - [--processes PROCESSES] [-q] [--debug] [-a] - [--log_file LOG_FILE] [-v] - [--filter_category FILTER_CATEGORY] - input +usage: WikiExtractor.py
+ [-h] [-o OUTPUT] [-b n[KMG]] [-c] [--json] [--html]
+ [-l] [-s] [--headersfooters] [--noLineAfterHeader]
+ [-no-title] [--squeeze_blank] [--for-bert]
+ [--remove-special-tokens] [--remove-html-tags]
+ [--point-separated] [--lists] [-ns ns1,ns2]
+ [--templates TEMPLATES] [--no-templates] [-r]
+ [--min_text_length MIN_TEXT_LENGTH]
+ [--filter_disambig_pages] [-it abbr,b,big]
+ [-de gallery,timeline,noinclude] [--keep_tables]
+ [--processes PROCESSES] [-q] [--debug] [-a]
+ [--log_file LOG_FILE] [-v]
+ [--filter_category FILTER_CATEGORY]
+ input Examples (tested for "correct" output): From 58d4352db46590aad2cd8f2fc8ca6c4182a21de9 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Sat, 4 Jan 2020 08:43:59 +0100 Subject: [PATCH 029/110] Concatenate many files to one file cat --squeeze-blank wiki/*/* > wiki/wiki.txt Notice the use of a wildcard on two levels, i.e. folders and files. Not possible usually. --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 81c30e0f..ace342da 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,8 @@ remove a line that starts with < and ends with > on all files in folder or subfo remove a line that starts with ( and ends with ) on all files in folder or subfolderfind wiki/* -type f -exec sed -E -i '/^[(][^(]*[)]$/d' {} \;
Search Internet for variations and how to use with other operating systems. One variation would be to remove option "-i" and write changes to new files, instead of -i[nline] - although not very useful if you do more than one cleaning operation. +For those use cases where only on large file is needed, in linux use: cat --squeeze-blank wiki/*/* > wiki/wiki.txt + Wikipedia Extractor: Extracts and cleans text from a Wikipedia database dump and stores output in a From bfda827dd2252baf7678f25de44121cdf283a7a5 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Sat, 4 Jan 2020 12:21:18 +0100 Subject: [PATCH 030/110] Template expansion Fix as proposed by chaojiang06: "Hi, I try to fix the template expansion function based on the current latest 2.75 version." "Basically, two lines got changed, #1269 and #1868." [not the same lines in this fork] --- WikiExtractor.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index 8e50bbab..db9958c9 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -702,7 +702,7 @@ def extract(self, out): text = self.transform(text) text = self.wiki2text(text) text = compact(self.clean(text)) - # from zwChan + # from zwChan ed. by HjalmarrSv if not options.titlefree: text = [title_str] + text @@ -1338,6 +1338,7 @@ def findMatchingBraces(text, ldelim=0): elif len(stack) == 1 and 0 < stack[0] < ldelim: # ambiguous {{{{{ }}} }} #yield m1.start() + stack[0], end + yield m1.start() + stack[0], end # Modified by Chao to solve Template expansion works well in an old version #197 cur = end break elif brac == '[': # [[ @@ -1940,7 +1941,8 @@ def sharp_invoke(module, function, args): if functions: funct = functions.get(function) if funct: - return text_type(funct(args)) + return funct(*[args.get(str(i + 1)) for i in range(len(args))]) # Modified by Chao for fixing Template expansion works well in an old version #197 + #return text_type(funct(args)) return '' From d7eb4069bbe7ede5de50fbca85f65df16434438f Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Sat, 4 Jan 2020 12:38:53 +0100 Subject: [PATCH 031/110] broken middoc interwiki link bugfix: handle broken middoc interwiki link gracefully from teffland/wikiextractor forked from karlstratos/wikiextractor --- WikiExtractor.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index db9958c9..4cf3ebed 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -2218,8 +2218,25 @@ def replaceInternalLinks(text): label = inner[pipe + 1:].strip() res += text[cur:s] + makeInternalLink(title, label) + trail cur = end - return res + text[cur:] - + #return res + text[cur:] + # BUGFIX:(teffland) - 2/14/19 + # If we've run out of balanced [[ ]] but there is still + # a [[ in the text, then there was an imbalanced link + # (typically due to a typo in the ]]) and we remove everything in the + # that link by removing everything between it and two ']'s away + # then restart the process + # Empirically this fixes a lot of broken pages. + text = res + text[cur:] + broken_link = text.find('[[') + if broken_link > -1: + right = text[broken_link+2:].find(']') + if right > -1: + right += text[broken_link+right+2:].find(']') + if right > -1: + text = text[:broken_link] + text[broken_link+right+2:] + return replaceInternalLinks(text) + else: + return text # the official version is a method in class Parser, similar to this: # def replaceInternalLinks2(text): From a4e4fdcfe912adebf851c767105e365da05f973f Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Sat, 4 Jan 2020 13:02:35 +0100 Subject: [PATCH 032/110] Augmented key regex to catch plus/minus signs karlstratos committed on Mar 17, 2018 --- WikiExtractor.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index 4cf3ebed..e304d629 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -2830,7 +2830,8 @@ def open(self, filename): tagRE = re.compile(r'(.*?)<(/?\w+)[^>]*?>(?:([^<]*)(<.*?>)?)?') # 1 2 3 4 -keyRE = re.compile(r'key="(\d*)"') +#keyRE = re.compile(r'key="(\d*)"') +keyRE = re.compile(r'key="([+-]?)(\d*)"') # karlstratos committed on Mar 17, 2018, Augmented key regex to catch plus/minus signs. catRE = re.compile(r'\[\[Category:([^\|]+).*\]\].*') # capture the category name [[Category:Category name|Sortkey]]" def load_templates(file, output_file=None): From 82f6d3cb109829c5111541a85747ebeeb9084efe Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Sat, 4 Jan 2020 15:41:09 +0100 Subject: [PATCH 033/110] New example for Bert and json python3 WikiExtractor.py -o wiki/test --filter_disambig_pages --templates templat.txt --titlefree --json --for-bert --min_text_length 100 wiki/wiki-20191101-pages-articles.xml --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index ace342da..c6daea71 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,8 @@ python3 WikiExtractor.py -o wiki/test --for-bert wiki/wiki-20191101-pages-articl python3 WikiExtractor.py -o wiki/test --filter_disambig_pages --no_templates --for-bert --min_text_length 100 wiki/wiki-20191101-pages-articles.xml +python3 WikiExtractor.py -o wiki/test --filter_disambig_pages --templates templat.txt --titlefree --json --for-bert --min_text_length 100 wiki/wiki-20191101-pages-articles.xml + After running the extractor there may be a need for cleaning the output. In linux you may use any of the following examples. Please copy all the files to a safe place first. ANY ERROR IN THE CODE WILL DESTROY YOUR TEXT. You can be sure your text will be destroyed many times before you find the right cleaning scripts.
left trim on one file: sed -i 's/^[ ]*//g' YOURTEXT
From 5ae4bcd3f534338c3402b4ff2c7c282652f1ef1a Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Sat, 4 Jan 2020 17:12:00 +0100 Subject: [PATCH 034/110] Force 'utf-8' encoding Force 'utf-8' encoding without relying on platform-dependent default #183 albertvillanova committed on Jul 13, 2019 --- WikiExtractor.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index e304d629..8e6602e1 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -2959,11 +2959,24 @@ def process_dump(input_file, template_file, out_file, file_size, file_compress, :param file_compress: whether to compress files with bzip. :param process_count: number of extraction processes to spawn. """ - + def hook_compressed_encoded(encoding): + def hook(filename, mode): + ext = os.path.splitext(filename)[1] + if ext == '.gz': + import gzip + return gzip.open(filename, mode, encoding=encoding) + elif ext == '.bz2': + import bz2 + return bz2.open(filename, mode, encoding=encoding) + else: + return open(filename, mode, encoding=encoding) + return hook + if input_file == '-': input = sys.stdin else: - input = fileinput.FileInput(input_file, openhook=fileinput.hook_compressed) + #input = fileinput.FileInput(input_file, openhook=fileinput.hook_compressed) + input = fileinput.FileInput(input_file, openhook=hook_compressed_encoded('utf-8')) # collect siteinfo for line in input: From c6b5d39d06b9b9a20a84e04714d804631d92ccd6 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Sat, 4 Jan 2020 17:21:41 +0100 Subject: [PATCH 035/110] Remove inline flags from the middle of a regex Remove inline flags from the middle of a regex #182 albertvillanova committed on Jul 13, 2019 --- WikiExtractor.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index 8e6602e1..7b589847 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -2539,16 +2539,20 @@ def makeInternalLink(title, label): EXT_LINK_URL_CLASS = r'[^][<>"\x00-\x20\x7F\s]' ANCHOR_CLASS = r'[^][\x00-\x08\x0a-\x1F]' ExtLinkBracketedRegex = re.compile( - '\[(((?i)' + '|'.join(wgUrlProtocols) + ')' + EXT_LINK_URL_CLASS + r'+)' + + #'\[(((?i)' + '|'.join(wgUrlProtocols) + ')' + EXT_LINK_URL_CLASS + r'+)' + + '\[((' + '|'.join(wgUrlProtocols) + ')' + EXT_LINK_URL_CLASS + r'+)' + r'\s*((?:' + ANCHOR_CLASS + r'|\[\[' + ANCHOR_CLASS + r'+\]\])' + r'*?)\]', - re.S | re.U) + #re.S | re.U) + re.I | re.S | re.U) # A simpler alternative: # ExtLinkBracketedRegex = re.compile(r'\[(.*?)\](?!])') EXT_IMAGE_REGEX = re.compile( r"""^(http://|https://)([^][<>"\x00-\x20\x7F\s]+) - /([A-Za-z0-9_.,~%\-+&;#*?!=()@\x80-\xFF]+)\.((?i)gif|png|jpg|jpeg)$""", - re.X | re.S | re.U) + #/([A-Za-z0-9_.,~%\-+&;#*?!=()@\x80-\xFF]+)\.((?i)gif|png|jpg|jpeg)$""", + #re.X | re.S | re.U) + /([A-Za-z0-9_.,~%\-+&;#*?!=()@\x80-\xFF]+)\.(gif|png|jpg|jpeg)$""", + re.I | re.X | re.S | re.U) def replaceExternalLinks(text): From 41fdefcfc2e65cc3488bdf1cca7f1118bc11d5bd Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Sun, 5 Jan 2020 00:38:10 +0100 Subject: [PATCH 036/110] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c6daea71..ba35b735 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ remove a line that starts with < and ends with > on all files in folder or subfo remove a line that starts with ( and ends with ) on all files in folder or subfolderfind wiki/* -type f -exec sed -E -i '/^[(][^(]*[)]$/d' {} \;
Search Internet for variations and how to use with other operating systems. One variation would be to remove option "-i" and write changes to new files, instead of -i[nline] - although not very useful if you do more than one cleaning operation. -For those use cases where only on large file is needed, in linux use: cat --squeeze-blank wiki/*/* > wiki/wiki.txt +For those use cases where only on large file is needed, in linux use: cat --squeeze-blank wiki/\*/\* > wiki/wiki.txt Wikipedia Extractor: From b2c4a47ba1c71118aeb402f05fbc202a1e3156eb Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Sun, 5 Jan 2020 13:54:40 +0100 Subject: [PATCH 037/110] Option to restrict to specific pages by title From: drgriffis/wikiextractor, drgriffis committed on Sep 26, 2018 --- WikiExtractor.py | 36 +++++++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index 7b589847..6ed68483 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -17,7 +17,9 @@ # orangain (orangain@gmail.com) # Seth Cleveland (scleveland@turnitin.com) # Bren Barn -# HjalmarrSv [Thanks to: josecannete for wikiextractorforBERT] +# HjalmarrSv [Thanks to: josecannete for wikiextractorforBERT, +# drgriffis for options.restrict_pages_to, +# # ============================================================================= # Copyright (c) 2011-2017. Giuseppe Attardi (attardi@di.unipi.it). # ============================================================================= @@ -172,9 +174,13 @@ def __eq__ (self, other): titlefree = False, ## - ## Point separated + # Point separated point_separated = False, + ## + # List of page IDs to restrict to + restrict_pages_to = None, + ## ## Remove tags like '<*>' remove_html_tags = False, @@ -241,10 +247,10 @@ def __eq__ (self, other): g_page_articl_total=0 g_page_articl_used_total=0 # page filtering logic -- remove templates, undesired xml namespaces, and disambiguation pages -def keepPage(ns, catSet, page): +def keepPage(ns, catSet, page, title): global g_page_articl_total,g_page_total,g_page_articl_used_total g_page_total += 1 - if ns != '0': # Aritcle + if ns != '0': # Article return False # remove disambig pages if desired g_page_articl_total += 1 @@ -252,6 +258,9 @@ def keepPage(ns, catSet, page): for line in page: if filter_disambig_page_pattern.match(line): return False + # check if filtering to specific list of pages + if options.restrict_pages_to and (not title in options.restrict_pages_to): + return False if len(options.filter_category_include) > 0 and len(options.filter_category_include & catSet)==0: logging.debug("***No include " + str(catSet)) return False @@ -262,6 +271,16 @@ def keepPage(ns, catSet, page): return True +def readRestrictedPageIDSet(f): + id_set = set() + with codecs.open(f, 'r', 'utf-8') as stream: + for line in stream: + id_set.add(line.strip().replace('_', ' ')) + return id_set + + + + def get_url(uid): return "%s?curid=%s" % (options.urlbase, uid) @@ -3078,7 +3097,7 @@ def hook(filename, mode): page_num = 0 for page_data in pages_from(input): id, revid, title, ns, catSet, page = page_data - if keepPage(ns, catSet, page): + if keepPage(ns, catSet, page, title): # slow down delay = 0 if spool_length.value > max_spool_length: @@ -3262,6 +3281,8 @@ def main(): help="to remove every html tag ie: between '<*/>'") groupP.add_argument("--point-separated", action="store_true", help="every line is separated") + groupP.add_argument("--restrict_pages_to", default=None, + help="List of page IDs to restrict to (one per line, case-sensitive)") groupP.add_argument("--lists", action="store_true", help="preserve lists") @@ -3326,6 +3347,11 @@ def main(): options.filter_disambig_pages = args.filter_disambig_pages options.keep_tables = args.keep_tables + if args.restrict_pages_to: + options.restrict_pages_to = readRestrictedPageIDSet(args.restrict_pages_to) + else: #Not strictly needed since None already declared twice, + options.restrict_pages_to = None #part of imported code that may be changed. + if args.for_bert: options.headersfooters = False options.titlefree = True From 2f80d1ce132ec5333cd79120bf9382040b4bcbe2 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Mon, 6 Jan 2020 18:20:11 +0100 Subject: [PATCH 038/110] --max_articles from tutcsis/wikiextractor --max_articles options is implemented. tsuchm committed on Feb 4, 2019 --- WikiExtractor.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/WikiExtractor.py b/WikiExtractor.py index 6ed68483..f8db7bf7 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -19,6 +19,7 @@ # Bren Barn # HjalmarrSv [Thanks to: josecannete for wikiextractorforBERT, # drgriffis for options.restrict_pages_to, +# tutcsis for --max_articles, # # ============================================================================= # Copyright (c) 2011-2017. Giuseppe Attardi (attardi@di.unipi.it). @@ -185,6 +186,10 @@ def __eq__ (self, other): ## Remove tags like '<*>' remove_html_tags = False, + ## + # Maximum count of articles (default 0 means unlimited) + max_articles = 0, + ## # Whether to output HTML instead of text toHTML = False, @@ -3110,6 +3115,8 @@ def hook(filename, mode): job = (id, revid, title, page, page_num) jobs_queue.put(job) # goes to any available extract_process page_num += 1 + if options.max_articles and page_num >= options.max_articles: # + break # any need to cleanup before break? page = None # free memory input.close() @@ -3283,6 +3290,8 @@ def main(): help="every line is separated") groupP.add_argument("--restrict_pages_to", default=None, help="List of page IDs to restrict to (one per line, case-sensitive)") + groupO.add_argument("--max_articles", type=int, default=0, + help="maximum count of articles (default 0 means unlimited)") groupP.add_argument("--lists", action="store_true", help="preserve lists") @@ -3342,6 +3351,7 @@ def main(): options.point_separated = args.point_separated options.remove_html_tags = args.remove_html_tags options.remove_special_tokens = args.remove_special_tokens + options.max_articles = args.max_articles options.expand_templates = args.no_templates options.filter_disambig_pages = args.filter_disambig_pages From 968324829a6860d817c0784f168223a0d984eceb Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Mon, 6 Jan 2020 19:41:16 +0100 Subject: [PATCH 039/110] bug fixes --- WikiExtractor.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index f8db7bf7..90c10461 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -728,14 +728,16 @@ def extract(self, out): text = compact(self.clean(text)) # from zwChan ed. by HjalmarrSv if not options.titlefree: - text = [title_str] + text - + text = [title_str] + text #does add title in the text of html and json, even + #though title in title also. use titlefree if not wanted. if options.remove_html_tags: text = [remove_tags(line) for line in text] + text = [line.replace(' ', ' ') for line in text] # squeezeblank only catches single space, above produces double (or moore?) if options.remove_special_tokens: text = [remove_special_tokens(line) for line in text] - + text = [line.replace(' ', ' ') for line in text] # squeezeblank only catches single space, above produces double (or moore?) + if options.point_separated: text = list(map(lambda t: t.replace('\n', '').replace('. ', '.\n').strip(), text)) text = list(filter(lambda t: t is not '', text)) @@ -2573,10 +2575,10 @@ def makeInternalLink(title, label): EXT_IMAGE_REGEX = re.compile( r"""^(http://|https://)([^][<>"\x00-\x20\x7F\s]+) - #/([A-Za-z0-9_.,~%\-+&;#*?!=()@\x80-\xFF]+)\.((?i)gif|png|jpg|jpeg)$""", - #re.X | re.S | re.U) /([A-Za-z0-9_.,~%\-+&;#*?!=()@\x80-\xFF]+)\.(gif|png|jpg|jpeg)$""", re.I | re.X | re.S | re.U) + #/([A-Za-z0-9_.,~%\-+&;#*?!=()@\x80-\xFF]+)\.((?i)gif|png|jpg|jpeg)$""", + #re.X | re.S | re.U) def replaceExternalLinks(text): From 761f67b53d32cb6153d7f770d288f16daf83a517 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Mon, 6 Jan 2020 19:54:18 +0100 Subject: [PATCH 040/110] max_articles, remove-html-tags in tested example and som tidying up. --- README.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ba35b735..bc567304 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,9 @@ usage: WikiExtractor.py
[-l] [-s] [--headersfooters] [--noLineAfterHeader]
[-no-title] [--squeeze_blank] [--for-bert]
[--remove-special-tokens] [--remove-html-tags]
- [--point-separated] [--lists] [-ns ns1,ns2]
+ [--point-separated]
+ [--restrict_pages_to RESTRICT_PAGES_TO]
+ [--max_articles MAX_ARTICLES] [--lists] [-ns ns1,ns2]
[--templates TEMPLATES] [--no-templates] [-r]
[--min_text_length MIN_TEXT_LENGTH]
[--filter_disambig_pages] [-it abbr,b,big]
@@ -62,6 +64,8 @@ python3 WikiExtractor.py -o wiki/test --filter_disambig_pages --no_templates --f python3 WikiExtractor.py -o wiki/test --filter_disambig_pages --templates templat.txt --titlefree --json --for-bert --min_text_length 100 wiki/wiki-20191101-pages-articles.xml +python3 WikiExtractor.py -o wiki/test --filter_disambig_pages --templates templat.txt --squeeze-blank --titlefree --max_articles 10 --remove-html-tags --min_text_length 100 wiki/wiki-20191101-pages-articles.xml + After running the extractor there may be a need for cleaning the output. In linux you may use any of the following examples. Please copy all the files to a safe place first. ANY ERROR IN THE CODE WILL DESTROY YOUR TEXT. You can be sure your text will be destroyed many times before you find the right cleaning scripts.
left trim on one file: sed -i 's/^[ ]*//g' YOURTEXT
@@ -69,8 +73,8 @@ right trim on one file: sed -i 's/[ ]*$//g' YOURTEXT
If you want to work many files at a time use (do NOT have any othe files in the folder or subfolders):
left trim on all files in folder or subfolder: find wiki/* -type f -exec sed -i 's/^[ ]*//g' {} \;
right trim on all files in folder or subfolder: find wiki/* -type f -exec sed -i 's/[ ]*$//g' {} \;
-remove a line that starts with < and ends with > on all files in folder or subfolderfind wiki/* -type f -exec sed -E -i '/^<[^<]*>$/d' {} \;
-remove a line that starts with ( and ends with ) on all files in folder or subfolderfind wiki/* -type f -exec sed -E -i '/^[(][^(]*[)]$/d' {} \;
+remove a line that starts with < and ends with > on all files in folder or subfolder: find wiki/* -type f -exec sed -E -i '/^<[^<]*>$/d' {} \;
+remove a line that starts with ( and ends with ) on all files in folder or subfolder: find wiki/* -type f -exec sed -E -i '/^[(][^(]*[)]$/d' {} \;
Search Internet for variations and how to use with other operating systems. One variation would be to remove option "-i" and write changes to new files, instead of -i[nline] - although not very useful if you do more than one cleaning operation. For those use cases where only on large file is needed, in linux use: cat --squeeze-blank wiki/\*/\* > wiki/wiki.txt From aef4bc8be70ce72695b978c6775ecee5c0fba92e Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Mon, 6 Jan 2020 20:25:49 +0100 Subject: [PATCH 041/110] remove mapframe, maplink Seem to be not useful. https://www.mediawiki.org/wiki/Help:Extension:Kartographer --- WikiExtractor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index 90c10461..e49fe861 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -235,7 +235,7 @@ def __eq__ (self, other): 'form', 'input', 'select', 'option', 'textarea', 'ul', 'li', 'ol', 'dl', 'dt', 'dd', 'menu', 'dir', 'ref', 'references', 'img', 'imagemap', 'source', 'small', - 'sub', 'sup', 'indicator' + 'sub', 'sup', 'indicator', ',mapframe', 'maplink' ], ) From 7c37e626619bf1994264d04aa63437a295c648c7 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Mon, 6 Jan 2020 22:48:53 +0100 Subject: [PATCH 042/110] discard score tags https://www.mediawiki.org/wiki/Extension:Score/en remove all tags, e.g. \relative c' { f d f a d f e d cis a cis e a g f e } --- WikiExtractor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index e49fe861..0b9151f7 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -235,7 +235,7 @@ def __eq__ (self, other): 'form', 'input', 'select', 'option', 'textarea', 'ul', 'li', 'ol', 'dl', 'dt', 'dd', 'menu', 'dir', 'ref', 'references', 'img', 'imagemap', 'source', 'small', - 'sub', 'sup', 'indicator', ',mapframe', 'maplink' + 'sub', 'sup', 'indicator', ',mapframe', 'maplink', 'score' ], ) From fe2b6f2ea03937dee0245c2311b51186076f0533 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Tue, 7 Jan 2020 20:21:01 +0100 Subject: [PATCH 043/110] cgi.escape with html.escape Replace cgi.escape with html.escape due to deprecation warning from infolab-csail/wikiextractor where michaelsilver committed 10 days ago --- WikiExtractor.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index 0b9151f7..fd4cc8f3 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -20,6 +20,7 @@ # HjalmarrSv [Thanks to: josecannete for wikiextractorforBERT, # drgriffis for options.restrict_pages_to, # tutcsis for --max_articles, +# michaelsilver for html.escape, # # ============================================================================= # Copyright (c) 2011-2017. Giuseppe Attardi (attardi@di.unipi.it). @@ -63,7 +64,11 @@ import argparse import bz2 import codecs -import cgi +import sys +if sys.version_info < (3, 2): + from cgi import escape as html_escape +else: + from html import escape as html_escape import fileinput import logging import os.path @@ -905,7 +910,7 @@ def clean(self, text): text = text.replace('|-', '') text = text.replace('|', '') if options.toHTML: - text = cgi.escape(text) + text = html_escape(text, quote=False) return text From 02757c61b3b1f4c12e0458a86151323c037a26b4 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Tue, 7 Jan 2020 22:12:05 +0100 Subject: [PATCH 044/110] Handle broken pipe and keyboard interrupts from versusvoid:master versusvoid committed on Aug 16, 2017 --- WikiExtractor.py | 189 ++++++++++++++++++++++++++++------------------- 1 file changed, 113 insertions(+), 76 deletions(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index fd4cc8f3..9b5a1704 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -75,8 +75,9 @@ import re # TODO use regex when it will be standard import time import json +import queue from io import StringIO -from multiprocessing import Queue, Process, Value, cpu_count +from multiprocessing import Event, Queue, Process, Value, cpu_count from timeit import default_timer @@ -2983,6 +2984,14 @@ def pages_from(input): title = None page = [] +def try_put_until(event, q, value): + while not event.is_set(): + try: + return q.put(value, False, 1) + except queue.Full: + pass + + def process_dump(input_file, template_file, out_file, file_size, file_compress, process_count): @@ -3086,10 +3095,12 @@ def hook(filename, mode): max_spool_length = 10000 spool_length = Value('i', 0, lock=False) + broken_pipe_event = Event() + # reduce job that sorts and prints output reduce = Process(target=reduce_process, args=(options, output_queue, spool_length, - out_file, file_size, file_compress)) + out_file, file_size, file_compress, broken_pipe_event)) reduce.start() # initialize jobs queue @@ -3100,45 +3111,50 @@ def hook(filename, mode): workers = [] for i in range(worker_count): extractor = Process(target=extract_process, - args=(options, i, jobs_queue, output_queue)) + args=(options, i, jobs_queue, output_queue, broken_pipe_event)) extractor.daemon = True # only live while parent process lives extractor.start() workers.append(extractor) # Mapper process - page_num = 0 - for page_data in pages_from(input): - id, revid, title, ns, catSet, page = page_data - if keepPage(ns, catSet, page, title): - # slow down - delay = 0 - if spool_length.value > max_spool_length: - # reduce to 10% - while spool_length.value > max_spool_length/10: - time.sleep(10) - delay += 10 - if delay: - logging.info('Delay %ds', delay) - job = (id, revid, title, page, page_num) - jobs_queue.put(job) # goes to any available extract_process - page_num += 1 + try: + page_num = 0 + for page_data in pages_from(input): + if broken_pipe_event.is_set(): + break + id, revid, title, ns, catSet, page = page_data + if keepPage(ns, catSet, page, title): + # slow down + delay = 0 + if spool_length.value > max_spool_length: + # reduce to 10% + while spool_length.value > max_spool_length/10: + time.sleep(10) + delay += 10 + if delay: + logging.info('Delay %ds', delay) + job = (id, revid, title, page, page_num) + # TODO if pipe is closed + try_put_until(broken_pipe_event, jobs_queue, job) # goes to any available extract_process + page_num += 1 if options.max_articles and page_num >= options.max_articles: # break # any need to cleanup before break? - page = None # free memory + page = None # free memory - input.close() + input.close() + + except KeyboardInterrupt: + logging.warn("Exiting due interrupt") - # signal termination - for _ in workers: - jobs_queue.put(None) # wait for workers to terminate for w in workers: w.join() - # signal end of work to reduce process - output_queue.put(None) - # wait for it to finish - reduce.join() + if reduce.is_alive(): + # signal end of work to reduce process + output_queue.put(None) + # wait for reduce process to finish + reduce.join() extract_duration = default_timer() - extract_start extract_rate = page_num / extract_duration @@ -3151,7 +3167,7 @@ def hook(filename, mode): # Multiprocess support -def extract_process(opts, i, jobs_queue, output_queue): +def extract_process(opts, i, jobs_queue, output_queue, broken_pipe_event): """Pull tuples of raw page content, do CPU/regex-heavy fixup, push finished text :param i: process id. :param jobs_queue: where to get jobs. @@ -3166,31 +3182,41 @@ def extract_process(opts, i, jobs_queue, output_queue): out = StringIO() # memory buffer - while True: - job = jobs_queue.get() # job is (id, title, page, page_num) - if job: - id, revid, title, page, page_num = job - try: - e = Extractor(*job[:4]) # (id, revid, title, page) - page = None # free memory - e.extract(out) - text = out.getvalue() - except: - text = '' - logging.exception('Processing page: %s %s', id, title) - - output_queue.put((page_num, text)) - out.truncate(0) - out.seek(0) - else: - logging.debug('Quit extractor') - break + try: + while not broken_pipe_event.is_set(): + job = jobs_queue.get() # job is (id, title, page, page_num) + if job: + id, revid, title, page, page_num = job + try: + e = Extractor(*job[:4]) # (id, revid, title, page) + page = None # free memory + e.extract(out) + text = out.getvalue() + except Exception: + text = '' + logging.exception('Processing page: %s %s', id, title) + + try_put_until(broken_pipe_event, output_queue, (page_num, text)) + out.truncate(0) + out.seek(0) + else: + logging.debug('Quit extractor') + break + + except KeyboardInterrupt: + logging.info('Aborting worker %d', i) + output_queue.cancel_join_thread() + jobs_queue.cancel_join_thread() + + if broken_pipe_event.is_set(): + output_queue.cancel_join_thread() + out.close() report_period = 10000 # progress report period def reduce_process(opts, output_queue, spool_length, - out_file=None, file_size=0, file_compress=True): + out_file, file_size, file_compress, broken_pipe_event)): """Pull finished article text, write series of files (or stdout) :param opts: global parameters. :param output_queue: text to be output. @@ -3217,33 +3243,44 @@ def reduce_process(opts, output_queue, spool_length, # FIXME: use a heap spool = {} # collected pages next_page = 0 # sequence numbering of page - while True: - if next_page in spool: - output.write(spool.pop(next_page).encode('utf-8')) - next_page += 1 - # tell mapper our load: - spool_length.value = len(spool) - # progress report - if next_page % report_period == 0: - interval_rate = report_period / (default_timer() - interval_start) - logging.info("Extracted %d articles (%.1f art/s)", - next_page, interval_rate) - interval_start = default_timer() - else: - # mapper puts None to signal finish - pair = output_queue.get() - if not pair: - break - page_num, text = pair - spool[page_num] = text - # tell mapper our load: - spool_length.value = len(spool) - # FIXME: if an extractor dies, process stalls; the other processes - # continue to produce pairs, filling up memory. - if len(spool) > 200: - logging.debug('Collected %d, waiting: %d, %d', len(spool), - next_page, next_page == page_num) - if output != sys.stdout: + try: + while True: + if next_page in spool: + try: + output.write(spool.pop(next_page).encode('utf-8')) + except BrokenPipeError: + # other side of pipe (like `head` or `grep`) is closed + # we can simply exit + broken_pipe_event.set() + break + + next_page += 1 + # tell mapper our load: + spool_length.value = len(spool) + # progress report + if next_page % report_period == 0: + interval_rate = report_period / (default_timer() - interval_start) + logging.info("Extracted %d articles (%.1f art/s)", + next_page, interval_rate) + interval_start = default_timer() + else: + # mapper puts None to signal finish + pair = output_queue.get() + if not pair: + break + page_num, text = pair + spool[page_num] = text + # tell mapper our load: + spool_length.value = len(spool) + # FIXME: if an extractor dies, process stalls; the other processes + # continue to produce pairs, filling up memory. + if len(spool) > 200: + logging.debug('Collected %d, waiting: %d, %d', len(spool), + next_page, next_page == page_num) + except KeyboardInterrupt: + pass + + if output != sys.stdout and not broken_pipe_event.is_set(): output.close() From 050b586518ebcb6f96cc00e783896813a8318cf9 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Tue, 7 Jan 2020 22:21:19 +0100 Subject: [PATCH 045/110] fixed the last introduced bug --- WikiExtractor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index 9b5a1704..3fd6bb23 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -3216,7 +3216,7 @@ def extract_process(opts, i, jobs_queue, output_queue, broken_pipe_event): report_period = 10000 # progress report period def reduce_process(opts, output_queue, spool_length, - out_file, file_size, file_compress, broken_pipe_event)): + out_file, file_size, file_compress, broken_pipe_event): """Pull finished article text, write series of files (or stdout) :param opts: global parameters. :param output_queue: text to be output. From a63ab3a9d9966c24737c6eb833e62da333ed9acf Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Wed, 8 Jan 2020 00:04:50 +0100 Subject: [PATCH 046/110] Rollback of code for termination Program would not terminate without this rollback (revert to previous code). Although it finished the job, but then sat idling until user termination. --- WikiExtractor.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index 3fd6bb23..10721ce9 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -3146,7 +3146,13 @@ def hook(filename, mode): except KeyboardInterrupt: logging.warn("Exiting due interrupt") - # wait for workers to terminate + # signal termination, either catch the pipe error on ctrl C or recode + for _ in workers: + jobs_queue.put(None) + + # wait for workers to terminate, this alone leaves all processes as zombies, above is needed + # it may still leave a couple of zombie processes even with tha above code - fix this + # https://docs.python.org/3/library/multiprocessing.html for w in workers: w.join() From 0b9091b7bd874602de6ba4986d0de3385063b64a Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Wed, 8 Jan 2020 12:32:36 +0100 Subject: [PATCH 047/110] --verbose and
from eric-haibin-lin:master who committed on 26 Mar 2019 --- WikiExtractor.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index 10721ce9..7424c581 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -195,6 +195,10 @@ def __eq__ (self, other): ## # Maximum count of articles (default 0 means unlimited) max_articles = 0, + + ## + # More information from program + verbose = False, ## # Whether to output HTML instead of text @@ -332,7 +336,7 @@ def get_url(uid): # ------------------------------------------------------------------------------ -selfClosingTags = ('br', 'hr', 'nobr', 'ref', 'references', 'nowiki') +selfClosingTags = ('BR', ''br', 'hr', 'nobr', 'ref', 'references', 'nowiki') # BR may not be needed placeholder_tags = {'math': 'formula', 'code': 'codice'} @@ -3342,6 +3346,8 @@ def main(): help="List of page IDs to restrict to (one per line, case-sensitive)") groupO.add_argument("--max_articles", type=int, default=0, help="maximum count of articles (default 0 means unlimited)") + groupO.add_argument("--verbose", action="store_true", + help="display extended information") groupP.add_argument("--lists", action="store_true", help="preserve lists") @@ -3402,6 +3408,7 @@ def main(): options.remove_html_tags = args.remove_html_tags options.remove_special_tokens = args.remove_special_tokens options.max_articles = args.max_articles + options.verbose = args.verbose options.expand_templates = args.no_templates options.filter_disambig_pages = args.filter_disambig_pages @@ -3442,6 +3449,8 @@ def main(): 'p', 'plaintext', 's', 'span', 'strike', 'strong', 'tt', 'u', 'var' ] + if args.verbose: + print("ignored tags = ", ignoredTags) # 'a' tag is handled separately for tag in ignoredTags: @@ -3449,6 +3458,9 @@ def main(): if args.discard_elements: options.discardElements = set(args.discard_elements.split(',')) + + if args.verbose: + print("discard elements = ", options.discardElements) FORMAT = '%(levelname)s: %(message)s' logging.basicConfig(format=FORMAT) From ea73ede54c13127af04aa87b20d86dbef76843e2 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Wed, 8 Jan 2020 20:04:14 +0100 Subject: [PATCH 048/110] bug fix + a bit of removing of debug comments --- WikiExtractor.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index 7424c581..75fd933c 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -336,7 +336,7 @@ def get_url(uid): # ------------------------------------------------------------------------------ -selfClosingTags = ('BR', ''br', 'hr', 'nobr', 'ref', 'references', 'nowiki') # BR may not be needed +selfClosingTags = ('BR', 'br', 'hr', 'nobr', 'ref', 'references', 'nowiki') # BR may not be needed placeholder_tags = {'math': 'formula', 'code': 'codice'} @@ -2578,7 +2578,6 @@ def makeInternalLink(title, label): #'\[(((?i)' + '|'.join(wgUrlProtocols) + ')' + EXT_LINK_URL_CLASS + r'+)' + '\[((' + '|'.join(wgUrlProtocols) + ')' + EXT_LINK_URL_CLASS + r'+)' + r'\s*((?:' + ANCHOR_CLASS + r'|\[\[' + ANCHOR_CLASS + r'+\]\])' + r'*?)\]', - #re.S | re.U) re.I | re.S | re.U) # A simpler alternative: # ExtLinkBracketedRegex = re.compile(r'\[(.*?)\](?!])') @@ -2587,8 +2586,6 @@ def makeInternalLink(title, label): r"""^(http://|https://)([^][<>"\x00-\x20\x7F\s]+) /([A-Za-z0-9_.,~%\-+&;#*?!=()@\x80-\xFF]+)\.(gif|png|jpg|jpeg)$""", re.I | re.X | re.S | re.U) - #/([A-Za-z0-9_.,~%\-+&;#*?!=()@\x80-\xFF]+)\.((?i)gif|png|jpg|jpeg)$""", - #re.X | re.S | re.U) def replaceExternalLinks(text): From e190f33a7fcace6cda520aeef6ca97a2ede1425b Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Wed, 8 Jan 2020 20:24:24 +0100 Subject: [PATCH 049/110] Update README.md --- README.md | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index bc567304..4f7fafc9 100644 --- a/README.md +++ b/README.md @@ -40,9 +40,9 @@ usage: WikiExtractor.py
[--remove-special-tokens] [--remove-html-tags]
[--point-separated]
[--restrict_pages_to RESTRICT_PAGES_TO]
- [--max_articles MAX_ARTICLES] [--lists] [-ns ns1,ns2]
- [--templates TEMPLATES] [--no-templates] [-r]
- [--min_text_length MIN_TEXT_LENGTH]
+ [--max_articles MAX_ARTICLES] [--verbose] [--lists]
+ [-ns ns1,ns2] [--templates TEMPLATES] [--no-templates]
+ [-r] [--min_text_length MIN_TEXT_LENGTH]
[--filter_disambig_pages] [-it abbr,b,big]
[-de gallery,timeline,noinclude] [--keep_tables]
[--processes PROCESSES] [-q] [--debug] [-a]
@@ -50,23 +50,28 @@ usage: WikiExtractor.py
[--filter_category FILTER_CATEGORY]
input -Examples (tested for "correct" output): +## Examples (tested for "correct" output) -python3 WikiExtractor.py -o wiki/test --headersfooters --titlefree --squeeze-blank wiki/wiki-20191101-pages-articles.xml +## Debug and testing (short and fast): +python3 WikiExtractor.py -o wiki/test --templates templat.txt --max_articles 10 --verbose wiki/wiki-20191101-pages-articles.xml -python3 WikiExtractor.py -o wiki/test --titlefree --squeeze-blank wiki/wiki-20191101-pages-articles.xml +## JSON (most extracted information): +python3 WikiExtractor.py -o wiki/test --filter_disambig_pages --templates templat.txt --titlefree --json --min_text_length 100 wiki/wiki-20191101-pages-articles.xml +python3 WikiExtractor.py -o wiki/test --filter_disambig_pages --templates templat.txt --titlefree --json --for-bert --min_text_length 100 wiki/wiki-20191101-pages-articles.xml -python3 WikiExtractor.py -o wiki/test --noLineAfterHeader --squeeze-blank wiki/wiki-20191101-pages-articles.xml +## Text only with "extra cleaning" (change --min_text_length to suit your use cases): +python3 WikiExtractor.py -o wiki/test --filter_disambig_pages --no_templates --for-bert --min_text_length 100 wiki/wiki-20191101-pages-articles.xml +## Other combinations: +python3 WikiExtractor.py -o wiki/test --headersfooters --titlefree --squeeze-blank wiki/wiki-20191101-pages-articles.xml +python3 WikiExtractor.py -o wiki/test --titlefree --squeeze-blank wiki/wiki-20191101-pages-articles.xml +python3 WikiExtractor.py -o wiki/test --noLineAfterHeader --squeeze-blank wiki/wiki-20191101-pages-articles.xml python3 WikiExtractor.py -o wiki/test --for-bert wiki/wiki-20191101-pages-articles.xml - python3 WikiExtractor.py -o wiki/test --filter_disambig_pages --no_templates --for-bert --min_text_length 100 wiki/wiki-20191101-pages-articles.xml - python3 WikiExtractor.py -o wiki/test --filter_disambig_pages --templates templat.txt --titlefree --json --for-bert --min_text_length 100 wiki/wiki-20191101-pages-articles.xml - python3 WikiExtractor.py -o wiki/test --filter_disambig_pages --templates templat.txt --squeeze-blank --titlefree --max_articles 10 --remove-html-tags --min_text_length 100 wiki/wiki-20191101-pages-articles.xml - +## Postprocessing After running the extractor there may be a need for cleaning the output. In linux you may use any of the following examples. Please copy all the files to a safe place first. ANY ERROR IN THE CODE WILL DESTROY YOUR TEXT. You can be sure your text will be destroyed many times before you find the right cleaning scripts.
left trim on one file: sed -i 's/^[ ]*//g' YOURTEXT
right trim on one file: sed -i 's/[ ]*$//g' YOURTEXT
@@ -80,6 +85,7 @@ Search Internet for variations and how to use with other operating systems. One For those use cases where only on large file is needed, in linux use: cat --squeeze-blank wiki/\*/\* > wiki/wiki.txt + Wikipedia Extractor: Extracts and cleans text from a Wikipedia database dump and stores output in a number of files of similar size in a given directory. From 5ab74852655bf9b1adb8178f91e4db4577fd7664 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Wed, 8 Jan 2020 20:36:57 +0100 Subject: [PATCH 050/110] Update README.md --- README.md | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 4f7fafc9..625b0f87 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # WikiExtractor [WikiExtractor.py](http://medialab.di.unipi.it/wiki/Wikipedia_Extractor) is a Python script that extracts and cleans text from a [Wikipedia database dump](http://download.wikimedia.org/). -The tool is written in Python and requires Python 2.7 or Python 3.3+ but no additional library. +The tool is written in Python and requires Python 2.7 or Python 3.3+ but no additional library. Python 2 may not work properly any longer, testing may be needed. For further information, see the [project Home Page](http://medialab.di.unipi.it/wiki/Wikipedia_Extractor) or the [Wiki](https://github.com/attardi/wikiextractor/wiki). @@ -50,26 +50,25 @@ usage: WikiExtractor.py
[--filter_category FILTER_CATEGORY]
input -## Examples (tested for "correct" output) - +# Examples (tested for "correct" output) ## Debug and testing (short and fast): python3 WikiExtractor.py -o wiki/test --templates templat.txt --max_articles 10 --verbose wiki/wiki-20191101-pages-articles.xml ## JSON (most extracted information): -python3 WikiExtractor.py -o wiki/test --filter_disambig_pages --templates templat.txt --titlefree --json --min_text_length 100 wiki/wiki-20191101-pages-articles.xml +python3 WikiExtractor.py -o wiki/test --filter_disambig_pages --templates templat.txt --titlefree --json --min_text_length 100 wiki/wiki-20191101-pages-articles.xml
python3 WikiExtractor.py -o wiki/test --filter_disambig_pages --templates templat.txt --titlefree --json --for-bert --min_text_length 100 wiki/wiki-20191101-pages-articles.xml ## Text only with "extra cleaning" (change --min_text_length to suit your use cases): -python3 WikiExtractor.py -o wiki/test --filter_disambig_pages --no_templates --for-bert --min_text_length 100 wiki/wiki-20191101-pages-articles.xml +python3 WikiExtractor.py -o wiki/test --filter_disambig_pages --no_templates --remove-html-tags --remove-special-tokens --min_text_length 100 wiki/wiki-20191101-pages-articles.xml ## Other combinations: -python3 WikiExtractor.py -o wiki/test --headersfooters --titlefree --squeeze-blank wiki/wiki-20191101-pages-articles.xml -python3 WikiExtractor.py -o wiki/test --titlefree --squeeze-blank wiki/wiki-20191101-pages-articles.xml -python3 WikiExtractor.py -o wiki/test --noLineAfterHeader --squeeze-blank wiki/wiki-20191101-pages-articles.xml -python3 WikiExtractor.py -o wiki/test --for-bert wiki/wiki-20191101-pages-articles.xml -python3 WikiExtractor.py -o wiki/test --filter_disambig_pages --no_templates --for-bert --min_text_length 100 wiki/wiki-20191101-pages-articles.xml -python3 WikiExtractor.py -o wiki/test --filter_disambig_pages --templates templat.txt --titlefree --json --for-bert --min_text_length 100 wiki/wiki-20191101-pages-articles.xml -python3 WikiExtractor.py -o wiki/test --filter_disambig_pages --templates templat.txt --squeeze-blank --titlefree --max_articles 10 --remove-html-tags --min_text_length 100 wiki/wiki-20191101-pages-articles.xml +python3 WikiExtractor.py -o wiki/test --headersfooters --titlefree --squeeze-blank wiki/wiki-20191101-pages-articles.xml
+python3 WikiExtractor.py -o wiki/test --titlefree --squeeze-blank wiki/wiki-20191101-pages-articles.xml
+python3 WikiExtractor.py -o wiki/test --noLineAfterHeader --squeeze-blank wiki/wiki-20191101-pages-articles.xml
+python3 WikiExtractor.py -o wiki/test --for-bert wiki/wiki-20191101-pages-articles.xml
+python3 WikiExtractor.py -o wiki/test --filter_disambig_pages --no_templates --for-bert --min_text_length 100 wiki/wiki-20191101-pages-articles.xml
+python3 WikiExtractor.py -o wiki/test --filter_disambig_pages --templates templat.txt --titlefree --json --for-bert --min_text_length 100 wiki/wiki-20191101-pages-articles.xml
+python3 WikiExtractor.py -o wiki/test --filter_disambig_pages --templates templat.txt --squeeze-blank --titlefree --max_articles 10 --remove-html-tags --min_text_length 100 wiki/wiki-20191101-pages-articles.xml
## Postprocessing After running the extractor there may be a need for cleaning the output. In linux you may use any of the following examples. Please copy all the files to a safe place first. ANY ERROR IN THE CODE WILL DESTROY YOUR TEXT. You can be sure your text will be destroyed many times before you find the right cleaning scripts.
From 48dc35116afd7027be54994a341b47a0faeda442 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Thu, 9 Jan 2020 13:48:45 +0100 Subject: [PATCH 051/110] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 625b0f87..57a25f1b 100644 --- a/README.md +++ b/README.md @@ -50,13 +50,13 @@ usage: WikiExtractor.py
[--filter_category FILTER_CATEGORY]
input -# Examples (tested for "correct" output) +## Examples (tested for "correct" output) ## Debug and testing (short and fast): python3 WikiExtractor.py -o wiki/test --templates templat.txt --max_articles 10 --verbose wiki/wiki-20191101-pages-articles.xml ## JSON (most extracted information): python3 WikiExtractor.py -o wiki/test --filter_disambig_pages --templates templat.txt --titlefree --json --min_text_length 100 wiki/wiki-20191101-pages-articles.xml
-python3 WikiExtractor.py -o wiki/test --filter_disambig_pages --templates templat.txt --titlefree --json --for-bert --min_text_length 100 wiki/wiki-20191101-pages-articles.xml +python3 WikiExtractor.py -o wiki/test --filter_disambig_pages --templates templat.txt --json --for-bert --min_text_length 100 wiki/wiki-20191101-pages-articles.xml ## Text only with "extra cleaning" (change --min_text_length to suit your use cases): python3 WikiExtractor.py -o wiki/test --filter_disambig_pages --no_templates --remove-html-tags --remove-special-tokens --min_text_length 100 wiki/wiki-20191101-pages-articles.xml From ff7cf4dc6667408d906f8d240492d6b2751a6290 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Thu, 9 Jan 2020 14:36:27 +0100 Subject: [PATCH 052/110] templates_only option and dropNested within and between ref tags From Xiao Ling: adds templates_only option and dropNested within and between ref tags --- WikiExtractor.py | 55 ++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 48 insertions(+), 7 deletions(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index 75fd933c..a6711433 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -21,6 +21,7 @@ # drgriffis for options.restrict_pages_to, # tutcsis for --max_articles, # michaelsilver for html.escape, +# Xiao Ling for templates_only and dropNested ref tags # # ============================================================================= # Copyright (c) 2011-2017. Giuseppe Attardi (attardi@di.unipi.it). @@ -199,6 +200,10 @@ def __eq__ (self, other): ## # More information from program verbose = False, + + ## + # Whether only parse out templates and no extraction + templates_only = False, ## # Whether to output HTML instead of text @@ -239,12 +244,16 @@ def __eq__ (self, other): log_file = None, + # xiaoling: The original authors have "small" in this discard list + # We keep small because sometimes they are used for tenure in band membership lists + # e.g., maroon 5 + discardElements = [ 'gallery', 'timeline', 'noinclude', 'pre', 'table', 'tr', 'td', 'th', 'caption', 'div', 'form', 'input', 'select', 'option', 'textarea', 'ul', 'li', 'ol', 'dl', 'dt', 'dd', 'menu', 'dir', - 'ref', 'references', 'img', 'imagemap', 'source', 'small', + 'ref', 'references', 'img', 'imagemap', 'source', # 'small', 'sub', 'sup', 'indicator', ',mapframe', 'maplink', 'score' ], ) @@ -2140,11 +2149,32 @@ def define_template(title, page): # ---------------------------------------------------------------------- def dropNested(text, openDelim, closeDelim): + """Hack: Chunk the text into ref segments "..." and dropNested braces + within a segment. + This is to prevent unbalanced braces (e.g., "{{xyz | abc={{ | opq}}") from messing up + the nesting detection. Xiaoling. """ - A matching function for nested expressions, e.g. namespaces and tables. - """ + tagRE = re.compile('(<ref.*?>)(.+?)</ref>', re.IGNORECASE) openRE = re.compile(openDelim, re.IGNORECASE) closeRE = re.compile(closeDelim, re.IGNORECASE) + + start = 0 + segments = [] + m = tagRE.search(text, start) + while m: + segments.append(dropNested1(text[start:m.start()], openRE, closeRE)) + segments.append('{}{}</ref>'.format( + m.group(1), dropNested1(m.group(2), openRE, closeRE))) + start = m.end() + m = tagRE.search(text, start) + segments.append(dropNested1(text[start:], openRE, closeRE)) + return ''.join(segments) + + +def dropNested1(text, openRE, closeRE): + """ + A matching function for nested expressions, e.g. namespaces and tables. + """ # partition text in separate blocks { } { } spans = [] # pairs (s, e) for each partition nest = 0 # nesting level @@ -2652,9 +2682,11 @@ def makeExternalImage(url, alt=''): listOpen = {'*': '
    ', '#': '
      ', ';': '
      ', ':': '
      '} listClose = {'*': '
', '#': '', ';': '', ':': ''} -listItem = {'*': '
  • %s
    • ', '#': '
  • %s', ';': '
    %s
    ', - ':': '
    %s
    '} - +#listItem = {'*': '
  • %s
    • ', '#': '
  • %s', ';': '
    %s
    ', +# ':': '
    %s
    '} +# default to li for any list items +listItem = defaultdict(lambda: '
  • %s
    • ', {'*': '
  • %s
    • ', '#': '
  • %s', ';': '
    %s
    ', + ':': '
    %s
    '}) def compact(text): """Deal with headers, lists, empty sections, residuals of tables. @@ -3064,7 +3096,7 @@ def hook(filename, mode): load_templates(file) file.close() else: - if input_file == '-': + if not options.templates_only and input_file == '-': # can't scan then reset stdin; must error w/ suggestion to specify template_file raise ValueError("to use templates with stdin dump, must supply explicit template-file") logging.info("Preprocessing '%s' to collect template definitions: this may take some time.", input_file) @@ -3074,6 +3106,12 @@ def hook(filename, mode): template_load_elapsed = default_timer() - template_load_start logging.info("Loaded %d templates in %.1fs", len(options.templates), template_load_elapsed) + if options.templates_only: + logging.info('Templates generation only. Exiting ...') + if args.verbose: + print("Templates generation only. Exiting ...") + return + # process pages logging.info("Starting page extraction from %s.", input_file) extract_start = default_timer() @@ -3345,6 +3383,8 @@ def main(): help="maximum count of articles (default 0 means unlimited)") groupO.add_argument("--verbose", action="store_true", help="display extended information") + groupP.add_argument("--templates_only", action="store_true", + help="only generates or loads templates file, no extraction.") groupP.add_argument("--lists", action="store_true", help="preserve lists") @@ -3406,6 +3446,7 @@ def main(): options.remove_special_tokens = args.remove_special_tokens options.max_articles = args.max_articles options.verbose = args.verbose + options.templates_only = args.templates_only options.expand_templates = args.no_templates options.filter_disambig_pages = args.filter_disambig_pages From 3255faea5694240e0ab0d7e52ef30a3563e1646a Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Sun, 12 Jan 2020 11:13:45 +0100 Subject: [PATCH 053/110] currently no install --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 57a25f1b..220af03a 100644 --- a/README.md +++ b/README.md @@ -24,9 +24,9 @@ In order to speed up processing: ## Installation -The script may be invoked directly, however it can be installed by doing: +Currently no installation. The script may be invoked directly. However it can be installed by doing: - (sudo) python setup.py install + (sudo) python setup.py install ## Usage The script is invoked with a Wikipedia dump file as an argument. @@ -51,17 +51,17 @@ usage: WikiExtractor.py
    input ## Examples (tested for "correct" output) -## Debug and testing (short and fast): +Debug and testing (short and fast): python3 WikiExtractor.py -o wiki/test --templates templat.txt --max_articles 10 --verbose wiki/wiki-20191101-pages-articles.xml -## JSON (most extracted information): +JSON (most extracted information): python3 WikiExtractor.py -o wiki/test --filter_disambig_pages --templates templat.txt --titlefree --json --min_text_length 100 wiki/wiki-20191101-pages-articles.xml
    python3 WikiExtractor.py -o wiki/test --filter_disambig_pages --templates templat.txt --json --for-bert --min_text_length 100 wiki/wiki-20191101-pages-articles.xml -## Text only with "extra cleaning" (change --min_text_length to suit your use cases): +Text only with "extra cleaning" (change --min_text_length to suit your use cases): python3 WikiExtractor.py -o wiki/test --filter_disambig_pages --no_templates --remove-html-tags --remove-special-tokens --min_text_length 100 wiki/wiki-20191101-pages-articles.xml -## Other combinations: +Other combinations: python3 WikiExtractor.py -o wiki/test --headersfooters --titlefree --squeeze-blank wiki/wiki-20191101-pages-articles.xml
    python3 WikiExtractor.py -o wiki/test --titlefree --squeeze-blank wiki/wiki-20191101-pages-articles.xml
    python3 WikiExtractor.py -o wiki/test --noLineAfterHeader --squeeze-blank wiki/wiki-20191101-pages-articles.xml
    @@ -70,7 +70,7 @@ python3 WikiExtractor.py -o wiki/test --filter_disambig_pages --no_templates --f python3 WikiExtractor.py -o wiki/test --filter_disambig_pages --templates templat.txt --titlefree --json --for-bert --min_text_length 100 wiki/wiki-20191101-pages-articles.xml
    python3 WikiExtractor.py -o wiki/test --filter_disambig_pages --templates templat.txt --squeeze-blank --titlefree --max_articles 10 --remove-html-tags --min_text_length 100 wiki/wiki-20191101-pages-articles.xml
    -## Postprocessing +Postprocessing After running the extractor there may be a need for cleaning the output. In linux you may use any of the following examples. Please copy all the files to a safe place first. ANY ERROR IN THE CODE WILL DESTROY YOUR TEXT. You can be sure your text will be destroyed many times before you find the right cleaning scripts.
    left trim on one file: sed -i 's/^[ ]*//g' YOURTEXT
    right trim on one file: sed -i 's/[ ]*$//g' YOURTEXT
    From eaf3d3480b1dfdf7895fe342bf7eac26ab892bc0 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Thu, 16 Jan 2020 10:31:48 +0100 Subject: [PATCH 054/110] --raw, --abstract_only from katsugeneration/wikiextractor commited by 110shimabukuro --- WikiExtractor.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/WikiExtractor.py b/WikiExtractor.py index a6711433..26a1f22f 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -22,6 +22,7 @@ # tutcsis for --max_articles, # michaelsilver for html.escape, # Xiao Ling for templates_only and dropNested ref tags +# shimabukuro for --raw and --abstract_only # # ============================================================================= # Copyright (c) 2011-2017. Giuseppe Attardi (attardi@di.unipi.it). @@ -201,6 +202,10 @@ def __eq__ (self, other): # More information from program verbose = False, + ## + # Output only abstract + abstract_only = False, + ## # Whether only parse out templates and no extraction templates_only = False, @@ -742,6 +747,8 @@ def extract(self, out): # $dom = $this->preprocessToDom( $text, $flag ); # $text = $frame->expand( $dom ); # + if options.abstract_only: + text = text.split('==')[0] text = self.transform(text) text = self.wiki2text(text) text = compact(self.clean(text)) @@ -3385,6 +3392,10 @@ def main(): help="display extended information") groupP.add_argument("--templates_only", action="store_true", help="only generates or loads templates file, no extraction.") + groupS.add_argument("--raw", action="store_true", default=False, + help="parse raw media wiki for debug") + groupS.add_argument("--abstract_only", action="store_true", default=False, + help="output text only abstract content") groupP.add_argument("--lists", action="store_true", help="preserve lists") @@ -3447,6 +3458,7 @@ def main(): options.max_articles = args.max_articles options.verbose = args.verbose options.templates_only = args.templates_only + options.abstract_only = args.abstract_only options.expand_templates = args.no_templates options.filter_disambig_pages = args.filter_disambig_pages @@ -3512,6 +3524,12 @@ def main(): if not options.keepLinks: ignoreTag('a') + + if args.raw: + file = fileinput.FileInput(input_file, openhook=fileinput.hook_compressed) + Extractor(0, 0, "raw", file).extract(open("raw.json", "w")) + return + # sharing cache of parser templates is too slow: # manager = Manager() From 6088107253522db4d4f80f05f51645a01e761fca Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Thu, 16 Jan 2020 10:41:55 +0100 Subject: [PATCH 055/110] Update README.md --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index 220af03a..84f57abf 100644 --- a/README.md +++ b/README.md @@ -24,9 +24,7 @@ In order to speed up processing: ## Installation -Currently no installation. The script may be invoked directly. However it can be installed by doing: - - (sudo) python setup.py install +Currently no installation. The script may be invoked directly. ## Usage The script is invoked with a Wikipedia dump file as an argument. From d831277b5756434901fd0db85de5cfd619649c1e Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Thu, 16 Jan 2020 21:06:49 +0100 Subject: [PATCH 056/110] bugfixes --- WikiExtractor.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index 26a1f22f..182b9262 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -77,6 +77,7 @@ import re # TODO use regex when it will be standard import time import json +from collections import defaultdict import queue from io import StringIO from multiprocessing import Event, Queue, Process, Value, cpu_count @@ -209,6 +210,10 @@ def __eq__ (self, other): ## # Whether only parse out templates and no extraction templates_only = False, + + ## + # + raw = False, ## # Whether to output HTML instead of text @@ -3392,9 +3397,9 @@ def main(): help="display extended information") groupP.add_argument("--templates_only", action="store_true", help="only generates or loads templates file, no extraction.") - groupS.add_argument("--raw", action="store_true", default=False, + groupP.add_argument("--raw", action="store_true", help="parse raw media wiki for debug") - groupS.add_argument("--abstract_only", action="store_true", default=False, + groupP.add_argument("--abstract_only", action="store_true", help="output text only abstract content") groupP.add_argument("--lists", action="store_true", @@ -3459,6 +3464,7 @@ def main(): options.verbose = args.verbose options.templates_only = args.templates_only options.abstract_only = args.abstract_only + options.raw = args.raw options.expand_templates = args.no_templates options.filter_disambig_pages = args.filter_disambig_pages @@ -3525,7 +3531,7 @@ def main(): if not options.keepLinks: ignoreTag('a') - if args.raw: + if options.raw: file = fileinput.FileInput(input_file, openhook=fileinput.hook_compressed) Extractor(0, 0, "raw", file).extract(open("raw.json", "w")) return From a3cc0363d916bf958994ce8e66aae4bc30b5658f Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Sun, 19 Jan 2020 22:49:10 +0100 Subject: [PATCH 057/110] fixed breaking errors TypeError: %c requires int or char TypeError: write() argument must be str, not bytes Solution existed in WikiExtractor --- cirrus-extract.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cirrus-extract.py b/cirrus-extract.py index 895970da..2a92d8bf 100755 --- a/cirrus-extract.py +++ b/cirrus-extract.py @@ -72,7 +72,7 @@ def next(self): def _dirname(self): char1 = self.dir_index % 26 - char2 = self.dir_index / 26 % 26 + char2 = self.dir_index // 26 % 26 return os.path.join(self.path_name, '%c%c' % (ord('A') + char2, ord('A') + char1)) def _filepath(self): @@ -85,7 +85,7 @@ class OutputSplitter(object): def __init__(self, nextFile, max_file_size=0, compress=True): """ - :param nextfile: a NextFile object from which to obtain filenames + :param nextFile: a NextFile object from which to obtain filenames to use. :param max_file_size: the maximum size of each file. :para compress: whether to write data with bzip compression. @@ -111,7 +111,7 @@ def open(self, filename): if self.compress: return bz2.BZ2File(filename + '.bz2', 'w') else: - return open(filename, 'w') + return open(filename, 'wb') # ---------------------------------------------------------------------- From 704739285267ed6f2d414fb506eefbb86ca1f357 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Sun, 19 Jan 2020 22:52:39 +0100 Subject: [PATCH 058/110] Example on cirrus dump --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 84f57abf..16feb2ca 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,8 @@ For further information, see the [project Home Page](http://medialab.di.unipi.it `cirrus-extractor.py` is a version of the script that performs extraction from a Wikipedia Cirrus dump. Cirrus dumps contain text with already expanded templates. +Example: python3 cirrus-extract.py -o wiki/test wiki/wiki-20191104-cirrussearch-content.json.gz + Cirrus dumps are available at: [cirrussearch](http://dumps.wikimedia.org/other/cirrussearch/). From de50af02e89441b1848422e05ae5cb3740baa9ba Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Mon, 20 Jan 2020 00:33:23 +0100 Subject: [PATCH 059/110] text only option --- cirrus-extract.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/cirrus-extract.py b/cirrus-extract.py index 2a92d8bf..426d0cad 100755 --- a/cirrus-extract.py +++ b/cirrus-extract.py @@ -47,6 +47,7 @@ urlbase = 'http://it.wikipedia.org/' + # ---------------------------------------------------------------------- class NextFile(object): @@ -136,7 +137,7 @@ def extract(self, out): out.write('\n') out.write(footer) -def process_dump(input_file, out_file, file_size, file_compress): +def process_dump(input_file, out_file, file_size, file_compress, text_only): """ :param input_file: name of the wikipedia dump file; '-' to read from stdin :param out_file: directory where to store extracted data, or '-' for stdout @@ -179,8 +180,12 @@ def process_dump(input_file, out_file, file_size, file_compress): text = re.sub(r' \^ .*', '', text) url = urlbase + 'wiki?curid=' + id header = '\n' % (id, url, title, language, revision) - page = header + title + '\n\n' + text + '\n\n' + if not text_only: + page = header + title + '\n\n' + text + '\n\n' + else: + page = text + '\n\n' output.write(page.encode('utf-8')) + # ---------------------------------------------------------------------- @@ -201,6 +206,8 @@ def main(): metavar="n[KMG]") groupO.add_argument("-c", "--compress", action="store_true", help="compress output files using bzip") + groupO.add_argument("-t", "--text", action="store_true", + help="text only") groupP = parser.add_argument_group('Processing') groupP.add_argument("-ns", "--namespaces", default="", metavar="ns1,ns2", @@ -215,6 +222,7 @@ def main(): args = parser.parse_args() + try: power = 'kmg'.find(args.bytes[-1].lower()) + 1 file_size = int(args.bytes[:-1]) * 1024 ** power @@ -241,7 +249,8 @@ def main(): logging.error('Could not create: %s', output_path) return - process_dump(input_file, output_path, file_size, args.compress) + + process_dump(input_file, output_path, file_size, args.compress, args.text) if __name__ == '__main__': From 7b8a9d1cea7268e651458931662843e13afab2a4 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Mon, 20 Jan 2020 00:44:16 +0100 Subject: [PATCH 060/110] Example on text only --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 16feb2ca..e0916d56 100644 --- a/README.md +++ b/README.md @@ -8,9 +8,13 @@ For further information, see the [project Home Page](http://medialab.di.unipi.it # Wikipedia Cirrus Extractor `cirrus-extractor.py` is a version of the script that performs extraction from a Wikipedia Cirrus dump. -Cirrus dumps contain text with already expanded templates. +Cirrus dumps contain text with already expanded templates. The Cirrus extractor does not suffer fron inadequate template expansion. Until WikiExtractors template expansion has been fixed this may be used instead. -Example: python3 cirrus-extract.py -o wiki/test wiki/wiki-20191104-cirrussearch-content.json.gz +Examples:
    +json output: python3 cirrus-extract.py -o wiki/test wiki/wiki-20191104-cirrussearch-content.json.gz
    +text output: python3 cirrus-extract.py -o wiki/test -t wiki/wiki-20191104-cirrussearch-content.json.gz + +Text output is without titles, etc. It contains only the article texts sepated with empty lines. Cirrus dumps are available at: [cirrussearch](http://dumps.wikimedia.org/other/cirrussearch/). From e399a97b8459da020be4f39a1f4e081fb2bc3570 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Mon, 20 Jan 2020 12:58:19 +0100 Subject: [PATCH 061/110] Remove code making gzip conditional - for future needs. --- WikiExtractor.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index 182b9262..d7d2ee68 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -65,6 +65,7 @@ import sys import argparse import bz2 +import gzip import codecs import sys if sys.version_info < (3, 2): @@ -3052,10 +3053,10 @@ def hook_compressed_encoded(encoding): def hook(filename, mode): ext = os.path.splitext(filename)[1] if ext == '.gz': - import gzip + #import gzip #line to be removed, do not make gzip or bz2 local and conditional return gzip.open(filename, mode, encoding=encoding) elif ext == '.bz2': - import bz2 + #import bz2 #line to be removed, do not make gzip or bz2 local and conditional return bz2.open(filename, mode, encoding=encoding) else: return open(filename, mode, encoding=encoding) From e3657c12e46bbc67e808cc4f5c4a64e93eef56be Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Mon, 20 Jan 2020 13:15:53 +0100 Subject: [PATCH 062/110] Add urlbase to verbosity if --verbose. For testing puposes. --- WikiExtractor.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/WikiExtractor.py b/WikiExtractor.py index d7d2ee68..d2c1f521 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -3081,6 +3081,8 @@ def hook(filename, mode): # /mediawiki/siteinfo/base base = m.group(3) options.urlbase = base[:base.rfind("/")] + if args.verbose: + print("urlbase = ", options.urlbase) elif tag == 'namespace': mk = keyRE.search(line) if mk: From d875198d08c0b25396e2a4c6fb1299cfac19eace Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Tue, 21 Jan 2020 00:45:34 +0100 Subject: [PATCH 063/110] only one space before caret to catch malformed tags --- cirrus-extract.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cirrus-extract.py b/cirrus-extract.py index 426d0cad..a2972642 100755 --- a/cirrus-extract.py +++ b/cirrus-extract.py @@ -177,7 +177,7 @@ def process_dump(input_file, out_file, file_size, file_compress, text_only): text = content['text'] # drop references: # ^ The Penguin Dictionary - text = re.sub(r' \^ .*', '', text) + text = re.sub(r' \^ .*', '', text) # only one space before caret to catch malformed tags url = urlbase + 'wiki?curid=' + id header = '\n' % (id, url, title, language, revision) if not text_only: From c4321d01e09a4e0d5196a41ca54916721735b77b Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Tue, 21 Jan 2020 20:38:21 +0100 Subject: [PATCH 064/110] remove broken sentences at the end of articles, caused by previous filter on caret. also remove articles that only consisted of one broken sentence. --- cirrus-extract.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/cirrus-extract.py b/cirrus-extract.py index a2972642..bf7d236b 100755 --- a/cirrus-extract.py +++ b/cirrus-extract.py @@ -178,13 +178,17 @@ def process_dump(input_file, out_file, file_size, file_compress, text_only): # drop references: # ^ The Penguin Dictionary text = re.sub(r' \^ .*', '', text) # only one space before caret to catch malformed tags - url = urlbase + 'wiki?curid=' + id - header = '\n' % (id, url, title, language, revision) - if not text_only: - page = header + title + '\n\n' + text + '\n\n' - else: - page = text + '\n\n' - output.write(page.encode('utf-8')) + text = re.sub(r'\. [^.]*$', '.', text) # remove incomplete last sentence + text = re.sub(r'^[^.]*$', '', text) # remove incomplete sentence, even if only sentence in article + if text != "" and text != " ": # do not create empty articles + url = urlbase + 'wiki?curid=' + id + header = '\n' % (id, url, title, language, revision) + if not text_only: + page = header + title + '\n\n' + text + '\n\n' + else: + page = text + '\n\n' + output.write(page.encode('utf-8')) + page = "" # ---------------------------------------------------------------------- From 8a78aea725b74209be4d29034e087e2a69f37f19 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Thu, 23 Jan 2020 13:00:11 +0100 Subject: [PATCH 065/110] --sentences and --raw Only complete point separated sentences. No filtering. --- cirrus-extract.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/cirrus-extract.py b/cirrus-extract.py index bf7d236b..ec3d15ae 100755 --- a/cirrus-extract.py +++ b/cirrus-extract.py @@ -137,7 +137,7 @@ def extract(self, out): out.write('\n') out.write(footer) -def process_dump(input_file, out_file, file_size, file_compress, text_only): +def process_dump(input_file, out_file, file_size, file_compress, text_only, sentences_only, raw_only): """ :param input_file: name of the wikipedia dump file; '-' to read from stdin :param out_file: directory where to store extracted data, or '-' for stdout @@ -175,11 +175,13 @@ def process_dump(input_file, out_file, file_size, file_compress, text_only): if type == 'page' and content['namespace'] == 0: title = content['title'] text = content['text'] - # drop references: - # ^ The Penguin Dictionary - text = re.sub(r' \^ .*', '', text) # only one space before caret to catch malformed tags - text = re.sub(r'\. [^.]*$', '.', text) # remove incomplete last sentence - text = re.sub(r'^[^.]*$', '', text) # remove incomplete sentence, even if only sentence in article + if not raw_only: + # drop references: + # ^ The Penguin Dictionary + text = re.sub(r' \^ .*', '', text) # only one space before caret to catch malformed tags + if sentences_only: + text = re.sub(r'\. [^.]*$', '.', text) # remove incomplete last sentence + text = re.sub(r'^[^.]*$', '', text) # remove incomplete sentence, even if only sentence in article if text != "" and text != " ": # do not create empty articles url = urlbase + 'wiki?curid=' + id header = '\n' % (id, url, title, language, revision) @@ -212,7 +214,11 @@ def main(): help="compress output files using bzip") groupO.add_argument("-t", "--text", action="store_true", help="text only") - + groupO.add_argument("-s", "--sentences", action="store_true", + help="Only complete point separated sentences.") + groupO.add_argument("-r", "--raw", action="store_true", + help="No filtering.") + groupP = parser.add_argument_group('Processing') groupP.add_argument("-ns", "--namespaces", default="", metavar="ns1,ns2", help="accepted namespaces") @@ -254,7 +260,7 @@ def main(): return - process_dump(input_file, output_path, file_size, args.compress, args.text) + process_dump(input_file, output_path, file_size, args.compress, args.text, args.sentences, args.raw) if __name__ == '__main__': From 24e6e85d95aaea6f72cda483030d760c25fe3e56 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Thu, 23 Jan 2020 20:29:15 +0100 Subject: [PATCH 066/110] fix tab error TabError: inconsistent use of tabs and spaces in indentation --- cirrus-extract.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cirrus-extract.py b/cirrus-extract.py index ec3d15ae..4eca0c89 100755 --- a/cirrus-extract.py +++ b/cirrus-extract.py @@ -175,7 +175,7 @@ def process_dump(input_file, out_file, file_size, file_compress, text_only, sent if type == 'page' and content['namespace'] == 0: title = content['title'] text = content['text'] - if not raw_only: + if not raw_only: # drop references: # ^ The Penguin Dictionary text = re.sub(r' \^ .*', '', text) # only one space before caret to catch malformed tags From 6c64e75411799806621f482dac27e9ab51c4f4e3 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Thu, 23 Jan 2020 23:18:58 +0100 Subject: [PATCH 067/110] --sentences, now at least 2 per article Only at least two complete point separated sentences in every article. And better filtering. --- cirrus-extract.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cirrus-extract.py b/cirrus-extract.py index 4eca0c89..e2c57da4 100755 --- a/cirrus-extract.py +++ b/cirrus-extract.py @@ -179,9 +179,12 @@ def process_dump(input_file, out_file, file_size, file_compress, text_only, sent # drop references: # ^ The Penguin Dictionary text = re.sub(r' \^ .*', '', text) # only one space before caret to catch malformed tags + text = re.sub(r'\s*$', '', text) # remove all trailing white space if sentences_only: - text = re.sub(r'\. [^.]*$', '.', text) # remove incomplete last sentence + text = re.sub(r'\. [^.]*$', '.', text) # remove incomplete last sentence (replaced by below?) + text = re.sub(r'\.\s(.(?!\.\s))*[^.]$', '.', text) # remove incomplete last sentence, no dot and space found as separator and no ending dot text = re.sub(r'^[^.]*$', '', text) # remove incomplete sentence, even if only sentence in article + text = re.sub(r'^(.(?!\.\s))*$', '', text) # remove if only one sentence in article, no dot and space found as separator if text != "" and text != " ": # do not create empty articles url = urlbase + 'wiki?curid=' + id header = '\n' % (id, url, title, language, revision) @@ -215,7 +218,7 @@ def main(): groupO.add_argument("-t", "--text", action="store_true", help="text only") groupO.add_argument("-s", "--sentences", action="store_true", - help="Only complete point separated sentences.") + help="Only at least two complete point separated sentences.") groupO.add_argument("-r", "--raw", action="store_true", help="No filtering.") From 00a322ec7ed1143a7017315ad41240e8ccccafdc Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Fri, 24 Jan 2020 10:44:47 +0100 Subject: [PATCH 068/110] Update README.md --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e0916d56..0975135c 100644 --- a/README.md +++ b/README.md @@ -8,13 +8,17 @@ For further information, see the [project Home Page](http://medialab.di.unipi.it # Wikipedia Cirrus Extractor `cirrus-extractor.py` is a version of the script that performs extraction from a Wikipedia Cirrus dump. -Cirrus dumps contain text with already expanded templates. The Cirrus extractor does not suffer fron inadequate template expansion. Until WikiExtractors template expansion has been fixed this may be used instead. +Cirrus dumps contain text with already expanded templates. The Cirrus extractor does not suffer fron somewhat inadequate template expansion. Until WikiExtractors template expansion has been fixed this may be used instead. Examples:
    json output: python3 cirrus-extract.py -o wiki/test wiki/wiki-20191104-cirrussearch-content.json.gz
    text output: python3 cirrus-extract.py -o wiki/test -t wiki/wiki-20191104-cirrussearch-content.json.gz -Text output is without titles, etc. It contains only the article texts sepated with empty lines. +Text output is without titles, etc. It contains only the article texts separated with empty lines. + +Some additional switches are:
    +--raw : basically no cleaning.
    +--sentences : basic sentence based cleaning, based on dot and space, producing at least two sentences ending with a dot - but can be tricked by dots in names, etc. Cirrus dumps are available at: [cirrussearch](http://dumps.wikimedia.org/other/cirrussearch/). From 8bb7497a83889a4a030ebeb8c24b7066c42fe1d2 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Sat, 25 Jan 2020 00:09:34 +0100 Subject: [PATCH 069/110] bug fix --- WikiExtractor.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index d2c1f521..08bbcb9b 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -3081,7 +3081,7 @@ def hook(filename, mode): # /mediawiki/siteinfo/base base = m.group(3) options.urlbase = base[:base.rfind("/")] - if args.verbose: + if options.verbose: print("urlbase = ", options.urlbase) elif tag == 'namespace': mk = keyRE.search(line) @@ -3123,7 +3123,7 @@ def hook(filename, mode): if options.templates_only: logging.info('Templates generation only. Exiting ...') - if args.verbose: + if options.verbose: print("Templates generation only. Exiting ...") return @@ -3508,7 +3508,7 @@ def main(): 'p', 'plaintext', 's', 'span', 'strike', 'strong', 'tt', 'u', 'var' ] - if args.verbose: + if options.verbose: print("ignored tags = ", ignoredTags) # 'a' tag is handled separately @@ -3518,7 +3518,7 @@ def main(): if args.discard_elements: options.discardElements = set(args.discard_elements.split(',')) - if args.verbose: + if options.verbose: print("discard elements = ", options.discardElements) FORMAT = '%(levelname)s: %(message)s' From 3b8bb7035dcb7c980c226862de946e4ebfc8d555 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Sat, 25 Jan 2020 14:18:58 +0100 Subject: [PATCH 070/110] Make visible what is not supported currently as far as I know. Now to implement some of them... --- WikiExtractor.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/WikiExtractor.py b/WikiExtractor.py index 08bbcb9b..12386d56 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -2030,6 +2030,12 @@ def sharp_invoke(module, function, args): '#titleparts': lambda *args: '', # not supported + '#dateformat': lambda *args: '', # not supported #HjalmarrSv + + '#formatdate': lambda *args: '', # not supported #HjalmarrSv + + '#tag': lambda *args: '', # not supported #HjalmarrSv + # This function is used in some pages to construct links # http://meta.wikimedia.org/wiki/Help:URL 'urlencode': lambda extr, string, *rest: quote(string.encode('utf-8')), @@ -2044,6 +2050,19 @@ def sharp_invoke(module, function, args): 'int': lambda extr, string, *rest: text_type(int(string)), + 'formatnum': lambda *args: '', # not supported #HjalmarrSv + + 'gender': lambda *args: '', # not supported #HjalmarrSv + + 'plural': lambda *args: '', # not supported #HjalmarrSv + + 'padleft': lambda *args: '', # not supported #HjalmarrSv + + 'padright': lambda *args: '', # not supported #HjalmarrSv + + 'as of': lambda *args: '', # not supported #HjalmarrSv #this may be the wrong place in code for this function + + 'plural': lambda *args: '', # not supported #HjalmarrSv } From 6d755bff3da999c65e82e857cb48e74912e3ea9d Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Sun, 26 Jan 2020 00:59:20 +0100 Subject: [PATCH 071/110] Added switch: '__NOGLOBAL__' --- WikiExtractor.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index 12386d56..2be6c0ce 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -1769,7 +1769,9 @@ def __getitem__(self, name): def __setitem__(self, name, value): self.values[name] = value + # https://www.mediawiki.org/wiki/Help:Magic_words#Behavior_switches + # switches = ( '__NOTOC__', '__FORCETOC__', @@ -1789,7 +1791,8 @@ def __setitem__(self, name, value): '__INDEX__', '__NOINDEX__', '__STATICREDIRECT__', - '__DISAMBIG__' + '__DISAMBIG__', + '__NOGLOBAL__' #added 200126 ) @@ -2065,7 +2068,7 @@ def sharp_invoke(module, function, args): 'plural': lambda *args: '', # not supported #HjalmarrSv } - +#functionName<-title[:colon], args<-parts , extractor<-self def callParserFunction(functionName, args, extractor): """ Parser functions have similar syntax as templates, except that From 49e3cd837485e40d156135251f049d546df70e6f Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Mon, 27 Jan 2020 14:05:26 +0100 Subject: [PATCH 072/110] Added {{grammar: ...}} not supported as of yet --- WikiExtractor.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index 2be6c0ce..89c43476 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -2055,8 +2055,9 @@ def sharp_invoke(module, function, args): 'formatnum': lambda *args: '', # not supported #HjalmarrSv - 'gender': lambda *args: '', # not supported #HjalmarrSv - + 'gender': lambda *args: '', # not supported #HjalmarrSv: not relevant here, because "such as in "inform the user on his/her talk page", which is better made "inform the user on {{GENDER:$1|his|her|their}} talk page" + # https://www.mediawiki.org/wiki/Localisation#%E2%80%A6on_use_context_inside_sentences_via_GRAMMAR + 'plural': lambda *args: '', # not supported #HjalmarrSv 'padleft': lambda *args: '', # not supported #HjalmarrSv @@ -2066,6 +2067,8 @@ def sharp_invoke(module, function, args): 'as of': lambda *args: '', # not supported #HjalmarrSv #this may be the wrong place in code for this function 'plural': lambda *args: '', # not supported #HjalmarrSv + + 'grammar': lambda *args: '', # not supported #HjalmarrSv } #functionName<-title[:colon], args<-parts , extractor<-self From 34ee0dc88c32f2c3ff58fb3c9f99e1e20427c0a3 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Mon, 27 Jan 2020 20:32:43 +0100 Subject: [PATCH 073/110] disambiguate command line parameter --- WikiExtractor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index 89c43476..15ec69e4 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -3423,7 +3423,7 @@ def main(): help="maximum count of articles (default 0 means unlimited)") groupO.add_argument("--verbose", action="store_true", help="display extended information") - groupP.add_argument("--templates_only", action="store_true", + groupP.add_argument("--only_templates", action="store_true", help="only generates or loads templates file, no extraction.") groupP.add_argument("--raw", action="store_true", help="parse raw media wiki for debug") @@ -3490,7 +3490,7 @@ def main(): options.remove_special_tokens = args.remove_special_tokens options.max_articles = args.max_articles options.verbose = args.verbose - options.templates_only = args.templates_only + options.templates_only = args.only_templates options.abstract_only = args.abstract_only options.raw = args.raw From fd1c7caeb1ee123753350e27a2200c68e35074f4 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Mon, 27 Jan 2020 20:34:39 +0100 Subject: [PATCH 074/110] Update WikiExtractor.py --- WikiExtractor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index 15ec69e4..10f9eabc 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -210,7 +210,7 @@ def __eq__ (self, other): ## # Whether only parse out templates and no extraction - templates_only = False, + only_templates = False, ## # From bbdb4a80a7c87b7c5107d8899179f0056fb2b698 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Mon, 27 Jan 2020 22:26:29 +0100 Subject: [PATCH 075/110] revert on 'small' discarding is better, for now, than keeping --- WikiExtractor.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index 10f9eabc..3c411d67 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -257,14 +257,15 @@ def __eq__ (self, other): # xiaoling: The original authors have "small" in this discard list # We keep small because sometimes they are used for tenure in band membership lists - # e.g., maroon 5 + # e.g., maroon 5. + # Edit by HjalmarrSv: we do not keep small bacause it introduces to much unwanted items (that could be cleaned another way) discardElements = [ 'gallery', 'timeline', 'noinclude', 'pre', 'table', 'tr', 'td', 'th', 'caption', 'div', 'form', 'input', 'select', 'option', 'textarea', 'ul', 'li', 'ol', 'dl', 'dt', 'dd', 'menu', 'dir', - 'ref', 'references', 'img', 'imagemap', 'source', # 'small', + 'ref', 'references', 'img', 'imagemap', 'source', 'small', 'sub', 'sup', 'indicator', ',mapframe', 'maplink', 'score' ], ) From 62d2b0648da3e4e4b3ce7fdc3e4783d9cef374fb Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Tue, 28 Jan 2020 19:51:12 +0100 Subject: [PATCH 076/110] basic support for 'formatnum' and one change from space to whitespace, just in case. --- WikiExtractor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index 3c411d67..3df42958 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -1038,7 +1038,7 @@ def templateParams(self, parameters): # The '=' might occurr within an HTML attribute: # "<ref name=value" # but we stop at first. - m = re.match(' *([^=]*?) *?=(.*)', param, re.DOTALL) + m = re.match(r'\s*([^=]*)=(.*)', param, re.DOTALL) #m = re.match(' *([^=]*?) *?=(.*)', param, re.DOTALL) if m: # This is a named parameter. This case also handles parameter # assignments like "2=xxx", where the number of an unnamed @@ -2054,7 +2054,7 @@ def sharp_invoke(module, function, args): 'int': lambda extr, string, *rest: text_type(int(string)), - 'formatnum': lambda *args: '', # not supported #HjalmarrSv + 'formatnum': lambda extr, string, *rest: string, #HjalmarrSv: dot is decimal separator! 'gender': lambda *args: '', # not supported #HjalmarrSv: not relevant here, because "such as in "inform the user on his/her talk page", which is better made "inform the user on {{GENDER:$1|his|her|their}} talk page" # https://www.mediawiki.org/wiki/Localisation#%E2%80%A6on_use_context_inside_sentences_via_GRAMMAR From 7ccd3f81f16fd90a5442c7844586683db60281f5 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Tue, 28 Jan 2020 20:15:53 +0100 Subject: [PATCH 077/110] added example --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 0975135c..a7d197c5 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,8 @@ usage: WikiExtractor.py
    ## Examples (tested for "correct" output) Debug and testing (short and fast): -python3 WikiExtractor.py -o wiki/test --templates templat.txt --max_articles 10 --verbose wiki/wiki-20191101-pages-articles.xml +python3 WikiExtractor.py -o wiki/test --templates templat.txt --max_articles 10 --verbose wiki/wiki-20191101-pages-articles.xml
    +Debug and testing (more info on screen and a log): python3 WikiExtractor.py -o wiki/test --templates templat.txt --max_articles 10 --verbose --debug --log_file log.txt wiki/wiki-20191101-pages-articles.xml JSON (most extracted information): python3 WikiExtractor.py -o wiki/test --filter_disambig_pages --templates templat.txt --titlefree --json --min_text_length 100 wiki/wiki-20191101-pages-articles.xml
    From 63278c7d29b7349fec1c1052c322694d188c6e3c Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Tue, 28 Jan 2020 21:31:08 +0100 Subject: [PATCH 078/110] refresh comments --- WikiExtractor.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index 3df42958..b5847d5d 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -1677,7 +1677,7 @@ class MagicWords(object): """ One copy in each Extractor. - @see https://doc.wikimedia.org/mediawiki-core/master/php/MagicWord_8php_source.html - invalid old link + https://doc.wikimedia.org/mediawiki-core/master/php/MagicWordFactory_8php_source.html https://www.mediawiki.org/wiki/Manual:Magic_words """ @@ -1757,7 +1757,7 @@ class MagicWords(object): 'localtimestamp', 'directionmark', 'contentlanguage', - 'pagelanguage', #added 191220 + 'pagelanguage', 'numberofadmins', 'cascadingsources', ] @@ -1777,18 +1777,18 @@ def __setitem__(self, name, value): '__NOTOC__', '__FORCETOC__', '__TOC__', - '__NOEDITSECTION__', #added 191229 + '__NOEDITSECTION__', '__NEWSECTIONLINK__', '__NONEWSECTIONLINK__', '__NOGALLERY__', '__HIDDENCAT__', - '__EXPECTUNUSEDCATEGORY__', #added 191229 + '__EXPECTUNUSEDCATEGORY__', '__NOCONTENTCONVERT__', '__NOCC__', '__NOTITLECONVERT__', '__NOTC__', - '__START__', - '__END__', + '__START__', # keep for now, but removed in r1695 and completely removed in r24784 + '__END__', # keep for now, but removed in 19213. '__INDEX__', '__NOINDEX__', '__STATICREDIRECT__', From 0950b893e1ba8a48e1e9663070dd154836383f56 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Wed, 29 Jan 2020 13:06:21 +0100 Subject: [PATCH 079/110] --decimalcomma; decimal separator comma (,) for formatnum Not tested yet --- WikiExtractor.py | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index b5847d5d..2e057055 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -213,8 +213,13 @@ def __eq__ (self, other): only_templates = False, ## - # + # As unprocessed as possible raw = False, + + ## + # In formatnum, dot is standard, e.g. 1.5. If decimalkomma is set, use comma as decimal separator instead, e.g. 1,5. + # Does not have effect outside parsing. Text and parsed text may differ if care is not taken. + decimalcomma = False, ## # Whether to output HTML instead of text @@ -1823,6 +1828,17 @@ def lcfirst(string): return string.lower() else: return '' + + +def formatnum(string): #function for parsing 'formatnum:xxx.yyy|zzz' where zzz can be '|R', '|NOSEP': https://www.mediawiki.org/wiki/Help:Magic_words + if options.decimalcomma: + newstring = re.sub(r'^(?P[^\.]*).(?P\d*).*','\g,\g', string) #a point is found, replace witk comma and clear rest after fraction + # fixme: for the special case that comma is a thousands separator that needs to be removed + elif not options.decimalcomma: + newstring = re.sub(r'^(?P[^\.]*).(?P\d*).*','\g.\g') # a point is found, clear rest after fraction + else: + newstring = re.sub(r'^(?P[^|]*).*','\g') # look for arguments (after pipe (|)), remove them + return newstring # if no dot or no pipe then following should be true: newstring == string def fullyQualifiedTemplateTitle(templateTitle): @@ -1895,9 +1911,9 @@ def __call__(self, value1, value2): ROUND = Infix(lambda x, y: round(x, y)) - from math import floor, ceil, pi, e, trunc, exp, log as ln, sin, cos, tan, asin, acos, atan + def sharp_expr(extr, expr): """Tries converting a lua expr into a Python expr.""" @@ -2054,7 +2070,7 @@ def sharp_invoke(module, function, args): 'int': lambda extr, string, *rest: text_type(int(string)), - 'formatnum': lambda extr, string, *rest: string, #HjalmarrSv: dot is decimal separator! + 'formatnum': lambda extr, string, *rest: formatnum(string), # was: 'formatnum': lambda extr, string, *rest: string, #HjalmarrSv: dot is decimal separator! 'gender': lambda *args: '', # not supported #HjalmarrSv: not relevant here, because "such as in "inform the user on his/her talk page", which is better made "inform the user on {{GENDER:$1|his|her|their}} talk page" # https://www.mediawiki.org/wiki/Localisation#%E2%80%A6on_use_context_inside_sentences_via_GRAMMAR @@ -3430,6 +3446,8 @@ def main(): help="parse raw media wiki for debug") groupP.add_argument("--abstract_only", action="store_true", help="output text only abstract content") + groupP.add_argument("--decimalcomma", action="store_true", + help="use comma, instead of dot, as decimal separator") groupP.add_argument("--lists", action="store_true", help="preserve lists") @@ -3494,6 +3512,7 @@ def main(): options.templates_only = args.only_templates options.abstract_only = args.abstract_only options.raw = args.raw + options.decimalcomma = args.decimalcomma options.expand_templates = args.no_templates options.filter_disambig_pages = args.filter_disambig_pages From 89104740a10bde9d1262a3af73f823070211df4f Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Wed, 29 Jan 2020 13:10:34 +0100 Subject: [PATCH 080/110] bug fix --- WikiExtractor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index 2e057055..46215611 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -1835,9 +1835,9 @@ def formatnum(string): #function for parsing 'formatnum:xxx.yyy|zzz' where zzz c newstring = re.sub(r'^(?P[^\.]*).(?P\d*).*','\g,\g', string) #a point is found, replace witk comma and clear rest after fraction # fixme: for the special case that comma is a thousands separator that needs to be removed elif not options.decimalcomma: - newstring = re.sub(r'^(?P[^\.]*).(?P\d*).*','\g.\g') # a point is found, clear rest after fraction + newstring = re.sub(r'^(?P[^\.]*).(?P\d*).*','\g.\g', string) # a point is found, clear rest after fraction else: - newstring = re.sub(r'^(?P[^|]*).*','\g') # look for arguments (after pipe (|)), remove them + newstring = re.sub(r'^(?P[^|]*).*','\g', string) # look for arguments (after pipe (|)), remove them return newstring # if no dot or no pipe then following should be true: newstring == string From a72954668928dbd8a093938783eea8a72d87cd33 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Wed, 29 Jan 2020 21:40:27 +0100 Subject: [PATCH 081/110] a little better formatnum overcomplicated maybe could be just: if choice then exchange all . for , --- WikiExtractor.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index 46215611..a36bf20c 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -1831,14 +1831,16 @@ def lcfirst(string): def formatnum(string): #function for parsing 'formatnum:xxx.yyy|zzz' where zzz can be '|R', '|NOSEP': https://www.mediawiki.org/wiki/Help:Magic_words + string = string.strip() + newstring = string if options.decimalcomma: - newstring = re.sub(r'^(?P[^\.]*).(?P\d*).*','\g,\g', string) #a point is found, replace witk comma and clear rest after fraction + newstring = re.sub(r'^(?P\d*)\.(?P\d*).*$','\g,\g', string) # fixme: for the special case that comma is a thousands separator that needs to be removed - elif not options.decimalcomma: - newstring = re.sub(r'^(?P[^\.]*).(?P\d*).*','\g.\g', string) # a point is found, clear rest after fraction - else: - newstring = re.sub(r'^(?P[^|]*).*','\g', string) # look for arguments (after pipe (|)), remove them - return newstring # if no dot or no pipe then following should be true: newstring == string + if not options.decimalcomma: + newstring = re.sub(r'^(?P\d*)\.(?P\d*).*$','\g.\g', string) + if newstring==string: #maybe not needed, but just in case something irregular comes in, at least arguments are removed + newstring = re.sub(r'^(?P[^|]*)\|.*','\g', string) # look for pipe (|), remove all from pipe and after + return newstring.strip() # if no dot or no pipe then following should be true: newstring == string def fullyQualifiedTemplateTitle(templateTitle): From fbb1199ad81e5a1e4e4eec3c57fe63dca2505ca1 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Wed, 29 Jan 2020 22:02:08 +0100 Subject: [PATCH 082/110] version update now, as in better late than never --- WikiExtractor.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index a36bf20c..20b94a57 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # ============================================================================= -# Version: 2.75 (March 4, 2017) +# Version: 3.0 (2020-01-29) # Author: Giuseppe Attardi (attardi@di.unipi.it), University of Pisa # # Contributors: @@ -115,7 +115,7 @@ def __eq__ (self, other): # =========================================================================== # Program version -version = '2.75' +version = '3.0' ## PARAMS #################################################################### @@ -362,7 +362,7 @@ def get_url(uid): # ------------------------------------------------------------------------------ -selfClosingTags = ('BR', 'br', 'hr', 'nobr', 'ref', 'references', 'nowiki') # BR may not be needed +selfClosingTags = ('BR', 'br', 'hr', 'nobr', 'ref', 'references', 'nowiki') # BR is kept, but may not be needed placeholder_tags = {'math': 'formula', 'code': 'codice'} From effcc94e831ff4f6dfa59c647b479b038bb4b56c Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Thu, 30 Jan 2020 00:45:17 +0100 Subject: [PATCH 083/110] clean up templatestyles with content within tags --- WikiExtractor.py | 1 + 1 file changed, 1 insertion(+) diff --git a/WikiExtractor.py b/WikiExtractor.py index 20b94a57..d322daba 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -934,6 +934,7 @@ def clean(self, text): text = re.sub('(\[\(«) ', r'\1', text) text = re.sub(r'\n\W+?\n', '\n', text, flags=re.U) # lines with only punctuations text = text.replace(',,', ',').replace(',.', '.') + text = re.sub(r']*>', '', text) if options.keep_tables: # the following regular expressions are used to remove the wikiml chartacters around table strucutures # yet keep the content. The order here is imporant so we remove certain markup like {| and then From 063d3d12f162c872b1682e92e408d9e6f19d28ea Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Sun, 2 Feb 2020 14:32:40 +0100 Subject: [PATCH 084/110] preparing for #dateformat tags --- WikiExtractor.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index d322daba..e472990a 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -2053,7 +2053,8 @@ def sharp_invoke(module, function, args): '#titleparts': lambda *args: '', # not supported - '#dateformat': lambda *args: '', # not supported #HjalmarrSv + '#dateformat': lambda *args: '', # not supported #HjalmarrSv {{#dateformat:25 dec 2009|ymd}} {{#formatdate:dec 25,2009|dmy}} + #{{#dateformat:2009-12-25|mdy}} {{#formatdate:2009 dec 25|ISO 8601}} {{#dateformat:25 decEmber|mdy}} '#formatdate': lambda *args: '', # not supported #HjalmarrSv @@ -2083,8 +2084,8 @@ def sharp_invoke(module, function, args): 'padleft': lambda *args: '', # not supported #HjalmarrSv 'padright': lambda *args: '', # not supported #HjalmarrSv - - 'as of': lambda *args: '', # not supported #HjalmarrSv #this may be the wrong place in code for this function + # {{As of|2020|post=,}} {{update after|2030}} {{As of|since=y|2020|02|post=,}} {{As of|alt=Beginning in early 2020|2020|01|post=,}} + 'as of': lambda *args: '', # not supported #HjalmarrSv #this may be the wrong place in code for this https://en.wikipedia.org/wiki/Wikipedia:As_of 'plural': lambda *args: '', # not supported #HjalmarrSv From 40bd592e506f479d6114dffd02103dcbddcc079d Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Sun, 2 Feb 2020 22:46:28 +0100 Subject: [PATCH 085/110] clean '! ...' - lines Leftovers from templates, when guessing. --- WikiExtractor.py | 1 + 1 file changed, 1 insertion(+) diff --git a/WikiExtractor.py b/WikiExtractor.py index e472990a..839f32cc 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -945,6 +945,7 @@ def clean(self, text): text = text.replace('|', '') if options.toHTML: text = html_escape(text, quote=False) + text = re.sub(r'^!\s.*$', '', text) #remove all '! data-sort-type=number | Area(km²)' return text From c2353574a22e51064348c8c3f683e55e4a0a3cbc Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Sun, 2 Feb 2020 23:07:14 +0100 Subject: [PATCH 086/110] this way it actually works as supposed to. --- WikiExtractor.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index 839f32cc..557306f0 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -943,9 +943,10 @@ def clean(self, text): text = re.sub(r'!(?:\s)?style="[a-z]+:(?:\d+)%;[a-z]+:(?:#)?(?:[0-9a-z]+)?"', r'', text) text = text.replace('|-', '') text = text.replace('|', '') + text = re.sub(r'\n!\s.*\n', '', text, re.DOTALL) #remove all '! data-sort-type=number | Area(km²)' if options.toHTML: text = html_escape(text, quote=False) - text = re.sub(r'^!\s.*$', '', text) #remove all '! data-sort-type=number | Area(km²)' + return text From 970f94be2dfd1d99366a28961071454bb25ffa63 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Mon, 3 Feb 2020 00:22:59 +0100 Subject: [PATCH 087/110] set 'options.cleaned = True' at row 3528, if you want to rid output of '( )', whether empty or full of text. --- WikiExtractor.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/WikiExtractor.py b/WikiExtractor.py index 557306f0..2b69a005 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -935,6 +935,12 @@ def clean(self, text): text = re.sub(r'\n\W+?\n', '\n', text, flags=re.U) # lines with only punctuations text = text.replace(',,', ',').replace(',.', '.') text = re.sub(r']*>', '', text) + if options.cleaned: + text = re.sub(r'[(][^)(]*[)]', '', text) # removes all '( ... )', whether empty or full of text; first level from within + text = re.sub(r'[(][^)(]*[)]', '', text) # second level. If nested three levels, the third level, outmost, will appear in text, unless cleaned, as below. This would better be a loop. + text = re.sub(r'[(]', '', text) # removes unbalanced '(' or if deeply nested + text = re.sub(r'[)]', '', text) # removes unbalanced ')' or if deeply nested + text = spaces.sub(' ', text) if options.keep_tables: # the following regular expressions are used to remove the wikiml chartacters around table strucutures # yet keep the content. The order here is imporant so we remove certain markup like {| and then @@ -3519,6 +3525,7 @@ def main(): options.abstract_only = args.abstract_only options.raw = args.raw options.decimalcomma = args.decimalcomma + options.cleaned = False options.expand_templates = args.no_templates options.filter_disambig_pages = args.filter_disambig_pages From 7a824b8c657cc2b7f73a45d4542c3fcc57750ea1 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Mon, 3 Feb 2020 00:34:53 +0100 Subject: [PATCH 088/110] Additional cleaning needed --- WikiExtractor.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index 2b69a005..b130191b 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -940,7 +940,8 @@ def clean(self, text): text = re.sub(r'[(][^)(]*[)]', '', text) # second level. If nested three levels, the third level, outmost, will appear in text, unless cleaned, as below. This would better be a loop. text = re.sub(r'[(]', '', text) # removes unbalanced '(' or if deeply nested text = re.sub(r'[)]', '', text) # removes unbalanced ')' or if deeply nested - text = spaces.sub(' ', text) + text = spaces.sub(' ', text) # when removing in sentence ' ' is created, remove. + text = re.sub(r'\s\.', '.', text) # when removing at end of sentence ' .' is created, remove. if options.keep_tables: # the following regular expressions are used to remove the wikiml chartacters around table strucutures # yet keep the content. The order here is imporant so we remove certain markup like {| and then From 56d98529502503840fad77dd05622c8b1d01b1a4 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Mon, 3 Feb 2020 21:13:00 +0100 Subject: [PATCH 089/110] improve cleaning --- WikiExtractor.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index b130191b..cf14f498 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -930,8 +930,8 @@ def clean(self, text): text = text.replace('\t', ' ') text = spaces.sub(' ', text) text = dots.sub('...', text) - text = re.sub(' (,:\.\)\]»)', r'\1', text) - text = re.sub('(\[\(«) ', r'\1', text) + text = re.sub(r' ([,:\.\)\]»])', r'\1', text) + text = re.sub(r'([\[\(«]) ', r'\1', text) text = re.sub(r'\n\W+?\n', '\n', text, flags=re.U) # lines with only punctuations text = text.replace(',,', ',').replace(',.', '.') text = re.sub(r']*>', '', text) @@ -940,8 +940,10 @@ def clean(self, text): text = re.sub(r'[(][^)(]*[)]', '', text) # second level. If nested three levels, the third level, outmost, will appear in text, unless cleaned, as below. This would better be a loop. text = re.sub(r'[(]', '', text) # removes unbalanced '(' or if deeply nested text = re.sub(r'[)]', '', text) # removes unbalanced ')' or if deeply nested - text = spaces.sub(' ', text) # when removing in sentence ' ' is created, remove. - text = re.sub(r'\s\.', '.', text) # when removing at end of sentence ' .' is created, remove. + text = re.sub(' ([,:\.\]»])', r'\1', text) + text = re.sub('([\[«]) ', r'\1', text) + # remove this: text = spaces.sub(' ', text) # when removing in sentence ' ' is created, remove. + # remove this: text = re.sub(r'\s\.', '.', text) # when removing at end of sentence ' .' is created, remove. if options.keep_tables: # the following regular expressions are used to remove the wikiml chartacters around table strucutures # yet keep the content. The order here is imporant so we remove certain markup like {| and then From 43a9b9d2a89227dc1fdd0eab473d42703dad2978 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Mon, 3 Feb 2020 22:02:29 +0100 Subject: [PATCH 090/110] basic formatdate and dateformat parsing --- WikiExtractor.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index cf14f498..319adf65 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -940,8 +940,8 @@ def clean(self, text): text = re.sub(r'[(][^)(]*[)]', '', text) # second level. If nested three levels, the third level, outmost, will appear in text, unless cleaned, as below. This would better be a loop. text = re.sub(r'[(]', '', text) # removes unbalanced '(' or if deeply nested text = re.sub(r'[)]', '', text) # removes unbalanced ')' or if deeply nested - text = re.sub(' ([,:\.\]»])', r'\1', text) - text = re.sub('([\[«]) ', r'\1', text) + text = re.sub(r'\s([\s,:\.\]»])', r'\1', text) + text = re.sub(r'([\[«])\s', r'\1', text) # remove this: text = spaces.sub(' ', text) # when removing in sentence ' ' is created, remove. # remove this: text = re.sub(r'\s\.', '.', text) # when removing at end of sentence ' .' is created, remove. if options.keep_tables: @@ -1854,6 +1854,11 @@ def formatnum(string): #function for parsing 'formatnum:xxx.yyy|zzz' where zzz c newstring = re.sub(r'^(?P[^|]*)\|.*','\g', string) # look for pipe (|), remove all from pipe and after return newstring.strip() # if no dot or no pipe then following should be true: newstring == string +# function for parsing '#dateformat', '#formatdate'; #dateformat:2009-12-25; #formatdate:2009 dec 25 +def formatdate(string): # |dmy, |mdy, |ISO 8601 + string = re.sub(r'^(?P[^|])\|(?P.*)','\g', string) # leave date "as is" for now, strip rest + return string + def fullyQualifiedTemplateTitle(templateTitle): """ @@ -2064,12 +2069,11 @@ def sharp_invoke(module, function, args): '#titleparts': lambda *args: '', # not supported - '#dateformat': lambda *args: '', # not supported #HjalmarrSv {{#dateformat:25 dec 2009|ymd}} {{#formatdate:dec 25,2009|dmy}} - #{{#dateformat:2009-12-25|mdy}} {{#formatdate:2009 dec 25|ISO 8601}} {{#dateformat:25 decEmber|mdy}} + '#dateformat': lambda extr, string, *rest: formatdate(string), #HjalmarrSv {{#dateformat:25 dec 2009|ymd}} {{#formatdate:dec 25,2009|dmy}} - '#formatdate': lambda *args: '', # not supported #HjalmarrSv + '#formatdate': lambda extr, string, *rest: formatdate(string), #HjalmarrSv {{#dateformat:2009-12-25|mdy}} {{#formatdate:2009 dec 25|ISO 8601}} {{#dateformat:25 decEmber|mdy}} - '#tag': lambda *args: '', # not supported #HjalmarrSv + '#tag': lambda *args: '', # not supported HjalmarrSv {{#tag:tagname|content|attribute1=value1|attribute2=value2}} # This function is used in some pages to construct links # http://meta.wikimedia.org/wiki/Help:URL From c4136d03d2d8f3684d2d4dae0bb55a5aa4a149e9 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Mon, 3 Feb 2020 23:52:23 +0100 Subject: [PATCH 091/110] remove multiples of '/n' as seen in json-output --- WikiExtractor.py | 1 + 1 file changed, 1 insertion(+) diff --git a/WikiExtractor.py b/WikiExtractor.py index 319adf65..27c43230 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -953,6 +953,7 @@ def clean(self, text): text = text.replace('|-', '') text = text.replace('|', '') text = re.sub(r'\n!\s.*\n', '', text, re.DOTALL) #remove all '! data-sort-type=number | Area(km²)' + text = re.sub(r'\n\s*\n', '\n', text) #remove empty lines, mostly affects .json, because .txt cleaned after this function if options.toHTML: text = html_escape(text, quote=False) From d90b9e75dbe21afdd052247fbde4706cfac824a8 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Tue, 4 Feb 2020 00:18:41 +0100 Subject: [PATCH 092/110] remove empty ( ) if they are not already cleaned --- WikiExtractor.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/WikiExtractor.py b/WikiExtractor.py index 27c43230..2111c09c 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -953,6 +953,9 @@ def clean(self, text): text = text.replace('|-', '') text = text.replace('|', '') text = re.sub(r'\n!\s.*\n', '', text, re.DOTALL) #remove all '! data-sort-type=number | Area(km²)' + if not options.cleaned: + text = re.sub(r'[(]\s*[)]', '', text) # removes all '()' and '( )' + text = spaces.sub(' ', text) text = re.sub(r'\n\s*\n', '\n', text) #remove empty lines, mostly affects .json, because .txt cleaned after this function if options.toHTML: text = html_escape(text, quote=False) From 9c7d502f00122488fe7fd60bc471c4c471786003 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Wed, 5 Feb 2020 11:08:56 +0100 Subject: [PATCH 093/110] Update comments in code --- WikiExtractor.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index 2111c09c..d0948bd8 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -927,6 +927,11 @@ def clean(self, text): ############################################# # Cleanup text + # This code exists because of errors in wikipedia, errors in templates, errors in template expansion, + # or because of lazy or difficult transclusion. Some cleaning is needed because of the difference between input format + # and what we want as output. Ideally this part of the code should be as short as possible. Fix as much as possible + # upstream, i.e. before this code. + # text = text.replace('\t', ' ') text = spaces.sub(' ', text) text = dots.sub('...', text) @@ -954,8 +959,8 @@ def clean(self, text): text = text.replace('|', '') text = re.sub(r'\n!\s.*\n', '', text, re.DOTALL) #remove all '! data-sort-type=number | Area(km²)' if not options.cleaned: - text = re.sub(r'[(]\s*[)]', '', text) # removes all '()' and '( )' - text = spaces.sub(' ', text) + text = re.sub(r'[(]\s*[)]', '', text) # removes all '()' and '( )' # this may have a negative effect on math articles + text = spaces.sub(' ', text) # removing text, without removing a space before or after, puts two spaces next to each other. text = re.sub(r'\n\s*\n', '\n', text) #remove empty lines, mostly affects .json, because .txt cleaned after this function if options.toHTML: text = html_escape(text, quote=False) From 01668eb045e0b52a5be1cef8fd8916a4a748028e Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Thu, 6 Feb 2020 11:09:26 +0100 Subject: [PATCH 094/110] use raw, especially where there are \ https://stackoverflow.com/questions/12871066/what-exactly-is-a-raw-string-regex-and-how-can-you-use-it Using raw does nothing for re., but stops python from interpreting anything in the string before passing to re., as stated above. --- WikiExtractor.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index d0948bd8..8895abc4 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -281,7 +281,7 @@ def __eq__ (self, other): ## # Regex for identifying disambig pages -filter_disambig_page_pattern = re.compile("{{disambig(uation)?(\|[^}]*)?}}|__DISAMBIG__") +filter_disambig_page_pattern = re.compile(r"{{[Dd]isambig(uation)?(\|[^}]*)?}}|__DISAMBIG__") ## g_page_total = 0 @@ -1947,10 +1947,10 @@ def sharp_expr(extr, expr): """Tries converting a lua expr into a Python expr.""" try: expr = extr.expand(expr) - expr = re.sub('(?])=', '==', expr) # negative lookbehind - expr = re.sub('mod', '%', expr) # no \b here - expr = re.sub('\bdiv\b', '/', expr) - expr = re.sub('\bround\b', '|ROUND|', expr) + expr = re.sub(r'(?])=', '==', expr) # negative lookbehind + expr = re.sub(r'mod', '%', expr) # no \b here + expr = re.sub(r'\bdiv\b', '/', expr) + expr = re.sub(r'\bround\b', '|ROUND|', expr) return text_type(eval(expr)) except: return '%s' % expr From 3f672b0905e491c92ba7b5bc1e85a0b26bb475c5 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Thu, 6 Feb 2020 16:26:17 +0100 Subject: [PATCH 095/110] expand formatnum functionality may contain errors, testing needed --- WikiExtractor.py | 47 ++++++++++++++++++++++++++++++++++++----------- 1 file changed, 36 insertions(+), 11 deletions(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index 8895abc4..897a9936 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -1849,19 +1849,44 @@ def lcfirst(string): return string.lower() else: return '' - - -def formatnum(string): #function for parsing 'formatnum:xxx.yyy|zzz' where zzz can be '|R', '|NOSEP': https://www.mediawiki.org/wiki/Help:Magic_words - string = string.strip() + +# Function for parsing 'formatnum:xxx.yyy|zzz' where zzz can be '|R', '|NOSEP': https://www.mediawiki.org/wiki/Help:Magic_words +# Takes an unformatted number (Arabic, no group separators and . as decimal separator) and outputs it in the localized digit script and formatted with decimal and decimal group separators, according to the wiki's default locale +# The |R parameter can be used to reverse the behavior, for use in mathematical situations: it's reliable and should be used only to deformat numbers which are known to be formatted exactly as formatnum formats them with the wiki's locale. +def formatnum(string): + formatnum_reverse = False + string = string.strip() #strip whitespace before and after, if any + if re.search(r'\.', string): + has_dot = True + else: + has_dot = False + if re.search(r',', string): + has_comma = True + if has_dot and re.search(r',[^\.]*\.', string): #fix: check for locales with dot first + comma_first = True # true if there is both dot and a comma and comma comes first + else: + comma_first = False + else: + has_comma = False + if re.search(r'|R', string): #check for |R in which case number is already formated and should be unformated + formatnum_reverse = True + if not formatnum_reverse: + string = string.lstrip('0') # strip leading zeroes, if any, but not if |R, because formated number may be 00,001 which is 1 parsed {{formatnum:00001}} newstring = string - if options.decimalcomma: - newstring = re.sub(r'^(?P\d*)\.(?P\d*).*$','\g,\g', string) - # fixme: for the special case that comma is a thousands separator that needs to be removed - if not options.decimalcomma: - newstring = re.sub(r'^(?P\d*)\.(?P\d*).*$','\g.\g', string) + + If formatnum_reverse and comma_first: # a limited |R function; note need for fix above + newstring = re.sub(r'[,\s], '', newstring) # remove commas and space in number like 123,456.789 or 123 456.789, but not in 123,456 which we do not know what it is + + if options.decimalcomma and has_dot: # if there is a dot and we want to change dot to comma; note need for fix above + if comma_first: # check first if there are commas as group separators + newstring = re.sub(r',', '', newstring) # remove group separator comma(s) before changing to decimal separator comma, not caring for |R + newstring = re.sub(r'^(?P\d*)\.(?P\d*).*$','\g,\g', newstring) + + #if not options.decimalcomma: fix: add commas as group separator here, or something locale specific: this may never be implemented + if newstring==string: #maybe not needed, but just in case something irregular comes in, at least arguments are removed - newstring = re.sub(r'^(?P[^|]*)\|.*','\g', string) # look for pipe (|), remove all from pipe and after - return newstring.strip() # if no dot or no pipe then following should be true: newstring == string + newstring = re.sub(r'^(?P[^|]*)\|.*','\g', newstring) # look for pipe (|), remove all from pipe and after + return newstring.strip() # function for parsing '#dateformat', '#formatdate'; #dateformat:2009-12-25; #formatdate:2009 dec 25 def formatdate(string): # |dmy, |mdy, |ISO 8601 From 1b2a6952084c7c889d05f4490ce3efacf3b0b8fe Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Thu, 6 Feb 2020 22:49:58 +0100 Subject: [PATCH 096/110] the obvious bugs looks like there is a nested bug that needs to be hunted down also --- WikiExtractor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index 897a9936..5dc7b0f9 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -1874,8 +1874,8 @@ def formatnum(string): string = string.lstrip('0') # strip leading zeroes, if any, but not if |R, because formated number may be 00,001 which is 1 parsed {{formatnum:00001}} newstring = string - If formatnum_reverse and comma_first: # a limited |R function; note need for fix above - newstring = re.sub(r'[,\s], '', newstring) # remove commas and space in number like 123,456.789 or 123 456.789, but not in 123,456 which we do not know what it is + if formatnum_reverse and comma_first: # a limited |R function; note need for fix above + newstring = re.sub(r'[,\s]', '', newstring) # remove commas and space in number like 123,456.789 or 123 456.789, but not in 123,456 which we do not know what it is if options.decimalcomma and has_dot: # if there is a dot and we want to change dot to comma; note need for fix above if comma_first: # check first if there are commas as group separators From e974947fc45b4e5a74893177960e70ef837657b5 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Fri, 7 Feb 2020 00:32:31 +0100 Subject: [PATCH 097/110] too buggy - works when function shortcut, else not fixing --- WikiExtractor.py | 1 + 1 file changed, 1 insertion(+) diff --git a/WikiExtractor.py b/WikiExtractor.py index 5dc7b0f9..f6d3db6f 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -1854,6 +1854,7 @@ def lcfirst(string): # Takes an unformatted number (Arabic, no group separators and . as decimal separator) and outputs it in the localized digit script and formatted with decimal and decimal group separators, according to the wiki's default locale # The |R parameter can be used to reverse the behavior, for use in mathematical situations: it's reliable and should be used only to deformat numbers which are known to be formatted exactly as formatnum formats them with the wiki's locale. def formatnum(string): + return(string) # disable function until fixed, actually works this way formatnum_reverse = False string = string.strip() #strip whitespace before and after, if any if re.search(r'\.', string): From 3133906f887b2d89609b83a5f54a47f6fa464e42 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Fri, 7 Feb 2020 11:09:31 +0100 Subject: [PATCH 098/110] Update comments: Lua wants! --- WikiExtractor.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index f6d3db6f..ef5da8cd 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -1494,6 +1494,9 @@ def findBalanced(text, openDelim=['[['], closeDelim=[']]']): # Only minimal support # FIXME: import Lua modules. +# https://commons.wikimedia.org/wiki/Commons:Lua/Modules, https://en.wikipedia.org/wiki/Wikipedia:Lua +# https://en.wikipedia.org/wiki/Category:Lua_metamodules +# def if_empty(*rest): """ @@ -1689,7 +1692,7 @@ def toRoman(n, romanNumeralMap): 'Roman': { 'main': roman_main }, - + # localised to Italian 'Numero romano': { 'main': roman_main } @@ -1813,13 +1816,13 @@ def __setitem__(self, name, value): '__NOCC__', '__NOTITLECONVERT__', '__NOTC__', - '__START__', # keep for now, but removed in r1695 and completely removed in r24784 - '__END__', # keep for now, but removed in 19213. + '__START__', # keep, but removed in r1695 and completely removed in r24784 + '__END__', # keep, but removed in 19213. '__INDEX__', '__NOINDEX__', '__STATICREDIRECT__', '__DISAMBIG__', - '__NOGLOBAL__' #added 200126 + '__NOGLOBAL__' ) @@ -1853,6 +1856,10 @@ def lcfirst(string): # Function for parsing 'formatnum:xxx.yyy|zzz' where zzz can be '|R', '|NOSEP': https://www.mediawiki.org/wiki/Help:Magic_words # Takes an unformatted number (Arabic, no group separators and . as decimal separator) and outputs it in the localized digit script and formatted with decimal and decimal group separators, according to the wiki's default locale # The |R parameter can be used to reverse the behavior, for use in mathematical situations: it's reliable and should be used only to deformat numbers which are known to be formatted exactly as formatnum formats them with the wiki's locale. +# Note: Lua function in Lua language: formatted_string = formatnum.formatNum(value, lang, prec, compact): +# Note: "Language:formatNum()" in MediaWiki's core libraries for Lua: +# Note: Templates: {{formatnum|1=value|2=lang|prec=prec|sep=compact}}: https://commons.wikimedia.org/wiki/Template:Formatnum, https://commons.wikimedia.org/wiki/Module:Formatnum +# Note: Template:Formatnum4: https://commons.wikimedia.org/wiki/Template:Formatnum4: example: {{Formatnum4|1|es}} → 1 def formatnum(string): return(string) # disable function until fixed, actually works this way formatnum_reverse = False @@ -2151,6 +2158,7 @@ def callParserFunction(functionName, args, extractor): :param: args not yet expanded (see branching functions). https://www.mediawiki.org/wiki/Help:Extension:ParserFunctions + https://en.wikipedia.org/wiki/Module:Fun """ try: From d0f2529b457bd1012120b5704d649fa48359a0bf Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Fri, 28 Feb 2020 12:14:35 +0100 Subject: [PATCH 099/110] fix for revid Revision ID wrong #203: https://github.com/attardi/wikiextractor/issues/203 --- WikiExtractor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index ef5da8cd..efb370b2 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -3101,7 +3101,7 @@ def pages_from(input): redirect = False elif tag == 'id' and not id: id = m.group(3) - elif tag == 'id' and id: + elif tag == 'id' and not revid: revid = m.group(3) elif tag == 'title': title = m.group(3) From 4a4650d14fce81431444f18c7c98d24ee1a29151 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Sat, 29 Feb 2020 20:47:18 +0100 Subject: [PATCH 100/110] creates article files instead if line 53 is changed to False. Note that if you want something else than ./ab/abcd/abcd as directory structure you need to change in the code. I have commented where (lines 117-119). Please, also look at line 247 for file name variations. --- cirrus-extract.py | 114 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 87 insertions(+), 27 deletions(-) diff --git a/cirrus-extract.py b/cirrus-extract.py index e2c57da4..cffdf795 100755 --- a/cirrus-extract.py +++ b/cirrus-extract.py @@ -2,8 +2,9 @@ # -*- coding: utf-8 -*- # # ============================================================================= -# Version: 1.00 (December 15, 2015) -# Author: Giuseppe Attardi (attardi@di.unipi.it), University of Pisa +# Version: 1.1 (February 29, 2020) +# Author(s): Giuseppe Attardi (attardi@di.unipi.it), University of Pisa +# HjalmarrSv # # ============================================================================= # Copyright (c) 2015. Giuseppe Attardi (attardi@di.unipi.it). @@ -43,10 +44,13 @@ import logging # Program version -version = '1.00' +version = '1.10' -urlbase = 'http://it.wikipedia.org/' +# Urlbase +urlbase = 'http://sv.wikipedia.org/' +# Numbered files is default output. Change to False if you want article files in "articled" directories. +numbered = True # ---------------------------------------------------------------------- @@ -57,27 +61,70 @@ class NextFile(object): filesPerDir = 100 - def __init__(self, path_name): + def __init__(self, path_name, title): self.path_name = path_name - self.dir_index = -1 - self.file_index = -1 + self.dir_index = -1 # for enumerated file names + self.file_index = -1 # + self.title = title # for article file names def next(self): - self.file_index = (self.file_index + 1) % NextFile.filesPerDir - if self.file_index == 0: - self.dir_index += 1 - dirname = self._dirname() - if not os.path.isdir(dirname): - os.makedirs(dirname) - return self._filepath() + if numbered: + self.file_index = (self.file_index + 1) % NextFile.filesPerDir + if self.file_index == 0: + self.dir_index += 1 + dirname = self._dirname() + if not os.path.isdir(dirname): + os.makedirs(dirname) + return self._filepath() + else: + dirname = self._dirname() + if not os.path.isdir(dirname): + os.makedirs(dirname) + return self._filepath() def _dirname(self): - char1 = self.dir_index % 26 - char2 = self.dir_index // 26 % 26 - return os.path.join(self.path_name, '%c%c' % (ord('A') + char2, ord('A') + char1)) + if numbered: + char1 = self.dir_index % 26 + char2 = self.dir_index // 26 % 26 + return os.path.join(self.path_name, '%c%c' % (ord('A') + char2, ord('A') + char1)) + else: + # Remove punctuation from title, for keeping paths from failing (both from path name and filename) + self.title = self.title.replace('.', '') + self.title = self.title.replace('\\', '') + self.title = self.title.replace('/', '') + title = self.title # if you want to limit the amount of symbols, clean title accordingly + if len(title)==0: # same for caps and non caps in directory names (now gw Gw and GW are possible) + return self.path_name # this can happen if "clean" title and title is only "." or other that is removed above + else: + dirname2 = dirname3 = dirname4 = "" # initialize before use (short titles will fail otherwise) + title = re.sub(r'_.*$', "", title) # do not add version nr, etc to directory structure + if title=="": + #title = "____" # replace (./__/____/____[_1234]) with your choice, for the possible few articles consisting + return self.path_name # of only the few forbidden file characters if uncommenting previous line and commenting this one. + dirname1 = title[0] + if len(title)==1: + dirname2 = dirname3 = "" # note "a" not in ./aa/aa but in ./a/a (both ./a/a and ./aa/aa will exist, change + else: # with: dirname2 = dirname3 = dirname1 (also below), then "a" will be in ./aa/aa + dirname2 = title[1] # present choice for machine readability + if len(title)==2: + dirname3 = "" + else: + dirname3 = title[2] + if len(title)==3: + dirname4 = "" + else: + dirname4 = title[3] + d1 = dirname1 + dirname2 # first level directory - first two letter (change to your needs) + d2 = dirname1 + dirname2 + dirname3 + dirname4 # second level directory - all four first letters (could be next two letters) + p = d1 + "/" + d2 # two directories deep, change if needed + return os.path.join(self.path_name, p) + def _filepath(self): - return '%s/wiki_%02d' % (self._dirname(), self.file_index) + if numbered: + return '%s/wiki_%02d' % (self._dirname(), self.file_index) + else: + return '%s/%s' % (self._dirname(), self.title) class OutputSplitter(object): """ @@ -154,8 +201,9 @@ def process_dump(input_file, out_file, file_size, file_compress, text_only, sent output = sys.stdout if file_compress: logging.warn("writing to stdout, so no output compression (use external tool)") - else: - nextFile = NextFile(out_file) + elif numbered: + title = "" + nextFile = NextFile(out_file, title) output = OutputSplitter(nextFile, file_size, file_compress) # process dump @@ -172,6 +220,7 @@ def process_dump(input_file, out_file, file_size, file_compress, text_only, sent id = index['index']['_id'] language = content['language'] revision = content['version'] + # date could be useful, fix date if type == 'page' and content['namespace'] == 0: title = content['title'] text = content['text'] @@ -179,12 +228,12 @@ def process_dump(input_file, out_file, file_size, file_compress, text_only, sent # drop references: # ^ The Penguin Dictionary text = re.sub(r' \^ .*', '', text) # only one space before caret to catch malformed tags - text = re.sub(r'\s*$', '', text) # remove all trailing white space if sentences_only: - text = re.sub(r'\. [^.]*$', '.', text) # remove incomplete last sentence (replaced by below?) - text = re.sub(r'\.\s(.(?!\.\s))*[^.]$', '.', text) # remove incomplete last sentence, no dot and space found as separator and no ending dot - text = re.sub(r'^[^.]*$', '', text) # remove incomplete sentence, even if only sentence in article - text = re.sub(r'^(.(?!\.\s))*$', '', text) # remove if only one sentence in article, no dot and space found as separator + #text = re.sub(r'\. [^.]*$', '.', text) # remove incomplete last sentence, no ending dot + text = re.sub(r'\.\s(.(?!\.\s))*[^.]$', '.', text) # remove incomplete last sentence, no dot and space found as separator and no ending dot + text = re.sub(r'^[^.]*$', '', text) # remove incomplete sentence, even if only sentence in article, no dot at all + + text = re.sub(r'^(.(?!\.\s))*$', '', text) # remove if only one sentence in article, no dot and space found as separator if text != "" and text != " ": # do not create empty articles url = urlbase + 'wiki?curid=' + id header = '\n' % (id, url, title, language, revision) @@ -192,7 +241,13 @@ def process_dump(input_file, out_file, file_size, file_compress, text_only, sent page = header + title + '\n\n' + text + '\n\n' else: page = text + '\n\n' - output.write(page.encode('utf-8')) + if numbered: + output.write(page.encode('utf-8')) + else: + title = title + "_" + id # + "_" + revision # remove this line if clean articles wanted (note the + nextFile = NextFile(out_file, title) # increased risk for overwriting articles). If you want all revisions, + output = OutputSplitter(nextFile, file_size, file_compress) # for all your wiki runs, remove appropriate comment + output.write(page.encode('utf-8')) page = "" @@ -221,6 +276,8 @@ def main(): help="Only at least two complete point separated sentences.") groupO.add_argument("-r", "--raw", action="store_true", help="No filtering.") + # groupO.add_argument("-a", "--articles", action="store_true", + # help="Output as separate articles.") groupP = parser.add_argument_group('Processing') groupP.add_argument("-ns", "--namespaces", default="", metavar="ns1,ns2", @@ -234,7 +291,10 @@ def main(): help="print program version") args = parser.parse_args() - + + # numbered = True + # if args.articles: + # numbered = False try: power = 'kmg'.find(args.bytes[-1].lower()) + 1 From 392fddf103b954f50ff0c77308b0a2f37fd36930 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Sat, 29 Feb 2020 20:52:58 +0100 Subject: [PATCH 101/110] Update README.md --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index a7d197c5..cea01870 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,9 @@ Text output is without titles, etc. It contains only the article texts separated --raw : basically no cleaning.
    --sentences : basic sentence based cleaning, based on dot and space, producing at least two sentences ending with a dot - but can be tricked by dots in names, etc. +If you want every article in a separate file
    +Change line 53 to False. Note that if you want something else than ./ab/abcd/abcd as directory structure you need to change in the code. I have commented where (lines 117-119). Please, also look at line 247 for file name variations. + Cirrus dumps are available at: [cirrussearch](http://dumps.wikimedia.org/other/cirrussearch/). From bc781f25aa1683e31ef74d8de85d7f9e58f31c3a Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Sun, 1 Mar 2020 00:31:05 +0100 Subject: [PATCH 102/110] Adding comments --- WikiExtractor.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index efb370b2..4c916921 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -1860,7 +1860,7 @@ def lcfirst(string): # Note: "Language:formatNum()" in MediaWiki's core libraries for Lua: # Note: Templates: {{formatnum|1=value|2=lang|prec=prec|sep=compact}}: https://commons.wikimedia.org/wiki/Template:Formatnum, https://commons.wikimedia.org/wiki/Module:Formatnum # Note: Template:Formatnum4: https://commons.wikimedia.org/wiki/Template:Formatnum4: example: {{Formatnum4|1|es}} → 1 -def formatnum(string): +# Use: >>> '{:,}'.format(1234567890) -> '1,234,567,890' return(string) # disable function until fixed, actually works this way formatnum_reverse = False string = string.strip() #strip whitespace before and after, if any @@ -1897,6 +1897,8 @@ def formatnum(string): return newstring.strip() # function for parsing '#dateformat', '#formatdate'; #dateformat:2009-12-25; #formatdate:2009 dec 25 +# Use: >>> import datetime >>> d = datetime.datetime(2010, 7, 4, 12, 15, 58) +# >>> '{:%Y-%m-%d %H:%M:%S}'.format(d)'2010-07-04 12:15:58'def formatnum(string): def formatdate(string): # |dmy, |mdy, |ISO 8601 string = re.sub(r'^(?P[^|])\|(?P.*)','\g', string) # leave date "as is" for now, strip rest return string From 86adb42a7291d88d2e1f10de1cd8367e5ea7a0e3 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Mon, 2 Mar 2020 10:41:13 +0100 Subject: [PATCH 103/110] If / gives problems in an os that uses \ in dir path Either change from "/" to e.g. "\", or call the python os.path.join function once per directory level created. I have not tested either. --- cirrus-extract.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cirrus-extract.py b/cirrus-extract.py index cffdf795..d18ea46e 100755 --- a/cirrus-extract.py +++ b/cirrus-extract.py @@ -116,9 +116,9 @@ def _dirname(self): dirname4 = title[3] d1 = dirname1 + dirname2 # first level directory - first two letter (change to your needs) d2 = dirname1 + dirname2 + dirname3 + dirname4 # second level directory - all four first letters (could be next two letters) - p = d1 + "/" + d2 # two directories deep, change if needed - return os.path.join(self.path_name, p) - + p = d1 + "/" + d2 # two directories deep, change if needed. Note: Not tested on Win (where \). + return os.path.join(self.path_name, p) # If problems with this code, consider using two consecutive os.path.join. In + # theory Python should take care of os variations. def _filepath(self): if numbered: From 1f89728ca8de16d82d386cf5748eee78ac451563 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Sat, 7 Mar 2020 15:31:48 +0100 Subject: [PATCH 104/110] fix for en, new folder structure adapt to your needs --- cirrus-extract.py | 76 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 53 insertions(+), 23 deletions(-) diff --git a/cirrus-extract.py b/cirrus-extract.py index d18ea46e..d21a1cc2 100755 --- a/cirrus-extract.py +++ b/cirrus-extract.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # # ============================================================================= -# Version: 1.1 (February 29, 2020) +# Version: 1.11 (Mars 7, 2020) # Author(s): Giuseppe Attardi (attardi@di.unipi.it), University of Pisa # HjalmarrSv # @@ -44,13 +44,13 @@ import logging # Program version -version = '1.10' +version = '1.11' # Urlbase -urlbase = 'http://sv.wikipedia.org/' +urlbase = 'http://de.wikipedia.org/' # Numbered files is default output. Change to False if you want article files in "articled" directories. -numbered = True +numbered = False # ---------------------------------------------------------------------- @@ -89,12 +89,16 @@ def _dirname(self): return os.path.join(self.path_name, '%c%c' % (ord('A') + char2, ord('A') + char1)) else: # Remove punctuation from title, for keeping paths from failing (both from path name and filename) - self.title = self.title.replace('.', '') - self.title = self.title.replace('\\', '') - self.title = self.title.replace('/', '') - title = self.title # if you want to limit the amount of symbols, clean title accordingly - if len(title)==0: # same for caps and non caps in directory names (now gw Gw and GW are possible) - return self.path_name # this can happen if "clean" title and title is only "." or other that is removed above + # self.title = self.title.replace('.', '') # should not be a problem in title/filename + # self.title = self.title.replace('\\', '') # not a problem in linux: + self.title = self.title.replace('/', '÷') # replace '' or '÷' https://stackoverflow.com/questions/1976007/what-characters-are-forbidden-in-windows-and-linux-directory-names + title = self.title # if you want to limit the amount of symbols, clean title accordingly + title = title.replace('.', '') # not dots in directory names + title = title.upper() # same for caps and non caps in directory names (else gw Gw and GW are possible) + title = re.sub(r'["”]', "", title) # no citation marks in directory names + title = title.lstrip() # remove space at beginning, since it may occur + if len(title)==0: # this can happen if "clean" title and title is only "." or other that is removed above + return self.path_name else: dirname2 = dirname3 = dirname4 = "" # initialize before use (short titles will fail otherwise) title = re.sub(r'_.*$', "", title) # do not add version nr, etc to directory structure @@ -114,11 +118,12 @@ def _dirname(self): dirname4 = "" else: dirname4 = title[3] - d1 = dirname1 + dirname2 # first level directory - first two letter (change to your needs) - d2 = dirname1 + dirname2 + dirname3 + dirname4 # second level directory - all four first letters (could be next two letters) - p = d1 + "/" + d2 # two directories deep, change if needed. Note: Not tested on Win (where \). - return os.path.join(self.path_name, p) # If problems with this code, consider using two consecutive os.path.join. In - # theory Python should take care of os variations. + d1 = dirname1 # first level directory - first letter (change to your needs) + d2 = dirname1 + dirname2 + dirname3 # + dirname4 # second level directory - all three first letters (could be next two letters) + d2 = re.sub(r'[ ,]$', "", d2) # remove ending space and comma + p = d1 + "/" + d2 # two directories deep, change if needed + return os.path.join(self.path_name, p) # replace line above with two joins? + def _filepath(self): if numbered: @@ -207,9 +212,18 @@ def process_dump(input_file, out_file, file_size, file_compress, text_only, sent output = OutputSplitter(nextFile, file_size, file_compress) # process dump - # format + # format (two lines: index and content) # {"index":{"_type":"page","_id":"3825914"}} # {"namespace":0,"title":TITLE,"timestamp":"2014-06-29T15:51:09Z","text":TEXT,...} + # or + # {"index":{"_type":"page","_id":"4274592"}} + # {"template":["Template:See also","Template:Studebaker","Template:Navbox","Template:Truck-stub","Template:Asbox","Module:Labelled list + # hatnote","Module:Hatnote","Module:Hatnote + # list","Module:Arguments","Module:Navbox","Module:Navbar","Module:Asbox","Module:Buffer"], + # "content_model":"wikitext","opening_text":"Transtar was ... 1963.","wiki":"enwiki","auxiliary_text":[ + # "This truck-related article is a stub. You can help Wikipedia by expanding it. v t e","Transtar Truck Model + # 3-E"],"language":"en","title":"Studebaker Transtar","text":"Transtar + while True: line = input.readline() if not line: @@ -218,10 +232,26 @@ def process_dump(input_file, out_file, file_size, file_compress, text_only, sent content = json.loads(input.readline()) type = index['index']['_type'] id = index['index']['_id'] - language = content['language'] - revision = content['version'] - # date could be useful, fix date - if type == 'page' and content['namespace'] == 0: + try: + language = content['language'] # use try: because en errors out on some lines + except: + language = "" + try: + revision = content['version'] + except: + revision = "0" + # date from timestamp + try: + time = content['timestamp'] + date = re.sub(r'[^d]*(?P\d{4})-(?P\d{2})-(?P\d{2}).*', '\g\g\g', time) + except: + time = "0" + try: + ns = content['namespace'] + except: + ns = -1 + + if type == 'page' and ns == 0: title = content['title'] text = content['text'] if not raw_only: @@ -244,9 +274,9 @@ def process_dump(input_file, out_file, file_size, file_compress, text_only, sent if numbered: output.write(page.encode('utf-8')) else: - title = title + "_" + id # + "_" + revision # remove this line if clean articles wanted (note the - nextFile = NextFile(out_file, title) # increased risk for overwriting articles). If you want all revisions, - output = OutputSplitter(nextFile, file_size, file_compress) # for all your wiki runs, remove appropriate comment + title = title + "_" + language + "_"+ id + "_" + str(revision) + "_" + date # remove this line if clean articles wanted (note the + nextFile = NextFile(out_file, title) # increased risk for overwriting articles). Chose if you want all revisions, + output = OutputSplitter(nextFile, file_size, file_compress) # for all your wiki runs, by adding/removing # output.write(page.encode('utf-8')) page = "" From 69d0e766c38c038ca5ace811f9abd9e1bae89d78 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Sat, 7 Mar 2020 15:41:34 +0100 Subject: [PATCH 105/110] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cea01870..510c9025 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ Text output is without titles, etc. It contains only the article texts separated --sentences : basic sentence based cleaning, based on dot and space, producing at least two sentences ending with a dot - but can be tricked by dots in names, etc. If you want every article in a separate file
    -Change line 53 to False. Note that if you want something else than ./ab/abcd/abcd as directory structure you need to change in the code. I have commented where (lines 117-119). Please, also look at line 247 for file name variations. +Change line 53 to False. Note that if you want something else than ./ab/abcd/abcd as directory structure you need to change in the code. I have commented where (lines 121-125). Please, also look at line 277 for file name variations. Cirrus dumps are available at: [cirrussearch](http://dumps.wikimedia.org/other/cirrussearch/). From 1f708177a2b37de64a4dd3057f0f229d1b6ab099 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Sun, 8 Mar 2020 18:41:43 +0100 Subject: [PATCH 106/110] Updated Nicer library structure. --- cirrus-extract.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/cirrus-extract.py b/cirrus-extract.py index d21a1cc2..cf23302b 100755 --- a/cirrus-extract.py +++ b/cirrus-extract.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # # ============================================================================= -# Version: 1.11 (Mars 7, 2020) +# Version: 1.12 (Mars 8, 2020) # Author(s): Giuseppe Attardi (attardi@di.unipi.it), University of Pisa # HjalmarrSv # @@ -92,16 +92,18 @@ def _dirname(self): # self.title = self.title.replace('.', '') # should not be a problem in title/filename # self.title = self.title.replace('\\', '') # not a problem in linux: self.title = self.title.replace('/', '÷') # replace '' or '÷' https://stackoverflow.com/questions/1976007/what-characters-are-forbidden-in-windows-and-linux-directory-names - title = self.title # if you want to limit the amount of symbols, clean title accordingly - title = title.replace('.', '') # not dots in directory names - title = title.upper() # same for caps and non caps in directory names (else gw Gw and GW are possible) - title = re.sub(r'["”]', "", title) # no citation marks in directory names - title = title.lstrip() # remove space at beginning, since it may occur - if len(title)==0: # this can happen if "clean" title and title is only "." or other that is removed above + title = self.title # If you want to limit the amount of symbols for directory names, clean title accordingly. + title = title.replace('.', '') # No dots in directory names. + title = title.upper() # Use caps for directory names (G/GW, or else g/gw G/Gw and G/GW are possible, and likely). + #if len(title) > 0: # Do not create directories for articles on (, [ (if any), by cleaning below. + if len(title) > 1: # Create directories for articles on (, [ (if any) by not cleaning one character articles below. + title = re.sub(r'[’"”\(\[\)\]]', "", title) # No citation marks, or (, [ in directory names (no reason, but does not look nice). + title = title.lstrip() # Remove space at beginning, since it may occur. + if len(title)==0: # This can happen if "clean" title and title is only "." or other that is removed above. return self.path_name else: dirname2 = dirname3 = dirname4 = "" # initialize before use (short titles will fail otherwise) - title = re.sub(r'_.*$', "", title) # do not add version nr, etc to directory structure + title = re.sub(r'_.*$', "", title) # Do not add version nr, etc to directory structure (if added before). if title=="": #title = "____" # replace (./__/____/____[_1234]) with your choice, for the possible few articles consisting return self.path_name # of only the few forbidden file characters if uncommenting previous line and commenting this one. @@ -119,10 +121,11 @@ def _dirname(self): else: dirname4 = title[3] d1 = dirname1 # first level directory - first letter (change to your needs) + dir_path_name = os.path.join(self.path_name, d1) d2 = dirname1 + dirname2 + dirname3 # + dirname4 # second level directory - all three first letters (could be next two letters) - d2 = re.sub(r'[ ,]$', "", d2) # remove ending space and comma - p = d1 + "/" + d2 # two directories deep, change if needed - return os.path.join(self.path_name, p) # replace line above with two joins? + d2 = re.sub(r'[ ,]$', "", d2) # remove ending space and comma, if any. + dir_path_name = os.path.join(dir_path_name, d2) # two directories deep, change if needed + return dir_path_name def _filepath(self): @@ -245,7 +248,7 @@ def process_dump(input_file, out_file, file_size, file_compress, text_only, sent time = content['timestamp'] date = re.sub(r'[^d]*(?P\d{4})-(?P\d{2})-(?P\d{2}).*', '\g\g\g', time) except: - time = "0" + time = date = "0" try: ns = content['namespace'] except: @@ -274,6 +277,7 @@ def process_dump(input_file, out_file, file_size, file_compress, text_only, sent if numbered: output.write(page.encode('utf-8')) else: + title = re.sub(r'_', ' ', title) # Replace all _ since they are reserved for use below. title = title + "_" + language + "_"+ id + "_" + str(revision) + "_" + date # remove this line if clean articles wanted (note the nextFile = NextFile(out_file, title) # increased risk for overwriting articles). Chose if you want all revisions, output = OutputSplitter(nextFile, file_size, file_compress) # for all your wiki runs, by adding/removing # From f3a18923bc0ea7a75904b56df1b8eb012aeadd8d Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Sun, 8 Mar 2020 18:52:52 +0100 Subject: [PATCH 107/110] Update README.md --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 510c9025..62b63a41 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ For further information, see the [project Home Page](http://medialab.di.unipi.it # Wikipedia Cirrus Extractor `cirrus-extractor.py` is a version of the script that performs extraction from a Wikipedia Cirrus dump. -Cirrus dumps contain text with already expanded templates. The Cirrus extractor does not suffer fron somewhat inadequate template expansion. Until WikiExtractors template expansion has been fixed this may be used instead. +Cirrus dumps contain text with already expanded templates. The Cirrus extractor does not suffer fron somewhat inadequate template expansion. Until WikiExtractors template expansion has been fixed this may be used instead. Although some templates, such as the one for stub articles, is not useful expanded. Examples:
    json output: python3 cirrus-extract.py -o wiki/test wiki/wiki-20191104-cirrussearch-content.json.gz
    @@ -20,8 +20,9 @@ Text output is without titles, etc. It contains only the article texts separated --raw : basically no cleaning.
    --sentences : basic sentence based cleaning, based on dot and space, producing at least two sentences ending with a dot - but can be tricked by dots in names, etc. -If you want every article in a separate file
    -Change line 53 to False. Note that if you want something else than ./ab/abcd/abcd as directory structure you need to change in the code. I have commented where (lines 121-125). Please, also look at line 277 for file name variations. +If you want, or do not want, every article in a separate file
    +Change line 53 accordingly. Note that if you want something else than ./A/ABC/abc... as directory structure you need to change in the code. I have commented where (lines 123-127). Please, also look at line 281 for file name variations.
    +Example: python3 cirrus-extract.py -o wiki/test -t --sentences wiki/wiki-20191216-cirrussearch-content.json.gz Cirrus dumps are available at: [cirrussearch](http://dumps.wikimedia.org/other/cirrussearch/). From 0654babb42e9f96365b6898535b43afec9ce66b9 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Fri, 13 Mar 2020 16:04:37 +0100 Subject: [PATCH 108/110] Update cirrus-extract.py --- cirrus-extract.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/cirrus-extract.py b/cirrus-extract.py index cf23302b..7a411cc5 100755 --- a/cirrus-extract.py +++ b/cirrus-extract.py @@ -50,7 +50,9 @@ urlbase = 'http://de.wikipedia.org/' # Numbered files is default output. Change to False if you want article files in "articled" directories. -numbered = False +numbered = True +# When article output is chosen above, there will be global differences between text_only and jason_like output. +textonly = False # ---------------------------------------------------------------------- @@ -91,6 +93,8 @@ def _dirname(self): # Remove punctuation from title, for keeping paths from failing (both from path name and filename) # self.title = self.title.replace('.', '') # should not be a problem in title/filename # self.title = self.title.replace('\\', '') # not a problem in linux: + if textonly: # less replacements needed for jason-like article titles + # self.title = self.title.replace('/', '÷') # replace '' or '÷' https://stackoverflow.com/questions/1976007/what-characters-are-forbidden-in-windows-and-linux-directory-names title = self.title # If you want to limit the amount of symbols for directory names, clean title accordingly. title = title.replace('.', '') # No dots in directory names. @@ -120,6 +124,10 @@ def _dirname(self): dirname4 = "" else: dirname4 = title[3] + #if not textonly: # A/AB/abc for jason-like articles. Less articles expected in this format, maybe about a million, + # because basically one directory level is inside the article that contains many articles. + # + #else: # A/ABC/abc for text only output, because of the large number of articles, tens of millions. d1 = dirname1 # first level directory - first letter (change to your needs) dir_path_name = os.path.join(self.path_name, d1) d2 = dirname1 + dirname2 + dirname3 # + dirname4 # second level directory - all three first letters (could be next two letters) @@ -276,6 +284,8 @@ def process_dump(input_file, out_file, file_size, file_compress, text_only, sent page = text + '\n\n' if numbered: output.write(page.encode('utf-8')) + #elif not text_only # json-like output, where every word only has one article, and this article has all variations in it + # #check if article exists, and if article exists check for id and version, if not exist then add or create else: title = re.sub(r'_', ' ', title) # Replace all _ since they are reserved for use below. title = title + "_" + language + "_"+ id + "_" + str(revision) + "_" + date # remove this line if clean articles wanted (note the From fbeef46befa0f460357307c266aaa902f22d538a Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Tue, 19 May 2020 22:06:41 +0200 Subject: [PATCH 109/110] Update cirrus-extract.py Slight improvement. Note sentences ends with . ! ?. Other punctuation marks have to be added, if they exist. --- cirrus-extract.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/cirrus-extract.py b/cirrus-extract.py index 7a411cc5..63ca0478 100755 --- a/cirrus-extract.py +++ b/cirrus-extract.py @@ -265,16 +265,20 @@ def process_dump(input_file, out_file, file_size, file_compress, text_only, sent if type == 'page' and ns == 0: title = content['title'] text = content['text'] - if not raw_only: + text = re.sub(r'\s', ' ', text) # change different whitespace characters to conventional space, e.g. wikipedia contains tabs + text = re.sub(r' ( )+', ' ', text) # replace multiple consecutive spaces, if any (e.g.tab+space), with a single space + text = text.strip() # remove space in beginning and end + if not raw_only: # drop references: # ^ The Penguin Dictionary text = re.sub(r' \^ .*', '', text) # only one space before caret to catch malformed tags if sentences_only: + text = re.sub(r'[.!?] [^.!?]*[^.!?]$', '.', text) # remove incomplete last sentence of at least two, no ending dot + text = re.sub(r'^.*[^.!?]$', '', text) # now remove incomplete only sentence, no ending dot. this article will not be. #text = re.sub(r'\. [^.]*$', '.', text) # remove incomplete last sentence, no ending dot - text = re.sub(r'\.\s(.(?!\.\s))*[^.]$', '.', text) # remove incomplete last sentence, no dot and space found as separator and no ending dot - text = re.sub(r'^[^.]*$', '', text) # remove incomplete sentence, even if only sentence in article, no dot at all - - text = re.sub(r'^(.(?!\.\s))*$', '', text) # remove if only one sentence in article, no dot and space found as separator + #text = re.sub(r'\.\s(.(?!\.\s))*[^.]$', '.', text) # remove incomplete last sentence, no dot and space found as separator and no ending dot + #text = re.sub(r'^[^.]*$', '', text) # remove incomplete sentence, even if only sentence in article, no dot at all + #text = re.sub(r'^(.(?!\.\s))*$', '', text) # remove if only one sentence in article, no dot and space found as separator if text != "" and text != " ": # do not create empty articles url = urlbase + 'wiki?curid=' + id header = '\n' % (id, url, title, language, revision) From c423d2d0a694cd058e7f76028dddd0d68b9e23a8 Mon Sep 17 00:00:00 2001 From: HjalmarrSv <58831450+HjalmarrSv@users.noreply.github.com> Date: Thu, 14 Oct 2021 00:31:29 +0200 Subject: [PATCH 110/110] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 62b63a41..c349e365 100644 --- a/README.md +++ b/README.md @@ -113,7 +113,7 @@ For those use cases where only on large file is needed, in linux use: cat --sque {"id": "", "revid": "", "url":"", "title": "", "text": "..."} - Template expansion requires preprocesssng first the whole dump and + Template expansion requires preprocessing first the whole dump and collecting template definitions. positional arguments: