Skip to content

divided up the text into summary, and contnt for NLP processing #249

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ertosns wants to merge 2 commits into
base: master
Choose a base branch
from
Open

divided up the text into summary, and contnt for NLP processing #249

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
40d0a37
divided up the text into summary, and contnt for NLP processing
ertosns Mar 16, 2021
09d3cb5
extractor summary list definition is simplified
ertosns Mar 17, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 16 additions & 7 deletions wikiextractor/extract.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,14 +192,21 @@ def compact(text, mark_headers=False):
"""

page = [] # list of paragraph
summary = [] # introduction/summary
headers = {} # Headers for unfilled sections
emptySection = False # empty sections are discarded
listLevel = '' # nesting of lists

summary_opened=True
summary_finished=False
for line in text.split('\n'):

if not line:
continue
if line.startswith('=='):
summary_finished=True
if summary_finished and summary_opened:
summary = page
page = []
# Handle section titles
m = section.match(line)
if m:
Expand Down Expand Up @@ -282,7 +289,7 @@ def compact(text, mark_headers=False):
# elif line[0] == ' ':
# continue

return page
return summary, page


# ----------------------------------------------------------------------
Expand Down Expand Up @@ -844,8 +851,8 @@ def clean_text(self, text, mark_headers=False, expand_templates=False,
text = clean(self, text, expand_templates=expand_templates,
html_safe=html_safe)

text = compact(text, mark_headers=mark_headers)
return text
summary, content = compact(text, mark_headers=mark_headers)
return summary, content

def extract(self, out, html_safe=True):
"""
Expand All @@ -854,15 +861,16 @@ def extract(self, out, html_safe=True):
"""
logging.debug("%s\t%s", self.id, self.title)
text = ''.join(self.page)
text = self.clean_text(text, html_safe=html_safe)
summary, content = self.clean_text(text, html_safe=html_safe)

if self.to_json:
json_data = {
'id': self.id,
'revid': self.revid,
'url': self.url,
'title': self.title,
'text': "\n".join(text)
'text': "\n".join(content),
'summary': '\n'.join(summary)
}
out_str = json.dumps(json_data)
out.write(out_str)
Expand All @@ -873,7 +881,8 @@ def extract(self, out, html_safe=True):
header += self.title + '\n\n'
footer = "\n</doc>\n"
out.write(header)
out.write('\n'.join(text))
out.write('\n'.join(summary))
out.write('\n'.join(content))
out.write('\n')
out.write(footer)

Expand Down