From 40d0a3786512e0e982a8a9b3dadb02eb1a5fb6ec Mon Sep 17 00:00:00 2001
From: mohab <mohab-metwally@riseup.net>
Date: Tue, 16 Mar 2021 18:40:01 +0200
Subject: [PATCH 1/2] divided up the text into summary, and contnt for NLP
 processing

---
 wikiextractor/extract.py | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)

diff --git a/wikiextractor/extract.py b/wikiextractor/extract.py
index 2180dc4e..0af5b1fd 100644
--- a/wikiextractor/extract.py
+++ b/wikiextractor/extract.py
@@ -192,14 +192,21 @@ def compact(text, mark_headers=False):
     """
 
     page = []  # list of paragraph
+    summary = [] # introduction/summary
     headers = {}  # Headers for unfilled sections
     emptySection = False  # empty sections are discarded
     listLevel = ''  # nesting of lists
-
+    summary_opened=True
+    summary_finished=False
     for line in text.split('\n'):
 
         if not line:
             continue
+        if line.startswith('=='):
+            summary_finished=True
+        if summary_finished and summary_opened:
+            summary = [i for i in page]
+            page = []
         # Handle section titles
         m = section.match(line)
         if m:
@@ -282,7 +289,7 @@ def compact(text, mark_headers=False):
             # elif line[0] == ' ':
             #     continue
 
-    return page
+    return summary, page
 
 
 # ----------------------------------------------------------------------
@@ -844,8 +851,8 @@ def clean_text(self, text, mark_headers=False, expand_templates=False,
         text = clean(self, text, expand_templates=expand_templates,
                      html_safe=html_safe)
 
-        text = compact(text, mark_headers=mark_headers)
-        return text
+        summary, content = compact(text, mark_headers=mark_headers)
+        return summary, content
 
     def extract(self, out, html_safe=True):
         """
@@ -854,7 +861,7 @@ def extract(self, out, html_safe=True):
         """
         logging.debug("%s\t%s", self.id, self.title)
         text = ''.join(self.page)
-        text = self.clean_text(text, html_safe=html_safe)
+        summary, content = self.clean_text(text, html_safe=html_safe)
 
         if self.to_json:
             json_data = {
@@ -862,7 +869,8 @@ def extract(self, out, html_safe=True):
                 'revid': self.revid,
                 'url': self.url,
                 'title': self.title,
-                'text': "\n".join(text)
+                'text': "\n".join(content),
+                'summary': '\n'.join(summary)
             }
             out_str = json.dumps(json_data)
             out.write(out_str)
@@ -873,7 +881,8 @@ def extract(self, out, html_safe=True):
             header += self.title + '\n\n'
             footer = "\n</doc>\n"
             out.write(header)
-            out.write('\n'.join(text))
+            out.write('\n'.join(summary))
+            out.write('\n'.join(content))
             out.write('\n')
             out.write(footer)
 

From 09d3cb58e04603512bf740a8bed8dc81c2805588 Mon Sep 17 00:00:00 2001
From: mohab <mohab-metwally@riseup.net>
Date: Wed, 17 Mar 2021 19:55:06 +0200
Subject: [PATCH 2/2] extractor summary list definition is simplified

---
 wikiextractor/extract.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/wikiextractor/extract.py b/wikiextractor/extract.py
index 0af5b1fd..f90c5929 100644
--- a/wikiextractor/extract.py
+++ b/wikiextractor/extract.py
@@ -205,7 +205,7 @@ def compact(text, mark_headers=False):
         if line.startswith('=='):
             summary_finished=True
         if summary_finished and summary_opened:
-            summary = [i for i in page]
+            summary = page
             page = []
         # Handle section titles
         m = section.match(line)