Skip to content

Fix some errors #349

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
afifafifafifafifali wants to merge 1 commit into
base: master
Choose a base branch
from
+40 −165
Open

Fix some errors #349

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
205 changes: 40 additions & 165 deletions wikiextractor/extract.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -199,12 +199,7 @@ def compact(text, mark_headers=False):
for line in text.split('\n'):

if not line:
if len(listLevel): # implies Extractor.HtmlFormatting
for c in reversed(listLevel):
page.append(listClose[c])
listLevel = ''
continue

# Handle section titles
m = section.match(line)
if m:
Expand Down Expand Up @@ -232,35 +227,36 @@ def compact(text, mark_headers=False):
page.append(title)
# handle indents
elif line[0] == ':':
page.append(line.lstrip(':'))
# page.append(line.lstrip(':*#;'))
continue
# handle lists
# @see https://www.mediawiki.org/wiki/Help:Formatting
elif line[0] in '*#;':
elif line[0] in '*#;:':
if Extractor.HtmlFormatting:
# close extra levels
l = 0
for c in listLevel:
if l < len(line) and c != line[l]:
for extra in reversed(listLevel[l:]):
page.append(listClose[extra])
listLevel = listLevel[:l]
break
l += 1
if l < len(line) and line[l] in '*#;:':
# add new level (only one, no jumps)
# FIXME: handle jumping levels
type = line[l]
page.append(listOpen[type])
listLevel += type
line = line[l+1:].strip()
else:
# continue on same level
type = line[l-1]
line = line[l:].strip()
page.append(listItem[type] % line)
i = 0
for c, n in zip_longest(listLevel, line, fillvalue=''):
if not n or n not in '*#;:':
if c:
page.append(listClose[c])
listLevel = listLevel[:-1]
continue
else:
break
# n != ''
if c != n and (not c or (c not in ';:' and n not in ';:')):
if c:
# close level
page.append(listClose[c])
listLevel = listLevel[:-1]
listLevel += n
page.append(listOpen[n])
i += 1
n = line[i - 1] # last list char
line = line[i:].strip()
if line: # FIXME: n is '"'
page.append(listItem[n] % line)
else:
continue
elif len(listLevel): # implies Extractor.HtmlFormatting
elif len(listLevel):
for c in reversed(listLevel):
page.append(listClose[c])
listLevel = []
Expand Down Expand Up @@ -380,14 +376,15 @@ def dropSpans(spans, text):
# as well as U+3000 is IDEOGRAPHIC SPACE for bug 19052
EXT_LINK_URL_CLASS = r'[^][<>"\x00-\x20\x7F\s]'
ExtLinkBracketedRegex = re.compile(
'\[(((?i)' + '|'.join(wgUrlProtocols) + ')' + EXT_LINK_URL_CLASS + r'+)\s*([^\]\x00-\x08\x0a-\x1F]*?)\]',
'(?i)\[((' + '|'.join(wgUrlProtocols) + ')' + EXT_LINK_URL_CLASS + r'+)\s*([^\]\x00-\x08\x0a-\x1F]*?)\]',
re.S | re.U)
EXT_IMAGE_REGEX = re.compile(
r"""^(http://|https://)([^][<>"\x00-\x20\x7F\s]+)
/([A-Za-z0-9_.,~%\-+&;#*?!=()@\x80-\xFF]+)\.((?i)gif|png|jpg|jpeg)$""",
r"""(?i)^(http://|https://)([^][<>"\x00-\x20\x7F\s]+)
/([A-Za-z0-9_.,~%\-+&;#*?!=()@\x80-\xFF]+)\.(gif|png|jpg|jpeg)$""",
re.X | re.S | re.U)



def replaceExternalLinks(text):
s = ''
cur = 0
Expand Down Expand Up @@ -790,114 +787,6 @@ def resetIgnoredTags():
# Matches dots
dots = re.compile(r'\.{4,}')

# ======================================================================

class Template(list):
"""
A Template is a list of TemplateText or TemplateArgs
"""

@classmethod
def parse(cls, body):
tpl = Template()
# we must handle nesting, s.a.
# {{{1|{{PAGENAME}}}
# {{{italics|{{{italic|}}}
# {{#if:{{{{{#if:{{{nominee|}}}|nominee|candidate}}|}}}|
#
start = 0
for s,e in findMatchingBraces(body, 3):
tpl.append(TemplateText(body[start:s]))
tpl.append(TemplateArg(body[s+3:e-3]))
start = e
tpl.append(TemplateText(body[start:])) # leftover
return tpl

def subst(self, params, extractor, depth=0):
# We perform parameter substitutions recursively.
# We also limit the maximum number of iterations to avoid too long or
# even endless loops (in case of malformed input).

# :see: http://meta.wikimedia.org/wiki/Help:Expansion#Distinction_between_variables.2C_parser_functions.2C_and_templates
#
# Parameter values are assigned to parameters in two (?) passes.
# Therefore a parameter name in a template can depend on the value of
# another parameter of the same template, regardless of the order in
# which they are specified in the template call, for example, using
# Template:ppp containing "{{{{{{p}}}}}}", {{ppp|p=q|q=r}} and even
# {{ppp|q=r|p=q}} gives r, but using Template:tvvv containing
# "{{{{{{{{{p}}}}}}}}}", {{tvvv|p=q|q=r|r=s}} gives s.

logging.debug('subst tpl (%d, %d) %s', len(extractor.frame), depth, self)

if depth > extractor.maxParameterRecursionLevels:
extractor.recursion_exceeded_3_errs += 1
return ''

return ''.join([tpl.subst(params, extractor, depth) for tpl in self])

def __str__(self):
return ''.join([str(x) for x in self])


class TemplateText(str):
"""Fixed text of template"""

def subst(self, params, extractor, depth):
return self


class TemplateArg():
"""
parameter to a template.
Has a name and a default value, both of which are Templates.
"""
def __init__(self, parameter):
"""
:param parameter: the parts of a tplarg.
"""
# the parameter name itself might contain templates, e.g.:
# appointe{{#if:{{{appointer14|}}}|r|d}}14|
# 4|{{{{{subst|}}}CURRENTYEAR}}

# any parts in a tplarg after the first (the parameter default) are
# ignored, and an equals sign in the first part is treated as plain text.
#logging.debug('TemplateArg %s', parameter)

parts = splitParts(parameter)
self.name = Template.parse(parts[0])
if len(parts) > 1:
# This parameter has a default value
self.default = Template.parse(parts[1])
else:
self.default = None

def __str__(self):
if self.default:
return '{{{%s|%s}}}' % (self.name, self.default)
else:
return '{{{%s}}}' % self.name

def subst(self, params, extractor, depth):
"""
Substitute value for this argument from dict :param params:
Use :param extractor: to evaluate expressions for name and default.
Limit substitution to the maximun :param depth:.
"""
# the parameter name itself might contain templates, e.g.:
# appointe{{#if:{{{appointer14|}}}|r|d}}14|
paramName = self.name.subst(params, extractor, depth+1)
paramName = extractor.expandTemplates(paramName)
res = ''
if paramName in params:
res = params[paramName] # use parameter value specified in template invocation
elif self.default: # use the default value
defaultValue = self.default.subst(params, extractor, depth+1)
res = extractor.expandTemplates(defaultValue)
#logging.debug('subst arg %d %s -> %s' % (depth, paramName, res))
return res


# ======================================================================

substWords = 'subst:|safesubst:'
Expand All @@ -923,10 +812,6 @@ class Extractor():
# Whether to produce json instead of the default <doc> output format.
toJson = False

##
# Obtained from TemplateNamespace
templatePrefix = ''

def __init__(self, id, revid, urlbase, title, page):
"""
:param page: a list of lines.
Expand All @@ -943,14 +828,12 @@ def __init__(self, id, revid, urlbase, title, page):
self.recursion_exceeded_3_errs = 0 # parameter recursion
self.template_title_errs = 0

def clean_text(self, text, mark_headers=False, expand_templates=True,
def clean_text(self, text, mark_headers=False, expand_templates=False,
html_safe=True):
"""
:param mark_headers: True to distinguish headers from paragraphs
e.g. "## Section 1"
"""
self.magicWords['namespace'] = self.title[:max(0, self.title.find(":"))]
#self.magicWords['namespacenumber'] = '0' # for article,
self.magicWords['pagename'] = self.title
self.magicWords['fullpagename'] = self.title
self.magicWords['currentyear'] = time.strftime('%Y')
Expand Down Expand Up @@ -1007,7 +890,7 @@ def extract(self, out, html_safe=True):
# Expand templates

maxTemplateRecursionLevels = 30
maxParameterRecursionLevels = 16
maxParameterRecursionLevels = 10

# check for template beginning
reOpen = re.compile('(?<!{){{(?!{)', re.DOTALL)
Expand Down Expand Up @@ -1096,11 +979,7 @@ def templateParams(self, parameters):
# The '=' might occurr within an HTML attribute:
# "&lt;ref name=value"
# but we stop at first.

# The '=' might occurr within quotes:
# ''''<span lang="pt-pt" xml:lang="pt-pt">cénicas</span>'''

m = re.match(" *([^=']*?) *=(.*)", param, re.DOTALL)
m = re.match(' *([^=]*?) *=(.*)', param, re.DOTALL)
if m:
# This is a named parameter. This case also handles parameter
# assignments like "2=xxx", where the number of an unnamed
Expand Down Expand Up @@ -1395,7 +1274,7 @@ def findMatchingBraces(text, ldelim=0):

if ldelim: # 2-3
reOpen = re.compile('[{]{%d,}' % ldelim) # at least ldelim
reNext = re.compile('[{]{2,}|}{2,}') # at least 2 open or close bracces
reNext = re.compile('[{]{2,}|}{2,}') # at least 2
else:
reOpen = re.compile('{{2,}|\[{2,}')
reNext = re.compile('{{2,}|}{2,}|\[{2,}|]{2,}') # at least 2
Expand Down Expand Up @@ -1561,7 +1440,7 @@ def fullyQualifiedTemplateTitle(templateTitle):
# space]], but having in the system a redirect page with an empty title
# causes numerous problems, so we'll live happier without it.
if templateTitle:
return Extractor.templatePrefix + ucfirst(templateTitle)
return templatePrefix + ucfirst(templateTitle)
else:
return '' # caller may log as error

Expand Down Expand Up @@ -1611,7 +1490,7 @@ def sharp_expr(expr):
expr = re.sub('mod', '%', expr)
expr = re.sub('\bdiv\b', '/', expr)
expr = re.sub('\bround\b', '|ROUND|', expr)
return str(eval(expr))
return unicode(eval(expr))
except:
return '<span class="error"></span>'

Expand Down Expand Up @@ -1763,17 +1642,13 @@ def sharp_invoke(module, function, frame):

'int': lambda string, *rest: str(int(string)),

'padleft': lambda char, width, string: string.ljust(char, int(pad)), # CHECK_ME

}


def callParserFunction(functionName, args, frame):
"""
Parser functions have similar syntax as templates, except that
the first argument is everything after the first colon.
:param functionName: nameof the parser function
:param args: the arguments to the function
:return: the result of the invocation, None in case of failure.

http://meta.wikimedia.org/wiki/Help:ParserFunctions
Expand All @@ -1783,11 +1658,11 @@ def callParserFunction(functionName, args, frame):
if functionName == '#invoke':
# special handling of frame
ret = sharp_invoke(args[0].strip(), args[1].strip(), frame)
# logging.debug('parserFunction> %s %s', args[1], ret)
# logging.debug('parserFunction> %s %s', functionName, ret)
return ret
if functionName in parserFunctions:
ret = parserFunctions[functionName](*args)
# logging.debug('parserFunction> %s(%s) %s', functionName, args, ret)
# logging.debug('parserFunction> %s %s', functionName, ret)
return ret
except:
return "" # FIXME: fix errors
Expand All @@ -1801,7 +1676,7 @@ def callParserFunction(functionName, args, frame):
reNoinclude = re.compile(r'<noinclude>(?:.*?)</noinclude>', re.DOTALL)
reIncludeonly = re.compile(r'<includeonly>|</includeonly>', re.DOTALL)

# These are built before spawning processes, hence they are shared.
# These are built before spawning processes, hence thay are shared.
templates = {}
redirects = {}
# cache of parser templates
Expand Down Expand Up @@ -1854,6 +1729,6 @@ def define_template(title, page):
text = reIncludeonly.sub('', text)

if text:
if title in templates and templates[title] != text:
if title in templates:
logging.warn('Redefining: %s', title)
templates[title] = text