From e093bcff24e31fe39bd4179e226c8eb398323e9f Mon Sep 17 00:00:00 2001 From: Afif Ali Saadman <98092569+afifafifafifafifali@users.noreply.github.com> Date: Thu, 16 Oct 2025 09:05:23 +0600 Subject: [PATCH] Fix some errors --- wikiextractor/extract.py | 205 ++++++++------------------------------- 1 file changed, 40 insertions(+), 165 deletions(-) diff --git a/wikiextractor/extract.py b/wikiextractor/extract.py index a00e23d2..004d3fc8 100644 --- a/wikiextractor/extract.py +++ b/wikiextractor/extract.py @@ -199,12 +199,7 @@ def compact(text, mark_headers=False): for line in text.split('\n'): if not line: - if len(listLevel): # implies Extractor.HtmlFormatting - for c in reversed(listLevel): - page.append(listClose[c]) - listLevel = '' continue - # Handle section titles m = section.match(line) if m: @@ -232,35 +227,36 @@ def compact(text, mark_headers=False): page.append(title) # handle indents elif line[0] == ':': - page.append(line.lstrip(':')) + # page.append(line.lstrip(':*#;')) + continue # handle lists - # @see https://www.mediawiki.org/wiki/Help:Formatting - elif line[0] in '*#;': + elif line[0] in '*#;:': if Extractor.HtmlFormatting: - # close extra levels - l = 0 - for c in listLevel: - if l < len(line) and c != line[l]: - for extra in reversed(listLevel[l:]): - page.append(listClose[extra]) - listLevel = listLevel[:l] - break - l += 1 - if l < len(line) and line[l] in '*#;:': - # add new level (only one, no jumps) - # FIXME: handle jumping levels - type = line[l] - page.append(listOpen[type]) - listLevel += type - line = line[l+1:].strip() - else: - # continue on same level - type = line[l-1] - line = line[l:].strip() - page.append(listItem[type] % line) + i = 0 + for c, n in zip_longest(listLevel, line, fillvalue=''): + if not n or n not in '*#;:': + if c: + page.append(listClose[c]) + listLevel = listLevel[:-1] + continue + else: + break + # n != '' + if c != n and (not c or (c not in ';:' and n not in ';:')): + if c: + # close level + page.append(listClose[c]) + listLevel = listLevel[:-1] + listLevel += n + page.append(listOpen[n]) + i += 1 + n = line[i - 1] # last list char + line = line[i:].strip() + if line: # FIXME: n is '"' + page.append(listItem[n] % line) else: continue - elif len(listLevel): # implies Extractor.HtmlFormatting + elif len(listLevel): for c in reversed(listLevel): page.append(listClose[c]) listLevel = [] @@ -380,14 +376,15 @@ def dropSpans(spans, text): # as well as U+3000 is IDEOGRAPHIC SPACE for bug 19052 EXT_LINK_URL_CLASS = r'[^][<>"\x00-\x20\x7F\s]' ExtLinkBracketedRegex = re.compile( - '\[(((?i)' + '|'.join(wgUrlProtocols) + ')' + EXT_LINK_URL_CLASS + r'+)\s*([^\]\x00-\x08\x0a-\x1F]*?)\]', + '(?i)\[((' + '|'.join(wgUrlProtocols) + ')' + EXT_LINK_URL_CLASS + r'+)\s*([^\]\x00-\x08\x0a-\x1F]*?)\]', re.S | re.U) EXT_IMAGE_REGEX = re.compile( - r"""^(http://|https://)([^][<>"\x00-\x20\x7F\s]+) - /([A-Za-z0-9_.,~%\-+&;#*?!=()@\x80-\xFF]+)\.((?i)gif|png|jpg|jpeg)$""", + r"""(?i)^(http://|https://)([^][<>"\x00-\x20\x7F\s]+) + /([A-Za-z0-9_.,~%\-+&;#*?!=()@\x80-\xFF]+)\.(gif|png|jpg|jpeg)$""", re.X | re.S | re.U) + def replaceExternalLinks(text): s = '' cur = 0 @@ -790,114 +787,6 @@ def resetIgnoredTags(): # Matches dots dots = re.compile(r'\.{4,}') -# ====================================================================== - -class Template(list): - """ - A Template is a list of TemplateText or TemplateArgs - """ - - @classmethod - def parse(cls, body): - tpl = Template() - # we must handle nesting, s.a. - # {{{1|{{PAGENAME}}} - # {{{italics|{{{italic|}}} - # {{#if:{{{{{#if:{{{nominee|}}}|nominee|candidate}}|}}}| - # - start = 0 - for s,e in findMatchingBraces(body, 3): - tpl.append(TemplateText(body[start:s])) - tpl.append(TemplateArg(body[s+3:e-3])) - start = e - tpl.append(TemplateText(body[start:])) # leftover - return tpl - - def subst(self, params, extractor, depth=0): - # We perform parameter substitutions recursively. - # We also limit the maximum number of iterations to avoid too long or - # even endless loops (in case of malformed input). - - # :see: http://meta.wikimedia.org/wiki/Help:Expansion#Distinction_between_variables.2C_parser_functions.2C_and_templates - # - # Parameter values are assigned to parameters in two (?) passes. - # Therefore a parameter name in a template can depend on the value of - # another parameter of the same template, regardless of the order in - # which they are specified in the template call, for example, using - # Template:ppp containing "{{{{{{p}}}}}}", {{ppp|p=q|q=r}} and even - # {{ppp|q=r|p=q}} gives r, but using Template:tvvv containing - # "{{{{{{{{{p}}}}}}}}}", {{tvvv|p=q|q=r|r=s}} gives s. - - logging.debug('subst tpl (%d, %d) %s', len(extractor.frame), depth, self) - - if depth > extractor.maxParameterRecursionLevels: - extractor.recursion_exceeded_3_errs += 1 - return '' - - return ''.join([tpl.subst(params, extractor, depth) for tpl in self]) - - def __str__(self): - return ''.join([str(x) for x in self]) - - -class TemplateText(str): - """Fixed text of template""" - - def subst(self, params, extractor, depth): - return self - - -class TemplateArg(): - """ - parameter to a template. - Has a name and a default value, both of which are Templates. - """ - def __init__(self, parameter): - """ - :param parameter: the parts of a tplarg. - """ - # the parameter name itself might contain templates, e.g.: - # appointe{{#if:{{{appointer14|}}}|r|d}}14| - # 4|{{{{{subst|}}}CURRENTYEAR}} - - # any parts in a tplarg after the first (the parameter default) are - # ignored, and an equals sign in the first part is treated as plain text. - #logging.debug('TemplateArg %s', parameter) - - parts = splitParts(parameter) - self.name = Template.parse(parts[0]) - if len(parts) > 1: - # This parameter has a default value - self.default = Template.parse(parts[1]) - else: - self.default = None - - def __str__(self): - if self.default: - return '{{{%s|%s}}}' % (self.name, self.default) - else: - return '{{{%s}}}' % self.name - - def subst(self, params, extractor, depth): - """ - Substitute value for this argument from dict :param params: - Use :param extractor: to evaluate expressions for name and default. - Limit substitution to the maximun :param depth:. - """ - # the parameter name itself might contain templates, e.g.: - # appointe{{#if:{{{appointer14|}}}|r|d}}14| - paramName = self.name.subst(params, extractor, depth+1) - paramName = extractor.expandTemplates(paramName) - res = '' - if paramName in params: - res = params[paramName] # use parameter value specified in template invocation - elif self.default: # use the default value - defaultValue = self.default.subst(params, extractor, depth+1) - res = extractor.expandTemplates(defaultValue) - #logging.debug('subst arg %d %s -> %s' % (depth, paramName, res)) - return res - - # ====================================================================== substWords = 'subst:|safesubst:' @@ -923,10 +812,6 @@ class Extractor(): # Whether to produce json instead of the default output format. toJson = False - ## - # Obtained from TemplateNamespace - templatePrefix = '' - def __init__(self, id, revid, urlbase, title, page): """ :param page: a list of lines. @@ -943,14 +828,12 @@ def __init__(self, id, revid, urlbase, title, page): self.recursion_exceeded_3_errs = 0 # parameter recursion self.template_title_errs = 0 - def clean_text(self, text, mark_headers=False, expand_templates=True, + def clean_text(self, text, mark_headers=False, expand_templates=False, html_safe=True): """ :param mark_headers: True to distinguish headers from paragraphs e.g. "## Section 1" """ - self.magicWords['namespace'] = self.title[:max(0, self.title.find(":"))] - #self.magicWords['namespacenumber'] = '0' # for article, self.magicWords['pagename'] = self.title self.magicWords['fullpagename'] = self.title self.magicWords['currentyear'] = time.strftime('%Y') @@ -1007,7 +890,7 @@ def extract(self, out, html_safe=True): # Expand templates maxTemplateRecursionLevels = 30 - maxParameterRecursionLevels = 16 + maxParameterRecursionLevels = 10 # check for template beginning reOpen = re.compile('(?cénicas''' - - m = re.match(" *([^=']*?) *=(.*)", param, re.DOTALL) + m = re.match(' *([^=]*?) *=(.*)', param, re.DOTALL) if m: # This is a named parameter. This case also handles parameter # assignments like "2=xxx", where the number of an unnamed @@ -1395,7 +1274,7 @@ def findMatchingBraces(text, ldelim=0): if ldelim: # 2-3 reOpen = re.compile('[{]{%d,}' % ldelim) # at least ldelim - reNext = re.compile('[{]{2,}|}{2,}') # at least 2 open or close bracces + reNext = re.compile('[{]{2,}|}{2,}') # at least 2 else: reOpen = re.compile('{{2,}|\[{2,}') reNext = re.compile('{{2,}|}{2,}|\[{2,}|]{2,}') # at least 2 @@ -1561,7 +1440,7 @@ def fullyQualifiedTemplateTitle(templateTitle): # space]], but having in the system a redirect page with an empty title # causes numerous problems, so we'll live happier without it. if templateTitle: - return Extractor.templatePrefix + ucfirst(templateTitle) + return templatePrefix + ucfirst(templateTitle) else: return '' # caller may log as error @@ -1611,7 +1490,7 @@ def sharp_expr(expr): expr = re.sub('mod', '%', expr) expr = re.sub('\bdiv\b', '/', expr) expr = re.sub('\bround\b', '|ROUND|', expr) - return str(eval(expr)) + return unicode(eval(expr)) except: return '' @@ -1763,8 +1642,6 @@ def sharp_invoke(module, function, frame): 'int': lambda string, *rest: str(int(string)), - 'padleft': lambda char, width, string: string.ljust(char, int(pad)), # CHECK_ME - } @@ -1772,8 +1649,6 @@ def callParserFunction(functionName, args, frame): """ Parser functions have similar syntax as templates, except that the first argument is everything after the first colon. - :param functionName: nameof the parser function - :param args: the arguments to the function :return: the result of the invocation, None in case of failure. http://meta.wikimedia.org/wiki/Help:ParserFunctions @@ -1783,11 +1658,11 @@ def callParserFunction(functionName, args, frame): if functionName == '#invoke': # special handling of frame ret = sharp_invoke(args[0].strip(), args[1].strip(), frame) - # logging.debug('parserFunction> %s %s', args[1], ret) + # logging.debug('parserFunction> %s %s', functionName, ret) return ret if functionName in parserFunctions: ret = parserFunctions[functionName](*args) - # logging.debug('parserFunction> %s(%s) %s', functionName, args, ret) + # logging.debug('parserFunction> %s %s', functionName, ret) return ret except: return "" # FIXME: fix errors @@ -1801,7 +1676,7 @@ def callParserFunction(functionName, args, frame): reNoinclude = re.compile(r'(?:.*?)', re.DOTALL) reIncludeonly = re.compile(r'|', re.DOTALL) -# These are built before spawning processes, hence they are shared. +# These are built before spawning processes, hence thay are shared. templates = {} redirects = {} # cache of parser templates @@ -1854,6 +1729,6 @@ def define_template(title, page): text = reIncludeonly.sub('', text) if text: - if title in templates and templates[title] != text: + if title in templates: logging.warn('Redefining: %s', title) templates[title] = text