diff --git a/CHANGES b/CHANGES index 01f56f4..3fea19e 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,43 @@ +0.14.4 August 15, 2026 +- fixed a regression in text boilerplate stripping. Affected only txt -> html (and thus epub, mobi, etc) +- fixed setup and added parse() invocation in tests_txt + +0.14.3 August 12, 2026 + +This release includes several important updates. There are quite a few issues still waiting to be addressed; they are not forgotten. +- To balance the long-standing policy of only using "REC" CSS against legal requirements for accessibility, we are significantly relaxing our CSS validation. This means that post-processors should take care to provide "graceful degradation" for CSS that might not be supported in every reading device. + - CSS Grid Layout Level 1 is now supported. Fixes #322. Grid layout should be strongly considered as a replacement for the use of table for layout + - CSS Flexbox Level 1 is now supported. Together with Grid, the should be no reason to use table for layout; going forward, layout tables will need to be converted to more accessible CSS based layout. + - CSS3 Text Decoration properties are now supported. These were the most often used properties removed by Ebookmaker as not yet REC. + - The first-letter property is now supported. While this is still not supported in every browser, it is supported by most ebook readers. Use this property to make drop caps in an accessible way. Fixes #310 + - Ebookmaker uses the Python module 'CSSUtils' to do CSS validation. It does not support the full grammar used in newer CSS. In particular it does not support the "var()" construct; do not expect this to change anytime soon. + - The current version of CSSUtils does not support the `rem` measurement, so CSS with rem in it will cause errors to be reported and CSS 2.1 rules containing `rem` will be removed. The current version of CSSUtils, that supports `rem`, requires a Python version newer than we have running in production. When our python version is updated, we will update CSSUtils. + +- In both in HTML and EPUB, for improved accessibility, added the CSS rule `a[href] {text-decoration: underline;}` to make sure links are underlined. Previously, we underlined all `a` elements, which ignored prior usage as an link target. + +- The `aside` element is now allowed. becomes div[class=aside] in epub2. + +- handling of inline SVG has been improved. + - the inline SVG element is now handled whether or not it has set the svg namespace. + - inline SVG is changed to standalone files in EPUB2 + - inline html5 SVG now adapted for EPUB3, which cares about namespaces + +- an error is no longer emitted when rst output is requested from html input + +- uses hN element to set the title element of each chunk in the epub. fixes #331 + +- refactored boilerplate stripping for txt files so it only needs to be done once per book. boilerplate is now detected in the text Parser, instead of in the text Writer. + +- support HTML5 `time` element. fixes #332 + +- `a` tags can't contain block tags in xhtml, so div in `a` and `p` in `a` cause EPUB2 validation errors, so Ebookmaker now changes `p` and `div` occurring in `a` to `span` for EPUB2. I'm not sure why anyone would want to use this markup in a book. Fixes #333 + +- empty `th` elements are replaced by empty `td` elements for better accessibility. Fixes #315. + + +0.14.2 July 4, 2026 +- `text-decoration` for `a` should be reset to `underline`, not initial. Fixes #316. + 0.14.1 June 12, 2026 - fixed bug where text directly under body is deleted. - updated idna and beautifulsoup dependencies diff --git a/setup.cfg b/setup.cfg index 3061be0..d6882d7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,7 +1,7 @@ [metadata] name = ebookmaker -version = 0.14.1 +version = 0.14.4 [options] package_dir= diff --git a/setup.py b/setup.py index cb02cc2..dd6cc64 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ from setuptools import setup -VERSION = '0.14.1' +VERSION = '0.14.4' if __name__ == "__main__": @@ -42,7 +42,7 @@ 'roman', 'requests', 'six>=1.4.1', - 'libgutenberg[covers]>=0.10.36', + 'libgutenberg[covers]>=0.11.2', 'cchardet==2.2.0a2', 'beautifulsoup4', 'html5lib', diff --git a/src/ebookmaker/HTMLChunker.py b/src/ebookmaker/HTMLChunker.py index 61c3b0d..4d40581 100644 --- a/src/ebookmaker/HTMLChunker.py +++ b/src/ebookmaker/HTMLChunker.py @@ -165,6 +165,9 @@ def shipout_chunk(self, attribs, chunk_id = None, comment = None): for e in xpath(self.chunk, '//mathml:math'): attribs.rel.add('mathml') break + for e in xpath(self.chunk, '//xhtml:svg'): + attribs.rel.add('svg') + break for e in xpath(self.chunk, '//svg:svg'): attribs.rel.add('svg') break @@ -305,3 +308,23 @@ def rewrite_internal_links_toc(self, toc): error("HTMLChunker: Cannot rewrite toc entry '%s'" % entry[0]) error(repr(self.idmap)) del entry + + def set_running_headers(self): + """ Use hN elements in chunks to set the chunk title + + """ + chunk_title = '' + for chunk in self.chunks: + for tag in ['h1', 'h2', 'h3', 'h4', 'h5', 'h6']: + hN_title = '' + for hN in xpath(chunk[0], f"//xhtml:{tag}"): + hN_title = hN.text_content() + if hN_title: + break + if hN_title: + chunk_title = hN_title + break + if chunk_title != '': + for title in xpath(chunk[0], f"//xhtml:title"): + title.text = chunk_title + break diff --git a/src/ebookmaker/ParserFactory.py b/src/ebookmaker/ParserFactory.py index cc7c39e..ac91be2 100644 --- a/src/ebookmaker/ParserFactory.py +++ b/src/ebookmaker/ParserFactory.py @@ -86,7 +86,7 @@ def create(cls, url, attribs=None): if attribs is None: attribs = parsers.ParserAttributes() - # debug("Need parser for %s" % url) + debug("Need parser for %s" % url) # first check if input url is in output directory (we've already made it!) if gg.is_same_path(os.path.abspath(options.outputdir), os.path.dirname(url)): @@ -101,7 +101,7 @@ def create(cls, url, attribs=None): if url in cls.parsers: - # debug("... reusing parser for %s" % url) + debug("... reusing parser for %s" % url) # reuse same parser, maybe already filled with data parser = cls.parsers[url] parser.reset() diff --git a/src/ebookmaker/Version.py b/src/ebookmaker/Version.py index c45d9d7..22f087c 100644 --- a/src/ebookmaker/Version.py +++ b/src/ebookmaker/Version.py @@ -1,2 +1,2 @@ -VERSION = '0.14.1' +VERSION = '0.14.4' GENERATOR = 'Ebookmaker %s by Project Gutenberg' diff --git a/src/ebookmaker/parsers/CSSParser.py b/src/ebookmaker/parsers/CSSParser.py index f79992a..3c57c59 100644 --- a/src/ebookmaker/parsers/CSSParser.py +++ b/src/ebookmaker/parsers/CSSParser.py @@ -28,10 +28,146 @@ RE_ELEMENT = re.compile(r'\[[^\]]*\]|((?:^|\s|\+|>|~|,)[a-z0-9]+)', re.I) mediatypes = (mt.css, ) -PG_CSS_PROFILE = ( - 'Added Properties for Project Gutenberg', - { - 'display': 'flex|initial', + +""" Flex and Grid layouts; generated by Claude +* The grid track-list grammar (repeat(), minmax(), fit-content(), + named lines, etc.) is approximated with regular expressions. It + covers the vast majority of real-world usage but is not a perfect + transcription of the CSS Grid formal grammar (which isn't regular). + Tighten or loosen the macros below if you hit edge cases. +* Known limitation: named grid lines written with square brackets, + e.g. `grid-template-columns: [full-start] 1fr [full-end];`, will + fail to parse at all -- not just fail validation. This is a + limitation of cssutils' own CSS tokenizer/parser (it doesn't know + `[` `]` can appear inside a property value), which sits below the + profile system and can't be fixed by adding a profile. Everything + else in this file (line-names macro, etc.) is kept for forward + compatibility / partial matches and in case you patch the tokenizer + yourself, but square-bracket named lines won't round-trip through + stock cssutils today. +""" + + +_MACROS = { + # basic tokens (defined locally so this profile doesn't depend on + # exactly which internal macro names cssutils happens to expose) + 'integer': r'[+-]?\d+', + 'num': r'[+-]?\d*\.?\d+(?:[eE][+-]?\d+)?', + 'length': r'(?:{num}(?:em|ex|px|in|cm|mm|pt|pc|ch|rem|vh|vw|vmin|vmax|q)|0)', + 'percentage': r'{num}%', + 'ident': r'-?[a-zA-Z_][a-zA-Z0-9_-]*', + 'string': r'''(?:"[^"]*"|'[^']*')''', + 'lop': r'(?:{length}|{percentage})', # + 'lopa': r'(?:{length}|{percentage}|auto)', # | auto + 'positive-integer': r'\d+', + + # --- Grid: track sizing --- + 'flex-value': r'{num}fr', + 'inflexible-breadth': r'(?:{length}|{percentage}|min-content|max-content|auto)', + 'track-breadth': r'(?:{length}|{percentage}|min-content|max-content|auto|{flex-value})', + 'track-size': r'(?:{track-breadth}|minmax\(\s*{inflexible-breadth}\s*,\s*{track-breadth}\s*\)|fit-content\(\s*{lop}\s*\))', + 'line-names': r'\[\s*{ident}(?:\s+{ident})*\s*\]', + 'track-repeat': r'repeat\(\s*(?:{integer}|auto-fill|auto-fit)\s*,\s*(?:\s*{line-names}?\s*{track-size}\s*)+\s*{line-names}?\s*\)', + 'track-list': r'(?:\s*{line-names}?\s*(?:{track-size}|{track-repeat})\s*)+\s*{line-names}?', + 'auto-track-list': r'(?:\s*{line-names}?\s*{track-size}\s*)*\s*{line-names}?\s*repeat\(\s*(?:auto-fill|auto-fit)\s*,\s*(?:\s*{line-names}?\s*{track-size}\s*)+\s*{line-names}?\s*\)\s*(?:\s*{line-names}?\s*{track-size}\s*)*\s*{line-names}?', + 'grid-line': r'(?:auto|span\s+(?:{integer}\s+{ident}|{ident}\s+{integer}|{integer}|{ident})|{integer}\s+{ident}|{ident}\s+{integer}|{integer}|{ident})', + + # --- Shared box-alignment values (used by both grid and flexbox) --- + 'content-position': r'(?:center|start|end|flex-start|flex-end|left|right)', + 'self-position': r'(?:center|start|end|flex-start|flex-end|self-start|self-end)', + 'overflow-position': r'(?:safe|unsafe)', + 'baseline-position': r'(?:(?:first|last)\s+)?baseline', + + # simplified : named/keyword ident, #hex, rgb()/rgba()/hsl()/hsla() + 'hexcolor': r'\#(?:[0-9a-fA-F]{3}|[0-9a-fA-F]{4}|[0-9a-fA-F]{6}|[0-9a-fA-F]{8})', + 'colorfunc': r'(?:rgb|rgba|hsl|hsla)\([^)]*\)', + 'namedcolor': r'(?:currentColor|transparent|{ident})', + 'color': r'(?:{hexcolor}|{colorfunc}|{namedcolor})', + + # text-shadow: ? && {2,3} (offsets/blur, in either order relative to color) + 'shadow-item': r'(?:{color}\s+)?{length}\s+{length}(?:\s+{length})?(?:\s+{color})?', + + # from PG_CSS_PROFILE + 'numeric-figure-values': 'lining-nums|oldstyle-nums', + 'numeric-spacing-values': 'proportional-nums|tabular-nums', + 'numeric-fraction-values': 'diagonal-fractions|stacked-fractions', + 'font-variant-attrs': '{numeric-figure-values}|{numeric-spacing-values}|{numeric-fraction-values}|ordinal|slashed-zero', + +} + +# --------------------------------------------------------------------------- +# CSS Grid Layout Module Level 1 +# https://www.w3.org/TR/css-grid-1/ +# --------------------------------------------------------------------------- + +GRID_PROPERTIES = { + 'display': r'(?:grid|inline-grid)', + + 'grid-template-columns': r'(?:none|subgrid|{track-list}|{auto-track-list})', + 'grid-template-rows': r'(?:none|subgrid|{track-list}|{auto-track-list})', + 'grid-template-areas': r'(?:none|(?:{string}\s*)+)', + 'grid-template': r'(?:none|.+)', # shorthand: permissive (complex grammar) + 'grid': r'.+', # shorthand: permissive (complex grammar) + + 'grid-auto-columns': r'(?:{track-size}\s*)+', + 'grid-auto-rows': r'(?:{track-size}\s*)+', + 'grid-auto-flow': r'(?:(?:row|column)(?:\s+dense)?|dense)', + + 'grid-row-start': r'{grid-line}', + 'grid-row-end': r'{grid-line}', + 'grid-column-start': r'{grid-line}', + 'grid-column-end': r'{grid-line}', + 'grid-row': r'{grid-line}(?:\s*/\s*{grid-line})?', + 'grid-column': r'{grid-line}(?:\s*/\s*{grid-line})?', + 'grid-area': r'{grid-line}(?:\s*/\s*{grid-line}){0,3}', + + # gap properties (introduced alongside Grid Level 1, later folded + # into Box Alignment Level 3 -- both spellings are included) + 'grid-row-gap': r'{lop}', + 'grid-column-gap': r'{lop}', + 'grid-gap': r'{lop}(?:\s+{lop})?', + 'row-gap': r'{lop}', + 'column-gap': r'{lop}', + 'gap': r'{lop}(?:\s+{lop})?', + + 'justify-items': r'(?:normal|stretch|{baseline-position}|{overflow-position}?\s*{self-position}|legacy(?:\s+(?:left|right|center))?)', + 'justify-self': r'(?:auto|normal|stretch|{baseline-position}|{overflow-position}?\s*{self-position})', +} + +# --------------------------------------------------------------------------- +# CSS Flexible Box Layout Module Level 1 +# https://www.w3.org/TR/css-flexbox-1/ +# --------------------------------------------------------------------------- + +FLEXBOX_PROPERTIES = { + 'display': r'(?:flex|inline-flex)', + + 'flex-direction': r'(?:row|row-reverse|column|column-reverse)', + 'flex-wrap': r'(?:nowrap|wrap|wrap-reverse)', + 'flex-flow': r'(?:(?:row|row-reverse|column|column-reverse)(?:\s+(?:nowrap|wrap|wrap-reverse))?|(?:nowrap|wrap|wrap-reverse)(?:\s+(?:row|row-reverse|column|column-reverse))?)', + + 'order': r'{integer}', + 'flex-grow': r'{num}', + 'flex-shrink': r'{num}', + 'flex-basis': r'(?:content|{lopa})', + 'flex': r'(?:none|(?:auto|{num})(?:\s+{num})?(?:\s+{lopa})?)', +} + +# --------------------------------------------------------------------------- +# Shared CSS Box Alignment values used by flex/grid containers and items +# (kept as a separate profile so it isn't tied to grid- or flex-only display) +# --------------------------------------------------------------------------- + +BOX_ALIGNMENT_PROPERTIES = { + 'align-items': r'(?:normal|stretch|{baseline-position}|{overflow-position}?\s*{self-position})', + 'align-self': r'(?:auto|normal|stretch|{baseline-position}|{overflow-position}?\s*{self-position})', + 'align-content': r'(?:normal|{baseline-position}|{overflow-position}?\s*{content-position}|space-between|space-around|space-evenly|stretch)', + 'justify-content': r'(?:normal|{overflow-position}?\s*{content-position}|space-between|space-around|space-evenly|stretch)', +} + + +PG_CSS_PROFILE = { + 'display': 'grid|inline-grid|flex|inline-flex|block|inline|none', 'justify-content': 'center', 'speak': r'auto|never|always', 'speak-as': 'normal|spell-out|digits|literal-punctuation|no-punctuation', @@ -60,26 +196,58 @@ 'padding-left': 'initial', 'padding': 'initial', 'text-align': 'initial', - 'text-decoration': 'initial', 'text-indent': 'initial', 'text-transform': 'initial', - # updated for https://www.w3.org/TR/css-writing-modes-3/ # direction and unicode-bidi are not supported based on the standard's recommendation 'writing-mode': 'vertical-lr|vertical-rl|horizontal-tb', 'text-orientation': 'mixed|upright|sideways', 'text-combine-upright': 'none | all', - }, - { - 'numeric-figure-values': 'lining-nums|oldstyle-nums', - 'numeric-spacing-values': 'proportional-nums|tabular-nums', - 'numeric-fraction-values': 'diagonal-fractions|stacked-fractions', - 'font-variant-attrs': '{numeric-figure-values}|{numeric-spacing-values}|{numeric-fraction-values}|ordinal|slashed-zero', - } -) - -cssutils.profile.addProfiles([PG_CSS_PROFILE]) + + # add css3 decoration properties + 'text-decoration-line': r'none|underline|overline|line-through', + 'text-decoration-style': r'solid|double|dotted|dashed|wavy', + 'text-decoration-color': r'{color}', + 'text-decoration': r'(?:none|(?:(?:underline|overline|line-through|blink)(?:\s+(?:underline|overline|line-through|blink)){0,3}|solid|double|dotted|dashed|wavy|{color})(?:\s+(?:(?:underline|overline|line-through|blink)(?:\s+(?:underline|overline|line-through|blink)){0,3}|solid|double|dotted|dashed|wavy|{color})){0,2})', + + # shorthand: || + 'text-emphasis': r'(?:none|(?:{string}|(?:filled|open)(?:\s+(?:dot|circle|double-circle|triangle|sesame))?|(?:dot|circle|double-circle|triangle|sesame)(?:\s+(?:filled|open))?|{color})(?:\s+(?:{string}|(?:filled|open)(?:\s+(?:dot|circle|double-circle|triangle|sesame))?|(?:dot|circle|double-circle|triangle|sesame)(?:\s+(?:filled|open))?|{color}))?)', + 'text-emphasis-style': r'(?:none|{string}|(?:filled|open)(?:\s+(?:dot|circle|double-circle|triangle|sesame))?|(?:dot|circle|double-circle|triangle|sesame)(?:\s+(?:filled|open))?)', + 'text-emphasis-color': r'{color}', + + # text-underline-position: auto | [ under || [ left | right ] ] + 'text-underline-position': r'(?:auto|under(?:\s+(?:left|right))?|(?:left|right)(?:\s+under)?)', + + # text-emphasis-position: [ over | under ] && [ right | left ] + 'text-emphasis-position': r'(?:(?:over|under)\s+(?:right|left)|(?:right|left)\s+(?:over|under))', + + # text-shadow: none | # + 'text-shadow': r'(?:none|{shadow-item}(?:\s*,\s*{shadow-item})*)', + + # initial letter for accessible drop caps + # = the sink's size, in number of lines; the optional + # = the sink, how many lines it descends (defaults to + # round() when omitted) + 'initial-letter': r'(?:normal|{num}(?:\s+{integer})?)', + + # initial-letter-align: [ auto | alphabetic | hanging | ideographic ] + 'initial-letter-align': r'(?:auto|alphabetic|hanging|ideographic)', + + # initial-letter-wrap: none | first | all | grid | + 'initial-letter-wrap': r'(?:none|first|all|grid|{lop})', + + +} + +# --------------------------------------------------------------------------- +# Register the profiles with cssutils +# --------------------------------------------------------------------------- + +cssutils.profile.addProfile('CSS Grid Layout Module Level 1', GRID_PROPERTIES, _MACROS) +cssutils.profile.addProfile('CSS Flexible Box Layout Module Level 1', FLEXBOX_PROPERTIES, _MACROS) +cssutils.profile.addProfile('CSS Box Alignment (Flexbox/Grid)', BOX_ALIGNMENT_PROPERTIES, _MACROS) +cssutils.profile.addProfile('Added Properties for Project Gutenberg', PG_CSS_PROFILE, _MACROS) class Parser(ParserBase): """ Parse an external CSS file. """ diff --git a/src/ebookmaker/parsers/GutenbergTextParser.py b/src/ebookmaker/parsers/GutenbergTextParser.py index 62fed67..af1595a 100644 --- a/src/ebookmaker/parsers/GutenbergTextParser.py +++ b/src/ebookmaker/parsers/GutenbergTextParser.py @@ -469,6 +469,14 @@ def __init__(self, attribs=None): self.body = 0 self.max_blanks = 0 self.pars = [] + self.text = "" + self.pg_header = "" + self.pg_footer = "" + + + def unicode_content(self): + return self.pg_header + self.text + self.pg_footer + def get_charset_from_meta(self): """ Parse text for hints about charset. """ @@ -625,13 +633,13 @@ def parse(self): if self.xhtml is not None: return - text = self.unicode_content() - text, pg_header, pg_footer = strip_headers_from_txt(text) - if 'x-header' in pg_header and options.production: + text = HTMLParserBase.unicode_content(self) + self.text, self.pg_header, self.pg_footer = strip_headers_from_txt(text) + if 'x-header' in self.pg_header and options.production: error('header marker is missing in %s', self.attribs.url) - if 'x-header' in pg_footer and options.production: + if 'x-header' in self.pg_footer and options.production: error('footer marker is missing in %s', self.attribs.url) - + text = self.text text = parsers.RE_RESTRICTED.sub('', text) text = gg.xmlspecialchars(text) @@ -684,12 +692,18 @@ def parse(self): for body in xpath(self.xhtml, '//xhtml:body'): xhtmlparser = lxml.html.XHTMLParser(huge_tree=True) - body.append(etree.fromstring(pg_header, xhtmlparser)) + pg_header_pre = etree.Element(NS.xhtml.pre) + pg_header_pre.attrib['id'] = 'pg-header' + pg_header_pre.text = self.pg_header + body.append(pg_header_pre) for par in self.pars: p = etree.fromstring(self.ship_out(par), xhtmlparser) p.tail = '\n\n' body.append(p) - body.append(etree.fromstring(pg_footer, xhtmlparser)) + pg_footer_pre = etree.Element(NS.xhtml.pre) + pg_footer_pre.text = self.pg_footer + pg_footer_pre.attrib['id'] = 'pg-footer' + body.append(pg_footer_pre) self.pars = [] diff --git a/src/ebookmaker/parsers/HTMLParser.py b/src/ebookmaker/parsers/HTMLParser.py index e4d0434..98f6ec7 100644 --- a/src/ebookmaker/parsers/HTMLParser.py +++ b/src/ebookmaker/parsers/HTMLParser.py @@ -423,6 +423,12 @@ def captionid(): colgroup.append(col) table.insert(0, colgroup) + # empty th should be td + + for th in xpath(self.xhtml, "//xhtml:th[not(normalize-space())]"): + # these should be td instead + th.tag = NS.xhtml.td + # move lang to xml:lang for elem in xpath(self.xhtml, "//xhtml:*[@lang]"): diff --git a/src/ebookmaker/parsers/ImageParser.py b/src/ebookmaker/parsers/ImageParser.py index 901a80a..b59200f 100644 --- a/src/ebookmaker/parsers/ImageParser.py +++ b/src/ebookmaker/parsers/ImageParser.py @@ -20,6 +20,7 @@ from PIL import Image, ImageFile from lxml import etree +from libgutenberg.GutenbergGlobals import NS from libgutenberg.Logger import debug, critical, error from libgutenberg.MediaTypes import mediatypes as mt from ebookmaker.parsers import ParserBase @@ -171,6 +172,7 @@ def serialize(self): atts_to_remove = ['data-variant', 'focusable', 'role'] try: tree = etree.parse(io.BytesIO(self.image_data)) + svg = tree.getroot() except etree.XMLSyntaxError as e: critical(f'SVG image {self.attribs.url} was badly formed XML: {e}') return self.image_data @@ -181,5 +183,11 @@ def serialize(self): for att in copy.copy(element.attrib): if att.startswith('aria-'): del element.attrib[att] + # strip namespaces hanging around, perhaps because it's an extracted image + element.tag = etree.QName(element).localname + etree.cleanup_namespaces(tree) + # restore the root namespace + svg.tag = NS.svg.svg + self.image_data = etree.tostring(tree, encoding="utf-8") return self.image_data diff --git a/src/ebookmaker/parsers/boilerplate.py b/src/ebookmaker/parsers/boilerplate.py index 2093a84..d51476f 100644 --- a/src/ebookmaker/parsers/boilerplate.py +++ b/src/ebookmaker/parsers/boilerplate.py @@ -51,6 +51,8 @@ re.compile(r"\** ?These\s+\w+\s+Were\s+Prepared\s+By\s+Thousands", re.I), ] MARKER_END = re.compile(r"\*+") +STRIPPED_START = '*** START OF THE PROJECT GUTENBERG ***' +STRIPPED_END = '*** END OF THE PROJECT GUTENBERG ***' def prune(root, divider, after=True): ''' prune parts of the root element before or after a divider ''' @@ -146,8 +148,9 @@ def mark_bp(node, mark, markers, top=True): def strip_headers_from_txt(text): ''' - when input is plain text, strip the heaters and return (stripped_text, pg_header, pg_footer) + when input is plain text, strip the headers and return (stripped_text, pg_header, pg_footer) ''' + debug('stripping headers from txt') def markers_split(text, markers): for marker in markers: divider = marker.search(text) @@ -156,32 +159,24 @@ def markers_split(text, markers): after_sections = MARKER_END.split(after, maxsplit=1) if len(after_sections) == 2 and len(after_sections[0]) < 500: after = after_sections[1] - return before, divider.group(0), after return text, None, text header_text, divider, text = markers_split(text, TOP_MARKERS + SMALLPRINT_MARKERS) if divider is None: - pg_header = '
'
+        pg_header = '\n*** x-header ***\n'
         info('No PG header found in txt file.')
 
     else:
         divider_tail = ''
         if '\n' in text:
             divider_tail, text = text.split('\n', maxsplit=1)
-        pg_header = '\n'.join([
-            '
',
-            xmlspecialchars(header_text),
-            xmlspecialchars(divider),
-            xmlspecialchars(divider_tail),
-            '
']) + pg_header = '\n'.join([header_text, STRIPPED_START, divider_tail]) text, divider, footer_text = markers_split(text, BOTTOM_MARKERS) if divider is None: - pg_footer = '' + pg_footer = '\n*** x-header ***\n' info('No PG footer found in txt file.') else: - pg_footer = '\n'.join(['']) + pg_footer = '\n'.join([STRIPPED_END, footer_text]) + return text, pg_header, pg_footer diff --git a/src/ebookmaker/writers/Epub3Writer.py b/src/ebookmaker/writers/Epub3Writer.py index 04332a5..48bc3df 100644 --- a/src/ebookmaker/writers/Epub3Writer.py +++ b/src/ebookmaker/writers/Epub3Writer.py @@ -113,7 +113,7 @@ } body.x-ebookmaker.x-ebookmaker-3 .pgshow { visibility: visible; - display: initial; + display: inline; } """ @@ -560,6 +560,10 @@ def html_for_epub3(xhtml): # add namespace to math elements for e in xpath(xhtml, "//xhtml:math"): e.attrib['xmlns'] = "http://www.w3.org/1998/Math/MathML" + # add namespace to svg + for e in xpath(xhtml, "//xhtml:svg"): + e.attrib['xmlns'] = "http://www.w3.org/2000/svg" + @staticmethod def fix_incompatible_css(sheet): @@ -783,6 +787,7 @@ def build(self, job): # after splitting html into chunks we have to rewrite all # internal links in HTML chunker.rewrite_internal_links() + chunker.set_running_headers() # also in the TOC if not ncx.toc: ncx.toc.append([job.spider.parsers[0].attribs.url, 'Start', 1]) diff --git a/src/ebookmaker/writers/EpubWriter.py b/src/ebookmaker/writers/EpubWriter.py index a1dd42b..89c4afb 100644 --- a/src/ebookmaker/writers/EpubWriter.py +++ b/src/ebookmaker/writers/EpubWriter.py @@ -110,6 +110,9 @@ a.pgkilled { text-decoration: none; } +a[href] { + text-decoration: underline; +} img.x-ebookmaker-cover {max-width: 100%;} #pg-header {page-break-after: always;} #pg-footer {page-break-before: always;} @@ -1047,7 +1050,8 @@ def fix_html5(xhtml): attrs_to_remove = [('*', 'role'), ('*', 'itemid'), ('*', 'itemprop'), ('*', 'itemref'), ('*', 'itemscope'), - ('*', 'itemtype'), ('ol', 'start'), ('li', 'value'), ('*', 'focusable')] + ('*', 'itemtype'), ('ol', 'start'), ('li', 'value'), ('*', 'focusable'), + ('time', 'datetime')] svgattrs_to_remove = [('*', 'role'), ('*', 'focusable')] @@ -1077,14 +1081,23 @@ def fix_html5(xhtml): # replace html5 block tags usedtags = set() - for newtag in ['article', 'figcaption', 'figure', 'footer', 'header', 'section', 'nav', 'main']: + for newtag in ['article', 'figcaption', 'figure', 'footer', 'header', 'section', 'nav', + 'main', 'aside']: for tag in xpath(xhtml, f'//xhtml:{newtag}'): usedtags.add(newtag) tag.tag = NS.xhtml.div writers.HTMLWriter.add_class(tag, newtag) + # replace html5 block tags in forbidden contexts + for badblock in [('a', 'p'), ('a', 'div')]: + tag, blocktag = badblock + for block in xpath(xhtml, f'//xhtml:{tag}/xhtml:{blocktag}'): + usedtags.add(block) + block.tag = NS.xhtml.span + writers.HTMLWriter.add_class(block, f'{tag}_{blocktag}') + # replace html5 inline tags - for newtag in ['u', 'ruby', 'rt', 'rp']: + for newtag in ['u', 'ruby', 'rt', 'rp', 'time']: for tag in xpath(xhtml, f'//xhtml:{newtag}'): usedtags.add(newtag) tag.tag = NS.xhtml.span @@ -1117,11 +1130,40 @@ def fix_html5(xhtml): tag.clear() tag.text = text + @staticmethod + def extract_svg(xhtml, parent_url): + """ + convert embedded svg elements to stand-alone svg files that work better n EPUB2. - - - - + """ + svgnum = 0 + new_parsers = [] + for svg in xpath(xhtml, '//xhtml:svg'): + # make a new parser + attribs = parsers.ParserAttributes() + svg_parser = parsers.ImageParser.Parser(attribs=attribs) + attribs.mediatype = mt.svg + attribs.id = f'extracted_svg_{svgnum}' + attribs.url = urllib.parse.urljoin(parent_url, f'images/extracted_svg_{svgnum}.svg') + svg_parser.image_data = etree.tostring(svg) + + # turn the svg element into an img element + for att in svg.attrib: + if att not in {"alt", "class", "dir", "height", "id", "ismap", "lang", "longdesc", + "src", "title", "usemap", "width", "xml:lang"}: + del svg.attrib[att] + for title in xpath(svg, '//xhtml:title'): + svg.attrib['alt'] = title.text_content() or '' + break + svg.tag = NS.xhtml.img + svg.attrib['src'] = attribs.url + svg.attrib['id'] = attribs.id + for child in list(svg): + svg.remove(child) + + # return the new parsers + new_parsers.append(svg_parser) + return new_parsers @staticmethod def strip_links(xhtml, manifest): @@ -1344,6 +1386,7 @@ def shipout(self, job, parserlist, ncx): if p.mediatype() == mt.xhtml: opf.spine_item_from_parser(p) else: + debug(f'adding {p.attribs.url} to manifest') opf.manifest_item_from_parser(p) except Exception as what: error("Could not process file %s: %s" % (p.attribs.url, what)) @@ -1449,8 +1492,14 @@ def build(self, job): p.remap_links(idmap) xhtml.make_links_absolute(base_url=p.attribs.url) - self.fix_html5(xhtml) + new_parsers = self.extract_svg(xhtml, p.attribs.url) + + # add the new parsers to parser list so the get handled properly + parserlist.extend(new_parsers) + job.spider.parsers.extend(new_parsers) + self.fix_html5(xhtml) + strip_classes = self.get_classes_with_prop(xhtml) strip_classes = strip_classes.intersection(STRIP_CLASSES) if strip_classes: @@ -1520,6 +1569,7 @@ def build(self, job): # after splitting html into chunks we have to rewrite all # internal links in HTML chunker.rewrite_internal_links() + chunker.set_running_headers() # also in the TOC if not ncx.toc: ncx.toc.append([job.spider.parsers[0].attribs.url, 'Start', 1]) diff --git a/src/ebookmaker/writers/HTMLWriter.py b/src/ebookmaker/writers/HTMLWriter.py index 3d01b96..878f09b 100644 --- a/src/ebookmaker/writers/HTMLWriter.py +++ b/src/ebookmaker/writers/HTMLWriter.py @@ -350,7 +350,7 @@ def style_filter(style, patt=SHOULD_MOVE ): # don't let the source change the body bgcolor or text color new_rule = css.CSSStyleRule(selectorText='body', style='background:initial;color:initial') sheet.add(new_rule) - new_rule = css.CSSStyleRule(selectorText='a', style='text-decoration:initial') + new_rule = css.CSSStyleRule(selectorText='a[href]', style='text-decoration:underline') sheet.add(new_rule) @staticmethod diff --git a/src/ebookmaker/writers/RSTWriter.py b/src/ebookmaker/writers/RSTWriter.py index 45a0c04..e216fbc 100644 --- a/src/ebookmaker/writers/RSTWriter.py +++ b/src/ebookmaker/writers/RSTWriter.py @@ -31,6 +31,11 @@ def build (self, job): debug ("Creating RST file: %s" % filename) parser = ParserFactory.ParserFactory.create (job.url) + + has_txt_source = 'text/plain' in str(parser.attribs.orig_mediatype) + if not has_txt_source: + debug("needs plain text file for conversion: %s from %s", filename, job.url) + return data = parser.preprocess ('utf-8').encode ('utf-8') diff --git a/src/ebookmaker/writers/TxtWriter.py b/src/ebookmaker/writers/TxtWriter.py index 1d59c7c..1bb68fd 100644 --- a/src/ebookmaker/writers/TxtWriter.py +++ b/src/ebookmaker/writers/TxtWriter.py @@ -38,6 +38,7 @@ def insert_boilerplate(job, text): + debug("inserting boilerplate") text, header, footer = strip_headers_from_txt(text) pg_header = pgheader(job.dc).text_content() pg_footer = pgfooter(job.dc).text_content() diff --git a/tests/files/43172/43172-h/43172-h.html b/tests/files/43172/43172-h/43172-h.html index d92962e..ea4bf1c 100644 --- a/tests/files/43172/43172-h/43172-h.html +++ b/tests/files/43172/43172-h/43172-h.html @@ -382,9 +382,9 @@

Just testing

-Quella che si suole chiamare Rivoluzione dell'89, +Quella che si suole chiamare Rivoluzione dell'89, , non fu che una grande rivolta e un grande -delitto politico che servì ad aumentare una triste +delitto politico che servì ad aumentare una triste serie di comuni delitti; per cui chi vuol trattare dei suoi elementi criminosi dovrebbe rifarne tutta la storia; il che nè è mio cómpito, nè sarebbe diff --git a/tests/test_txt.py b/tests/test_txt.py index 88e4c30..faac1ed 100755 --- a/tests/test_txt.py +++ b/tests/test_txt.py @@ -6,9 +6,15 @@ import ebookmaker +from ebookmaker.ParserFactory import load_parsers, ParserFactory +from ebookmaker.CommonCode import Options +from ebookmaker.EbookMaker import config + +options = Options() class TestFromTxt(unittest.TestCase): def setUp(self): + config() self.sample_dir = os.path.join(os.path.dirname(__file__), 'files') self.out_dir = os.path.join(os.path.dirname(__file__), 'out') @@ -33,4 +39,15 @@ def test_69030(self): for out in outs: self.assertTrue(os.path.exists(os.path.join(self.out_dir, out % book_id))) os.remove(os.path.join(self.out_dir, out % book_id)) - \ No newline at end of file + + def test_parser(self): + load_parsers() + options.outputdir = '' + book_id = '69030' + dir = os.path.join(self.sample_dir, book_id) + srcfile = os.path.join(dir, '%s-0.txt' % book_id) + parser = ParserFactory.create(srcfile) + parser.parse() + self.assertTrue(len(parser.unicode_content()) > len(parser.text)) + self.assertTrue(len(parser.pg_header) > 500) + self.assertTrue(len(parser.pg_footer) > 1500)