Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions legistar/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -232,9 +232,9 @@ def _parse_detail(self, key, field_1, field_2):
return None

def _stringify(self, field):
for br in field.xpath("*//br"):
for br in field.xpath(".//br"):
br.tail = "\n" + br.tail if br.tail else "\n"
for em in field.xpath("*//em"):
for em in field.xpath(".//em"):
if em.text:
em.text = "--em--" + em.text + "--em--"
return field.text_content().replace(' ', ' ').strip()
Expand Down
11 changes: 11 additions & 0 deletions tests/test_parsing.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,24 @@
import os

import lxml
import lxml.html
import pytest

from legistar.base import LegistarScraper
from legistar.bills import LegistarBillScraper
from legistar.events import LegistarEventsScraper
from legistar.people import LegistarPersonScraper


@pytest.mark.parametrize('markup, expected', [
('<td>Line1<br/>Line2</td>', 'Line1\nLine2'), # <br> is a direct child of the cell
('<td><font>Line1<br/>Line2</font></td>', 'Line1\nLine2'), # <br> nested in a child element
])
def test_stringify_br_becomes_newline(markup, expected):
scraper = LegistarScraper()
assert scraper._stringify(lxml.html.fromstring(markup)) == expected


@pytest.mark.parametrize('jurisdiction', ['chicago', 'metro', 'nyc'])
def test_parse_bills(project_directory, jurisdiction):
bills_fixture = os.path.join(project_directory, 'tests', 'fixtures', jurisdiction, 'bills.html')
Expand Down