8000 Fix unpaired close tags and self-closing tags by kojiishi · Pull Request #360 · google/budoux · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

Fix unpaired close tags and self-closing tags #360

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Nov 11, 2023
Merged

Fix unpaired close tags and self-closing tags #360

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0df2fb2
Fix unpaired close tags and self-closing tags
kojiishi Nov 10, 2023
448a267
yapf
kojiishi Nov 10, 2023
a563f85
Update message in `test_img_in_nobr`
kojiishi Nov 11, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10000
29 changes: 26 additions & 3 deletions budoux/html_processor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,19 @@
SKIP_NODES: typing.Set[str] = set(json.load(f))


class ElementState(object):
"""Represents the state for an element.

Attributes:
tag (str): The tag name.
to_skip (bool): Whether the content should be skipped or not.
"""

def __init__(self, tag: str, to_skip: bool) -> None:
self.tag = tag
self.to_skip = to_skip


class TextContentExtractor(HTMLParser):
"""An HTML parser to extract text content.

Expand Down Expand Up @@ -61,7 +74,7 @@ def __init__(self, chunks: typing.List[str], separator: str):
self.separator = separator
self.to_skip = False
self.scan_index = 0
self.element_stack: queue.LifoQueue[bool] = queue.LifoQueue()
self.element_stack: queue.LifoQueue[ElementState] = queue.LifoQueue()

def handle_starttag(self, tag: str, attrs: HTMLAttr) -> None:
attr_pairs = []
Expand All @@ -71,7 +84,7 @@ def handle_starttag(self, tag: str, attrs: HTMLAttr) -> None:
else:
attr_pairs.append(' %s="%s"' % (attr[0], attr[1]))
encoded_attrs = ''.join(attr_pairs)
self.element_stack.put(self.to_skip)
self.element_stack.put(ElementState(tag, self.to_skip))
if tag.upper() in SKIP_NODES:
if not self.to_skip and self.chunks_joined[self.scan_index] == SEP:
self.scan_index += 1
Expand All @@ -81,7 +94,17 @@ def handle_starttag(self, tag: str, attrs: HTMLAttr) -> None:

def handle_endtag(self, tag: str) -> None:
self.output += '</%s>' % (tag)
self.to_skip = self.element_stack.get_nowait()
while not self.element_stack.empty():
state = self.element_stack.get_nowait()
if state.tag == tag:
self.to_skip = state.to_skip
break
# If the close tag doesn't match the open tag, remove it and keep looking.
# This means that close tags close their corresponding open tags.
# e.g., `<span>abc<img>def</span>` or `<p>abc<span>def</p>` are both valid
# HTML as per the HTML spec.
# Note the HTML "adoption agency algorithm" isn't fully supported.
# See https://html.spec.whatwg.org/multipage/parsing.html#an-introduction-to-error-handling-and-strange-cases-in-the-parser

def handle_data(self, data: str) -> None:
for char in data:
Expand Down
32 changes: 32 additions & 0 deletions tests/test_html_processor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,38 @@ def test_output(self) -> None:
self.assertEqual(resolver.output, expected,
'WBR tags should be inserted as specified by chunks.')

def test_unpaired(self) -> None:
input = '<p>abcdef</p></p>'
expected = '<p>abc<wbr>def</p></p>'
resolver = html_processor.HTMLChunkResolver(['abc', 'def'], '<wbr>')
resolver.feed(input)
self.assertEqual(resolver.output, expected,
'Unpaired close tag should not cause errors.')

def test_nobr(self) -> None:
input = '<p>ab<nobr>cde</nobr>f</p>'
expected = '<p>ab<nobr>cde</nobr>f</p>'
resolver = html_processor.HTMLChunkResolver(['abc', 'def'], '<wbr>')
resolver.feed(input)
self.assertEqual(resolver.output, expected,
'WBR tags should not be inserted if in NOBR.')

def test_after_nobr(self) -> None:
input = '<p>ab<nobr>xy</nobr>abcdef</p>'
expected = '<p>ab<nobr>xy</nobr>abc<wbr>def</p>'
resolver = html_processor.HTMLChunkResolver(['abxyabc', 'def'], '<wbr>')
resolver.feed(input)
self.assertEqual(resolver.output, expected,
'WBR tags should be inserted if after NOBR.')

def test_img_in_nobr(self) -> None:
input = '<p>ab<nobr>x<img>y</nobr>abcdef</p>'
expected = '<p>ab<nobr>x<img>y</nobr>abc<wbr>def</p>'
resolver = html_processor.HTMLChunkResolver(['abxyabc', 'def'], '<wbr>')
resolver.feed(input)
self.assertEqual(resolver.output, expected,
'IMG should not affect surrounding NOBR.')


class TestResolve(unittest.TestCase):

Expand Down
0