You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

810 lines
32 KiB

4 years ago
  1. # encoding: utf-8
  2. """Helper classes for tests."""
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. __license__ = "MIT"
  6. import pickle
  7. import copy
  8. import functools
  9. import unittest
  10. from unittest import TestCase
  11. from bs4 import BeautifulSoup
  12. from bs4.element import (
  13. CharsetMetaAttributeValue,
  14. Comment,
  15. ContentMetaAttributeValue,
  16. Doctype,
  17. SoupStrainer,
  18. )
  19. from bs4.builder import HTMLParserTreeBuilder
  20. default_builder = HTMLParserTreeBuilder
  21. class SoupTest(unittest.TestCase):
  22. @property
  23. def default_builder(self):
  24. return default_builder()
  25. def soup(self, markup, **kwargs):
  26. """Build a Beautiful Soup object from markup."""
  27. builder = kwargs.pop('builder', self.default_builder)
  28. return BeautifulSoup(markup, builder=builder, **kwargs)
  29. def document_for(self, markup):
  30. """Turn an HTML fragment into a document.
  31. The details depend on the builder.
  32. """
  33. return self.default_builder.test_fragment_to_document(markup)
  34. def assertSoupEquals(self, to_parse, compare_parsed_to=None):
  35. builder = self.default_builder
  36. obj = BeautifulSoup(to_parse, builder=builder)
  37. if compare_parsed_to is None:
  38. compare_parsed_to = to_parse
  39. self.assertEqual(obj.decode(), self.document_for(compare_parsed_to))
  40. def assertConnectedness(self, element):
  41. """Ensure that next_element and previous_element are properly
  42. set for all descendants of the given element.
  43. """
  44. earlier = None
  45. for e in element.descendants:
  46. if earlier:
  47. self.assertEqual(e, earlier.next_element)
  48. self.assertEqual(earlier, e.previous_element)
  49. earlier = e
  50. class HTMLTreeBuilderSmokeTest(object):
  51. """A basic test of a treebuilder's competence.
  52. Any HTML treebuilder, present or future, should be able to pass
  53. these tests. With invalid markup, there's room for interpretation,
  54. and different parsers can handle it differently. But with the
  55. markup in these tests, there's not much room for interpretation.
  56. """
  57. def test_empty_element_tags(self):
  58. """Verify that all HTML4 and HTML5 empty element (aka void element) tags
  59. are handled correctly.
  60. """
  61. for name in [
  62. 'area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', 'keygen', 'link', 'menuitem', 'meta', 'param', 'source', 'track', 'wbr',
  63. 'spacer', 'frame'
  64. ]:
  65. soup = self.soup("")
  66. new_tag = soup.new_tag(name)
  67. self.assertEqual(True, new_tag.is_empty_element)
  68. def test_pickle_and_unpickle_identity(self):
  69. # Pickling a tree, then unpickling it, yields a tree identical
  70. # to the original.
  71. tree = self.soup("<a><b>foo</a>")
  72. dumped = pickle.dumps(tree, 2)
  73. loaded = pickle.loads(dumped)
  74. self.assertEqual(loaded.__class__, BeautifulSoup)
  75. self.assertEqual(loaded.decode(), tree.decode())
  76. def assertDoctypeHandled(self, doctype_fragment):
  77. """Assert that a given doctype string is handled correctly."""
  78. doctype_str, soup = self._document_with_doctype(doctype_fragment)
  79. # Make sure a Doctype object was created.
  80. doctype = soup.contents[0]
  81. self.assertEqual(doctype.__class__, Doctype)
  82. self.assertEqual(doctype, doctype_fragment)
  83. self.assertEqual(str(soup)[:len(doctype_str)], doctype_str)
  84. # Make sure that the doctype was correctly associated with the
  85. # parse tree and that the rest of the document parsed.
  86. self.assertEqual(soup.p.contents[0], 'foo')
  87. def _document_with_doctype(self, doctype_fragment):
  88. """Generate and parse a document with the given doctype."""
  89. doctype = '<!DOCTYPE %s>' % doctype_fragment
  90. markup = doctype + '\n<p>foo</p>'
  91. soup = self.soup(markup)
  92. return doctype, soup
  93. def test_normal_doctypes(self):
  94. """Make sure normal, everyday HTML doctypes are handled correctly."""
  95. self.assertDoctypeHandled("html")
  96. self.assertDoctypeHandled(
  97. 'html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"')
  98. def test_empty_doctype(self):
  99. soup = self.soup("<!DOCTYPE>")
  100. doctype = soup.contents[0]
  101. self.assertEqual("", doctype.strip())
  102. def test_public_doctype_with_url(self):
  103. doctype = 'html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"'
  104. self.assertDoctypeHandled(doctype)
  105. def test_system_doctype(self):
  106. self.assertDoctypeHandled('foo SYSTEM "http://www.example.com/"')
  107. def test_namespaced_system_doctype(self):
  108. # We can handle a namespaced doctype with a system ID.
  109. self.assertDoctypeHandled('xsl:stylesheet SYSTEM "htmlent.dtd"')
  110. def test_namespaced_public_doctype(self):
  111. # Test a namespaced doctype with a public id.
  112. self.assertDoctypeHandled('xsl:stylesheet PUBLIC "htmlent.dtd"')
  113. def test_real_xhtml_document(self):
  114. """A real XHTML document should come out more or less the same as it went in."""
  115. markup = b"""<?xml version="1.0" encoding="utf-8"?>
  116. <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN">
  117. <html xmlns="http://www.w3.org/1999/xhtml">
  118. <head><title>Hello.</title></head>
  119. <body>Goodbye.</body>
  120. </html>"""
  121. soup = self.soup(markup)
  122. self.assertEqual(
  123. soup.encode("utf-8").replace(b"\n", b""),
  124. markup.replace(b"\n", b""))
  125. def test_namespaced_html(self):
  126. """When a namespaced XML document is parsed as HTML it should
  127. be treated as HTML with weird tag names.
  128. """
  129. markup = b"""<ns1:foo>content</ns1:foo><ns1:foo/><ns2:foo/>"""
  130. soup = self.soup(markup)
  131. self.assertEqual(2, len(soup.find_all("ns1:foo")))
  132. def test_processing_instruction(self):
  133. # We test both Unicode and bytestring to verify that
  134. # process_markup correctly sets processing_instruction_class
  135. # even when the markup is already Unicode and there is no
  136. # need to process anything.
  137. markup = """<?PITarget PIContent?>"""
  138. soup = self.soup(markup)
  139. self.assertEqual(markup, soup.decode())
  140. markup = b"""<?PITarget PIContent?>"""
  141. soup = self.soup(markup)
  142. self.assertEqual(markup, soup.encode("utf8"))
  143. def test_deepcopy(self):
  144. """Make sure you can copy the tree builder.
  145. This is important because the builder is part of a
  146. BeautifulSoup object, and we want to be able to copy that.
  147. """
  148. copy.deepcopy(self.default_builder)
  149. def test_p_tag_is_never_empty_element(self):
  150. """A <p> tag is never designated as an empty-element tag.
  151. Even if the markup shows it as an empty-element tag, it
  152. shouldn't be presented that way.
  153. """
  154. soup = self.soup("<p/>")
  155. self.assertFalse(soup.p.is_empty_element)
  156. self.assertEqual(str(soup.p), "<p></p>")
  157. def test_unclosed_tags_get_closed(self):
  158. """A tag that's not closed by the end of the document should be closed.
  159. This applies to all tags except empty-element tags.
  160. """
  161. self.assertSoupEquals("<p>", "<p></p>")
  162. self.assertSoupEquals("<b>", "<b></b>")
  163. self.assertSoupEquals("<br>", "<br/>")
  164. def test_br_is_always_empty_element_tag(self):
  165. """A <br> tag is designated as an empty-element tag.
  166. Some parsers treat <br></br> as one <br/> tag, some parsers as
  167. two tags, but it should always be an empty-element tag.
  168. """
  169. soup = self.soup("<br></br>")
  170. self.assertTrue(soup.br.is_empty_element)
  171. self.assertEqual(str(soup.br), "<br/>")
  172. def test_nested_formatting_elements(self):
  173. self.assertSoupEquals("<em><em></em></em>")
  174. def test_double_head(self):
  175. html = '''<!DOCTYPE html>
  176. <html>
  177. <head>
  178. <title>Ordinary HEAD element test</title>
  179. </head>
  180. <script type="text/javascript">
  181. alert("Help!");
  182. </script>
  183. <body>
  184. Hello, world!
  185. </body>
  186. </html>
  187. '''
  188. soup = self.soup(html)
  189. self.assertEqual("text/javascript", soup.find('script')['type'])
  190. def test_comment(self):
  191. # Comments are represented as Comment objects.
  192. markup = "<p>foo<!--foobar-->baz</p>"
  193. self.assertSoupEquals(markup)
  194. soup = self.soup(markup)
  195. comment = soup.find(text="foobar")
  196. self.assertEqual(comment.__class__, Comment)
  197. # The comment is properly integrated into the tree.
  198. foo = soup.find(text="foo")
  199. self.assertEqual(comment, foo.next_element)
  200. baz = soup.find(text="baz")
  201. self.assertEqual(comment, baz.previous_element)
  202. def test_preserved_whitespace_in_pre_and_textarea(self):
  203. """Whitespace must be preserved in <pre> and <textarea> tags,
  204. even if that would mean not prettifying the markup.
  205. """
  206. pre_markup = "<pre> </pre>"
  207. textarea_markup = "<textarea> woo\nwoo </textarea>"
  208. self.assertSoupEquals(pre_markup)
  209. self.assertSoupEquals(textarea_markup)
  210. soup = self.soup(pre_markup)
  211. self.assertEqual(soup.pre.prettify(), pre_markup)
  212. soup = self.soup(textarea_markup)
  213. self.assertEqual(soup.textarea.prettify(), textarea_markup)
  214. soup = self.soup("<textarea></textarea>")
  215. self.assertEqual(soup.textarea.prettify(), "<textarea></textarea>")
  216. def test_nested_inline_elements(self):
  217. """Inline elements can be nested indefinitely."""
  218. b_tag = "<b>Inside a B tag</b>"
  219. self.assertSoupEquals(b_tag)
  220. nested_b_tag = "<p>A <i>nested <b>tag</b></i></p>"
  221. self.assertSoupEquals(nested_b_tag)
  222. double_nested_b_tag = "<p>A <a>doubly <i>nested <b>tag</b></i></a></p>"
  223. self.assertSoupEquals(nested_b_tag)
  224. def test_nested_block_level_elements(self):
  225. """Block elements can be nested."""
  226. soup = self.soup('<blockquote><p><b>Foo</b></p></blockquote>')
  227. blockquote = soup.blockquote
  228. self.assertEqual(blockquote.p.b.string, 'Foo')
  229. self.assertEqual(blockquote.b.string, 'Foo')
  230. def test_correctly_nested_tables(self):
  231. """One table can go inside another one."""
  232. markup = ('<table id="1">'
  233. '<tr>'
  234. "<td>Here's another table:"
  235. '<table id="2">'
  236. '<tr><td>foo</td></tr>'
  237. '</table></td>')
  238. self.assertSoupEquals(
  239. markup,
  240. '<table id="1"><tr><td>Here\'s another table:'
  241. '<table id="2"><tr><td>foo</td></tr></table>'
  242. '</td></tr></table>')
  243. self.assertSoupEquals(
  244. "<table><thead><tr><td>Foo</td></tr></thead>"
  245. "<tbody><tr><td>Bar</td></tr></tbody>"
  246. "<tfoot><tr><td>Baz</td></tr></tfoot></table>")
  247. def test_deeply_nested_multivalued_attribute(self):
  248. # html5lib can set the attributes of the same tag many times
  249. # as it rearranges the tree. This has caused problems with
  250. # multivalued attributes.
  251. markup = '<table><div><div class="css"></div></div></table>'
  252. soup = self.soup(markup)
  253. self.assertEqual(["css"], soup.div.div['class'])
  254. def test_multivalued_attribute_on_html(self):
  255. # html5lib uses a different API to set the attributes ot the
  256. # <html> tag. This has caused problems with multivalued
  257. # attributes.
  258. markup = '<html class="a b"></html>'
  259. soup = self.soup(markup)
  260. self.assertEqual(["a", "b"], soup.html['class'])
  261. def test_angle_brackets_in_attribute_values_are_escaped(self):
  262. self.assertSoupEquals('<a b="<a>"></a>', '<a b="&lt;a&gt;"></a>')
  263. def test_strings_resembling_character_entity_references(self):
  264. # "&T" and "&p" look like incomplete character entities, but they are
  265. # not.
  266. self.assertSoupEquals(
  267. "<p>&bull; AT&T is in the s&p 500</p>",
  268. "<p>\u2022 AT&amp;T is in the s&amp;p 500</p>"
  269. )
  270. def test_entities_in_foreign_document_encoding(self):
  271. # &#147; and &#148; are invalid numeric entities referencing
  272. # Windows-1252 characters. &#45; references a character common
  273. # to Windows-1252 and Unicode, and &#9731; references a
  274. # character only found in Unicode.
  275. #
  276. # All of these entities should be converted to Unicode
  277. # characters.
  278. markup = "<p>&#147;Hello&#148; &#45;&#9731;</p>"
  279. soup = self.soup(markup)
  280. self.assertEqual("“Hello” -☃", soup.p.string)
  281. def test_entities_in_attributes_converted_to_unicode(self):
  282. expect = '<p id="pi\N{LATIN SMALL LETTER N WITH TILDE}ata"></p>'
  283. self.assertSoupEquals('<p id="pi&#241;ata"></p>', expect)
  284. self.assertSoupEquals('<p id="pi&#xf1;ata"></p>', expect)
  285. self.assertSoupEquals('<p id="pi&#Xf1;ata"></p>', expect)
  286. self.assertSoupEquals('<p id="pi&ntilde;ata"></p>', expect)
  287. def test_entities_in_text_converted_to_unicode(self):
  288. expect = '<p>pi\N{LATIN SMALL LETTER N WITH TILDE}ata</p>'
  289. self.assertSoupEquals("<p>pi&#241;ata</p>", expect)
  290. self.assertSoupEquals("<p>pi&#xf1;ata</p>", expect)
  291. self.assertSoupEquals("<p>pi&#Xf1;ata</p>", expect)
  292. self.assertSoupEquals("<p>pi&ntilde;ata</p>", expect)
  293. def test_quot_entity_converted_to_quotation_mark(self):
  294. self.assertSoupEquals("<p>I said &quot;good day!&quot;</p>",
  295. '<p>I said "good day!"</p>')
  296. def test_out_of_range_entity(self):
  297. expect = "\N{REPLACEMENT CHARACTER}"
  298. self.assertSoupEquals("&#10000000000000;", expect)
  299. self.assertSoupEquals("&#x10000000000000;", expect)
  300. self.assertSoupEquals("&#1000000000;", expect)
  301. def test_multipart_strings(self):
  302. "Mostly to prevent a recurrence of a bug in the html5lib treebuilder."
  303. soup = self.soup("<html><h2>\nfoo</h2><p></p></html>")
  304. self.assertEqual("p", soup.h2.string.next_element.name)
  305. self.assertEqual("p", soup.p.name)
  306. self.assertConnectedness(soup)
  307. def test_empty_element_tags(self):
  308. """Verify consistent handling of empty-element tags,
  309. no matter how they come in through the markup.
  310. """
  311. self.assertSoupEquals('<br/><br/><br/>', "<br/><br/><br/>")
  312. self.assertSoupEquals('<br /><br /><br />', "<br/><br/><br/>")
  313. def test_head_tag_between_head_and_body(self):
  314. "Prevent recurrence of a bug in the html5lib treebuilder."
  315. content = """<html><head></head>
  316. <link></link>
  317. <body>foo</body>
  318. </html>
  319. """
  320. soup = self.soup(content)
  321. self.assertNotEqual(None, soup.html.body)
  322. self.assertConnectedness(soup)
  323. def test_multiple_copies_of_a_tag(self):
  324. "Prevent recurrence of a bug in the html5lib treebuilder."
  325. content = """<!DOCTYPE html>
  326. <html>
  327. <body>
  328. <article id="a" >
  329. <div><a href="1"></div>
  330. <footer>
  331. <a href="2"></a>
  332. </footer>
  333. </article>
  334. </body>
  335. </html>
  336. """
  337. soup = self.soup(content)
  338. self.assertConnectedness(soup.article)
  339. def test_basic_namespaces(self):
  340. """Parsers don't need to *understand* namespaces, but at the
  341. very least they should not choke on namespaces or lose
  342. data."""
  343. markup = b'<html xmlns="http://www.w3.org/1999/xhtml" xmlns:mathml="http://www.w3.org/1998/Math/MathML" xmlns:svg="http://www.w3.org/2000/svg"><head></head><body><mathml:msqrt>4</mathml:msqrt><b svg:fill="red"></b></body></html>'
  344. soup = self.soup(markup)
  345. self.assertEqual(markup, soup.encode())
  346. html = soup.html
  347. self.assertEqual('http://www.w3.org/1999/xhtml', soup.html['xmlns'])
  348. self.assertEqual(
  349. 'http://www.w3.org/1998/Math/MathML', soup.html['xmlns:mathml'])
  350. self.assertEqual(
  351. 'http://www.w3.org/2000/svg', soup.html['xmlns:svg'])
  352. def test_multivalued_attribute_value_becomes_list(self):
  353. markup = b'<a class="foo bar">'
  354. soup = self.soup(markup)
  355. self.assertEqual(['foo', 'bar'], soup.a['class'])
  356. #
  357. # Generally speaking, tests below this point are more tests of
  358. # Beautiful Soup than tests of the tree builders. But parsers are
  359. # weird, so we run these tests separately for every tree builder
  360. # to detect any differences between them.
  361. #
  362. def test_can_parse_unicode_document(self):
  363. # A seemingly innocuous document... but it's in Unicode! And
  364. # it contains characters that can't be represented in the
  365. # encoding found in the declaration! The horror!
  366. markup = '<html><head><meta encoding="euc-jp"></head><body>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</body>'
  367. soup = self.soup(markup)
  368. self.assertEqual('Sacr\xe9 bleu!', soup.body.string)
  369. def test_soupstrainer(self):
  370. """Parsers should be able to work with SoupStrainers."""
  371. strainer = SoupStrainer("b")
  372. soup = self.soup("A <b>bold</b> <meta/> <i>statement</i>",
  373. parse_only=strainer)
  374. self.assertEqual(soup.decode(), "<b>bold</b>")
  375. def test_single_quote_attribute_values_become_double_quotes(self):
  376. self.assertSoupEquals("<foo attr='bar'></foo>",
  377. '<foo attr="bar"></foo>')
  378. def test_attribute_values_with_nested_quotes_are_left_alone(self):
  379. text = """<foo attr='bar "brawls" happen'>a</foo>"""
  380. self.assertSoupEquals(text)
  381. def test_attribute_values_with_double_nested_quotes_get_quoted(self):
  382. text = """<foo attr='bar "brawls" happen'>a</foo>"""
  383. soup = self.soup(text)
  384. soup.foo['attr'] = 'Brawls happen at "Bob\'s Bar"'
  385. self.assertSoupEquals(
  386. soup.foo.decode(),
  387. """<foo attr="Brawls happen at &quot;Bob\'s Bar&quot;">a</foo>""")
  388. def test_ampersand_in_attribute_value_gets_escaped(self):
  389. self.assertSoupEquals('<this is="really messed up & stuff"></this>',
  390. '<this is="really messed up &amp; stuff"></this>')
  391. self.assertSoupEquals(
  392. '<a href="http://example.org?a=1&b=2;3">foo</a>',
  393. '<a href="http://example.org?a=1&amp;b=2;3">foo</a>')
  394. def test_escaped_ampersand_in_attribute_value_is_left_alone(self):
  395. self.assertSoupEquals('<a href="http://example.org?a=1&amp;b=2;3"></a>')
  396. def test_entities_in_strings_converted_during_parsing(self):
  397. # Both XML and HTML entities are converted to Unicode characters
  398. # during parsing.
  399. text = "<p>&lt;&lt;sacr&eacute;&#32;bleu!&gt;&gt;</p>"
  400. expected = "<p>&lt;&lt;sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!&gt;&gt;</p>"
  401. self.assertSoupEquals(text, expected)
  402. def test_smart_quotes_converted_on_the_way_in(self):
  403. # Microsoft smart quotes are converted to Unicode characters during
  404. # parsing.
  405. quote = b"<p>\x91Foo\x92</p>"
  406. soup = self.soup(quote)
  407. self.assertEqual(
  408. soup.p.string,
  409. "\N{LEFT SINGLE QUOTATION MARK}Foo\N{RIGHT SINGLE QUOTATION MARK}")
  410. def test_non_breaking_spaces_converted_on_the_way_in(self):
  411. soup = self.soup("<a>&nbsp;&nbsp;</a>")
  412. self.assertEqual(soup.a.string, "\N{NO-BREAK SPACE}" * 2)
  413. def test_entities_converted_on_the_way_out(self):
  414. text = "<p>&lt;&lt;sacr&eacute;&#32;bleu!&gt;&gt;</p>"
  415. expected = "<p>&lt;&lt;sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!&gt;&gt;</p>".encode("utf-8")
  416. soup = self.soup(text)
  417. self.assertEqual(soup.p.encode("utf-8"), expected)
  418. def test_real_iso_latin_document(self):
  419. # Smoke test of interrelated functionality, using an
  420. # easy-to-understand document.
  421. # Here it is in Unicode. Note that it claims to be in ISO-Latin-1.
  422. unicode_html = '<html><head><meta content="text/html; charset=ISO-Latin-1" http-equiv="Content-type"/></head><body><p>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</p></body></html>'
  423. # That's because we're going to encode it into ISO-Latin-1, and use
  424. # that to test.
  425. iso_latin_html = unicode_html.encode("iso-8859-1")
  426. # Parse the ISO-Latin-1 HTML.
  427. soup = self.soup(iso_latin_html)
  428. # Encode it to UTF-8.
  429. result = soup.encode("utf-8")
  430. # What do we expect the result to look like? Well, it would
  431. # look like unicode_html, except that the META tag would say
  432. # UTF-8 instead of ISO-Latin-1.
  433. expected = unicode_html.replace("ISO-Latin-1", "utf-8")
  434. # And, of course, it would be in UTF-8, not Unicode.
  435. expected = expected.encode("utf-8")
  436. # Ta-da!
  437. self.assertEqual(result, expected)
  438. def test_real_shift_jis_document(self):
  439. # Smoke test to make sure the parser can handle a document in
  440. # Shift-JIS encoding, without choking.
  441. shift_jis_html = (
  442. b'<html><head></head><body><pre>'
  443. b'\x82\xb1\x82\xea\x82\xcdShift-JIS\x82\xc5\x83R\x81[\x83f'
  444. b'\x83B\x83\x93\x83O\x82\xb3\x82\xea\x82\xbd\x93\xfa\x96{\x8c'
  445. b'\xea\x82\xcc\x83t\x83@\x83C\x83\x8b\x82\xc5\x82\xb7\x81B'
  446. b'</pre></body></html>')
  447. unicode_html = shift_jis_html.decode("shift-jis")
  448. soup = self.soup(unicode_html)
  449. # Make sure the parse tree is correctly encoded to various
  450. # encodings.
  451. self.assertEqual(soup.encode("utf-8"), unicode_html.encode("utf-8"))
  452. self.assertEqual(soup.encode("euc_jp"), unicode_html.encode("euc_jp"))
  453. def test_real_hebrew_document(self):
  454. # A real-world test to make sure we can convert ISO-8859-9 (a
  455. # Hebrew encoding) to UTF-8.
  456. hebrew_document = b'<html><head><title>Hebrew (ISO 8859-8) in Visual Directionality</title></head><body><h1>Hebrew (ISO 8859-8) in Visual Directionality</h1>\xed\xe5\xec\xf9</body></html>'
  457. soup = self.soup(
  458. hebrew_document, from_encoding="iso8859-8")
  459. # Some tree builders call it iso8859-8, others call it iso-8859-9.
  460. # That's not a difference we really care about.
  461. assert soup.original_encoding in ('iso8859-8', 'iso-8859-8')
  462. self.assertEqual(
  463. soup.encode('utf-8'),
  464. hebrew_document.decode("iso8859-8").encode("utf-8"))
  465. def test_meta_tag_reflects_current_encoding(self):
  466. # Here's the <meta> tag saying that a document is
  467. # encoded in Shift-JIS.
  468. meta_tag = ('<meta content="text/html; charset=x-sjis" '
  469. 'http-equiv="Content-type"/>')
  470. # Here's a document incorporating that meta tag.
  471. shift_jis_html = (
  472. '<html><head>\n%s\n'
  473. '<meta http-equiv="Content-language" content="ja"/>'
  474. '</head><body>Shift-JIS markup goes here.') % meta_tag
  475. soup = self.soup(shift_jis_html)
  476. # Parse the document, and the charset is seemingly unaffected.
  477. parsed_meta = soup.find('meta', {'http-equiv': 'Content-type'})
  478. content = parsed_meta['content']
  479. self.assertEqual('text/html; charset=x-sjis', content)
  480. # But that value is actually a ContentMetaAttributeValue object.
  481. self.assertTrue(isinstance(content, ContentMetaAttributeValue))
  482. # And it will take on a value that reflects its current
  483. # encoding.
  484. self.assertEqual('text/html; charset=utf8', content.encode("utf8"))
  485. # For the rest of the story, see TestSubstitutions in
  486. # test_tree.py.
  487. def test_html5_style_meta_tag_reflects_current_encoding(self):
  488. # Here's the <meta> tag saying that a document is
  489. # encoded in Shift-JIS.
  490. meta_tag = ('<meta id="encoding" charset="x-sjis" />')
  491. # Here's a document incorporating that meta tag.
  492. shift_jis_html = (
  493. '<html><head>\n%s\n'
  494. '<meta http-equiv="Content-language" content="ja"/>'
  495. '</head><body>Shift-JIS markup goes here.') % meta_tag
  496. soup = self.soup(shift_jis_html)
  497. # Parse the document, and the charset is seemingly unaffected.
  498. parsed_meta = soup.find('meta', id="encoding")
  499. charset = parsed_meta['charset']
  500. self.assertEqual('x-sjis', charset)
  501. # But that value is actually a CharsetMetaAttributeValue object.
  502. self.assertTrue(isinstance(charset, CharsetMetaAttributeValue))
  503. # And it will take on a value that reflects its current
  504. # encoding.
  505. self.assertEqual('utf8', charset.encode("utf8"))
  506. def test_tag_with_no_attributes_can_have_attributes_added(self):
  507. data = self.soup("<a>text</a>")
  508. data.a['foo'] = 'bar'
  509. self.assertEqual('<a foo="bar">text</a>', data.a.decode())
  510. class XMLTreeBuilderSmokeTest(object):
  511. def test_pickle_and_unpickle_identity(self):
  512. # Pickling a tree, then unpickling it, yields a tree identical
  513. # to the original.
  514. tree = self.soup("<a><b>foo</a>")
  515. dumped = pickle.dumps(tree, 2)
  516. loaded = pickle.loads(dumped)
  517. self.assertEqual(loaded.__class__, BeautifulSoup)
  518. self.assertEqual(loaded.decode(), tree.decode())
  519. def test_docstring_generated(self):
  520. soup = self.soup("<root/>")
  521. self.assertEqual(
  522. soup.encode(), b'<?xml version="1.0" encoding="utf-8"?>\n<root/>')
  523. def test_xml_declaration(self):
  524. markup = b"""<?xml version="1.0" encoding="utf8"?>\n<foo/>"""
  525. soup = self.soup(markup)
  526. self.assertEqual(markup, soup.encode("utf8"))
  527. def test_processing_instruction(self):
  528. markup = b"""<?xml version="1.0" encoding="utf8"?>\n<?PITarget PIContent?>"""
  529. soup = self.soup(markup)
  530. self.assertEqual(markup, soup.encode("utf8"))
  531. def test_real_xhtml_document(self):
  532. """A real XHTML document should come out *exactly* the same as it went in."""
  533. markup = b"""<?xml version="1.0" encoding="utf-8"?>
  534. <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN">
  535. <html xmlns="http://www.w3.org/1999/xhtml">
  536. <head><title>Hello.</title></head>
  537. <body>Goodbye.</body>
  538. </html>"""
  539. soup = self.soup(markup)
  540. self.assertEqual(
  541. soup.encode("utf-8"), markup)
  542. def test_nested_namespaces(self):
  543. doc = b"""<?xml version="1.0" encoding="utf-8"?>
  544. <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
  545. <parent xmlns="http://ns1/">
  546. <child xmlns="http://ns2/" xmlns:ns3="http://ns3/">
  547. <grandchild ns3:attr="value" xmlns="http://ns4/"/>
  548. </child>
  549. </parent>"""
  550. soup = self.soup(doc)
  551. self.assertEqual(doc, soup.encode())
  552. def test_formatter_processes_script_tag_for_xml_documents(self):
  553. doc = """
  554. <script type="text/javascript">
  555. </script>
  556. """
  557. soup = BeautifulSoup(doc, "lxml-xml")
  558. # lxml would have stripped this while parsing, but we can add
  559. # it later.
  560. soup.script.string = 'console.log("< < hey > > ");'
  561. encoded = soup.encode()
  562. self.assertTrue(b"&lt; &lt; hey &gt; &gt;" in encoded)
  563. def test_can_parse_unicode_document(self):
  564. markup = '<?xml version="1.0" encoding="euc-jp"><root>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</root>'
  565. soup = self.soup(markup)
  566. self.assertEqual('Sacr\xe9 bleu!', soup.root.string)
  567. def test_popping_namespaced_tag(self):
  568. markup = '<rss xmlns:dc="foo"><dc:creator>b</dc:creator><dc:date>2012-07-02T20:33:42Z</dc:date><dc:rights>c</dc:rights><image>d</image></rss>'
  569. soup = self.soup(markup)
  570. self.assertEqual(
  571. str(soup.rss), markup)
  572. def test_docstring_includes_correct_encoding(self):
  573. soup = self.soup("<root/>")
  574. self.assertEqual(
  575. soup.encode("latin1"),
  576. b'<?xml version="1.0" encoding="latin1"?>\n<root/>')
  577. def test_large_xml_document(self):
  578. """A large XML document should come out the same as it went in."""
  579. markup = (b'<?xml version="1.0" encoding="utf-8"?>\n<root>'
  580. + b'0' * (2**12)
  581. + b'</root>')
  582. soup = self.soup(markup)
  583. self.assertEqual(soup.encode("utf-8"), markup)
  584. def test_tags_are_empty_element_if_and_only_if_they_are_empty(self):
  585. self.assertSoupEquals("<p>", "<p/>")
  586. self.assertSoupEquals("<p>foo</p>")
  587. def test_namespaces_are_preserved(self):
  588. markup = '<root xmlns:a="http://example.com/" xmlns:b="http://example.net/"><a:foo>This tag is in the a namespace</a:foo><b:foo>This tag is in the b namespace</b:foo></root>'
  589. soup = self.soup(markup)
  590. root = soup.root
  591. self.assertEqual("http://example.com/", root['xmlns:a'])
  592. self.assertEqual("http://example.net/", root['xmlns:b'])
  593. def test_closing_namespaced_tag(self):
  594. markup = '<p xmlns:dc="http://purl.org/dc/elements/1.1/"><dc:date>20010504</dc:date></p>'
  595. soup = self.soup(markup)
  596. self.assertEqual(str(soup.p), markup)
  597. def test_namespaced_attributes(self):
  598. markup = '<foo xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"><bar xsi:schemaLocation="http://www.example.com"/></foo>'
  599. soup = self.soup(markup)
  600. self.assertEqual(str(soup.foo), markup)
  601. def test_namespaced_attributes_xml_namespace(self):
  602. markup = '<foo xml:lang="fr">bar</foo>'
  603. soup = self.soup(markup)
  604. self.assertEqual(str(soup.foo), markup)
  605. def test_find_by_prefixed_name(self):
  606. doc = """<?xml version="1.0" encoding="utf-8"?>
  607. <Document xmlns="http://example.com/ns0"
  608. xmlns:ns1="http://example.com/ns1"
  609. xmlns:ns2="http://example.com/ns2"
  610. <ns1:tag>foo</ns1:tag>
  611. <ns1:tag>bar</ns1:tag>
  612. <ns2:tag key="value">baz</ns2:tag>
  613. </Document>
  614. """
  615. soup = self.soup(doc)
  616. # There are three <tag> tags.
  617. self.assertEqual(3, len(soup.find_all('tag')))
  618. # But two of them are ns1:tag and one of them is ns2:tag.
  619. self.assertEqual(2, len(soup.find_all('ns1:tag')))
  620. self.assertEqual(1, len(soup.find_all('ns2:tag')))
  621. self.assertEqual(1, len(soup.find_all('ns2:tag', key='value')))
  622. self.assertEqual(3, len(soup.find_all(['ns1:tag', 'ns2:tag'])))
  623. def test_copy_tag_preserves_namespace(self):
  624. xml = """<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
  625. <w:document xmlns:w="http://example.com/ns0"/>"""
  626. soup = self.soup(xml)
  627. tag = soup.document
  628. duplicate = copy.copy(tag)
  629. # The two tags have the same namespace prefix.
  630. self.assertEqual(tag.prefix, duplicate.prefix)
  631. class HTML5TreeBuilderSmokeTest(HTMLTreeBuilderSmokeTest):
  632. """Smoke test for a tree builder that supports HTML5."""
  633. def test_real_xhtml_document(self):
  634. # Since XHTML is not HTML5, HTML5 parsers are not tested to handle
  635. # XHTML documents in any particular way.
  636. pass
  637. def test_html_tags_have_namespace(self):
  638. markup = "<a>"
  639. soup = self.soup(markup)
  640. self.assertEqual("http://www.w3.org/1999/xhtml", soup.a.namespace)
  641. def test_svg_tags_have_namespace(self):
  642. markup = '<svg><circle/></svg>'
  643. soup = self.soup(markup)
  644. namespace = "http://www.w3.org/2000/svg"
  645. self.assertEqual(namespace, soup.svg.namespace)
  646. self.assertEqual(namespace, soup.circle.namespace)
  647. def test_mathml_tags_have_namespace(self):
  648. markup = '<math><msqrt>5</msqrt></math>'
  649. soup = self.soup(markup)
  650. namespace = 'http://www.w3.org/1998/Math/MathML'
  651. self.assertEqual(namespace, soup.math.namespace)
  652. self.assertEqual(namespace, soup.msqrt.namespace)
  653. def test_xml_declaration_becomes_comment(self):
  654. markup = '<?xml version="1.0" encoding="utf-8"?><html></html>'
  655. soup = self.soup(markup)
  656. self.assertTrue(isinstance(soup.contents[0], Comment))
  657. self.assertEqual(soup.contents[0], '?xml version="1.0" encoding="utf-8"?')
  658. self.assertEqual("html", soup.contents[0].next_element.name)
  659. def skipIf(condition, reason):
  660. def nothing(test, *args, **kwargs):
  661. return None
  662. def decorator(test_item):
  663. if condition:
  664. return nothing
  665. else:
  666. return test_item
  667. return decorator