From 1be9532f79fd7744be0945c4ab42d2f5b41e4e73 Mon Sep 17 00:00:00 2001 From: Ritwik Gupta Date: Mon, 24 Nov 2014 16:23:01 -0500 Subject: [PATCH 001/223] Added iframe seamless boolean attribute --- html5lib/constants.py | 1 + 1 file changed, 1 insertion(+) diff --git a/html5lib/constants.py b/html5lib/constants.py index e7089846..659f2b5e 100644 --- a/html5lib/constants.py +++ b/html5lib/constants.py @@ -535,6 +535,7 @@ "input": frozenset(("disabled", "readonly", "required", "autofocus", "checked", "ismap")), "select": frozenset(("disabled", "readonly", "autofocus", "multiple")), "output": frozenset(("disabled", "readonly")), + "iframe": frozenset(("seamless")), } # entitiesWindows1252 has to be _ordered_ and needs to have an index. It From 4dfe3cd9f97ce51c53463d633308f4a3fe6ad9e6 Mon Sep 17 00:00:00 2001 From: Ritwik Gupta Date: Mon, 24 Nov 2014 16:25:04 -0500 Subject: [PATCH 002/223] Update CHANGES.rst --- CHANGES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 1431b3c9..89e48f94 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -6,7 +6,7 @@ Change Log Released on XXX, 2014 -* XXX +* Fix #XXX: added the seamless attribute for iframes. 0.999 From 7fd79e31e083ab75305b3e837ea9aa8c9b4675ff Mon Sep 17 00:00:00 2001 From: Ritwik Gupta Date: Mon, 24 Nov 2014 16:25:28 -0500 Subject: [PATCH 003/223] Update AUTHORS.rst --- AUTHORS.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS.rst b/AUTHORS.rst index 4148a6ed..787c3b94 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -32,3 +32,4 @@ Patches and suggestions - Juan Carlos Garcia Segovia - Mike West - Marc DM +- Ritwik Gupta From 4aa79f113e7486c7ec5d15a6e1777bfe546d3259 Mon Sep 17 00:00:00 2001 From: Geoffrey Sneddon Date: Sat, 17 May 2014 21:48:32 +0100 Subject: [PATCH 004/223] Fix optionaltags filter to not error when filtering nothing. --- html5lib/filters/optionaltags.py | 3 ++- html5lib/tests/test_optionaltags_filter.py | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 html5lib/tests/test_optionaltags_filter.py diff --git a/html5lib/filters/optionaltags.py b/html5lib/filters/optionaltags.py index dab0574a..8f11fff4 100644 --- a/html5lib/filters/optionaltags.py +++ b/html5lib/filters/optionaltags.py @@ -11,7 +11,8 @@ def slider(self): yield previous2, previous1, token previous2 = previous1 previous1 = token - yield previous2, previous1, None + if previous1 is not None: + yield previous2, previous1, None def __iter__(self): for previous, token, next in self.slider(): diff --git a/html5lib/tests/test_optionaltags_filter.py b/html5lib/tests/test_optionaltags_filter.py new file mode 100644 index 00000000..cd282149 --- /dev/null +++ b/html5lib/tests/test_optionaltags_filter.py @@ -0,0 +1,7 @@ +from __future__ import absolute_import, division, unicode_literals + +from html5lib.filters.optionaltags import Filter + + +def test_empty(): + assert list(Filter([])) == [] From a2917e950df0a06b0c46c3ac7f84c3c797af0e43 Mon Sep 17 00:00:00 2001 From: Geoffrey Sneddon Date: Mon, 19 May 2014 15:48:16 +0100 Subject: [PATCH 005/223] Fix #72: rewrite the sanitizer to be a treewalker filter only. As we no longer need the sanitizer to be shared between a filter and a tokenizer, move the entire sanitizer to the filter module. --- CHANGES.rst | 4 + html5lib/filters/sanitizer.py | 864 ++++++++++++++++++++++++++++++- html5lib/sanitizer.py | 300 ----------- html5lib/tests/test_sanitizer.py | 146 +++--- parse.py | 10 +- 5 files changed, 934 insertions(+), 390 deletions(-) delete mode 100644 html5lib/sanitizer.py diff --git a/CHANGES.rst b/CHANGES.rst index 1b557816..51a254c6 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -40,6 +40,10 @@ Released on XXX and the new default), and "spec" (the old False value, and the old default).** +* **Fix #72 by rewriting the sanitizer to apply only to treewalkers + (instead of the tokenizer); as such, this will require amending all + callers of it to use it via the treewalker API.** + 0.9999999/1.0b8 ~~~~~~~~~~~~~~~ diff --git a/html5lib/filters/sanitizer.py b/html5lib/filters/sanitizer.py index b206b54e..caddd318 100644 --- a/html5lib/filters/sanitizer.py +++ b/html5lib/filters/sanitizer.py @@ -1,12 +1,872 @@ from __future__ import absolute_import, division, unicode_literals +import re +from xml.sax.saxutils import escape, unescape + +from six.moves import urllib_parse as urlparse + from . import _base -from ..sanitizer import HTMLSanitizerMixin +from ..constants import namespaces, prefixes + +__all__ = ["Filter"] + + +acceptable_elements = frozenset(( + (namespaces['html'], 'a'), + (namespaces['html'], 'abbr'), + (namespaces['html'], 'acronym'), + (namespaces['html'], 'address'), + (namespaces['html'], 'area'), + (namespaces['html'], 'article'), + (namespaces['html'], 'aside'), + (namespaces['html'], 'audio'), + (namespaces['html'], 'b'), + (namespaces['html'], 'big'), + (namespaces['html'], 'blockquote'), + (namespaces['html'], 'br'), + (namespaces['html'], 'button'), + (namespaces['html'], 'canvas'), + (namespaces['html'], 'caption'), + (namespaces['html'], 'center'), + (namespaces['html'], 'cite'), + (namespaces['html'], 'code'), + (namespaces['html'], 'col'), + (namespaces['html'], 'colgroup'), + (namespaces['html'], 'command'), + (namespaces['html'], 'datagrid'), + (namespaces['html'], 'datalist'), + (namespaces['html'], 'dd'), + (namespaces['html'], 'del'), + (namespaces['html'], 'details'), + (namespaces['html'], 'dfn'), + (namespaces['html'], 'dialog'), + (namespaces['html'], 'dir'), + (namespaces['html'], 'div'), + (namespaces['html'], 'dl'), + (namespaces['html'], 'dt'), + (namespaces['html'], 'em'), + (namespaces['html'], 'event-source'), + (namespaces['html'], 'fieldset'), + (namespaces['html'], 'figcaption'), + (namespaces['html'], 'figure'), + (namespaces['html'], 'footer'), + (namespaces['html'], 'font'), + (namespaces['html'], 'form'), + (namespaces['html'], 'header'), + (namespaces['html'], 'h1'), + (namespaces['html'], 'h2'), + (namespaces['html'], 'h3'), + (namespaces['html'], 'h4'), + (namespaces['html'], 'h5'), + (namespaces['html'], 'h6'), + (namespaces['html'], 'hr'), + (namespaces['html'], 'i'), + (namespaces['html'], 'img'), + (namespaces['html'], 'input'), + (namespaces['html'], 'ins'), + (namespaces['html'], 'keygen'), + (namespaces['html'], 'kbd'), + (namespaces['html'], 'label'), + (namespaces['html'], 'legend'), + (namespaces['html'], 'li'), + (namespaces['html'], 'm'), + (namespaces['html'], 'map'), + (namespaces['html'], 'menu'), + (namespaces['html'], 'meter'), + (namespaces['html'], 'multicol'), + (namespaces['html'], 'nav'), + (namespaces['html'], 'nextid'), + (namespaces['html'], 'ol'), + (namespaces['html'], 'output'), + (namespaces['html'], 'optgroup'), + (namespaces['html'], 'option'), + (namespaces['html'], 'p'), + (namespaces['html'], 'pre'), + (namespaces['html'], 'progress'), + (namespaces['html'], 'q'), + (namespaces['html'], 's'), + (namespaces['html'], 'samp'), + (namespaces['html'], 'section'), + (namespaces['html'], 'select'), + (namespaces['html'], 'small'), + (namespaces['html'], 'sound'), + (namespaces['html'], 'source'), + (namespaces['html'], 'spacer'), + (namespaces['html'], 'span'), + (namespaces['html'], 'strike'), + (namespaces['html'], 'strong'), + (namespaces['html'], 'sub'), + (namespaces['html'], 'sup'), + (namespaces['html'], 'table'), + (namespaces['html'], 'tbody'), + (namespaces['html'], 'td'), + (namespaces['html'], 'textarea'), + (namespaces['html'], 'time'), + (namespaces['html'], 'tfoot'), + (namespaces['html'], 'th'), + (namespaces['html'], 'thead'), + (namespaces['html'], 'tr'), + (namespaces['html'], 'tt'), + (namespaces['html'], 'u'), + (namespaces['html'], 'ul'), + (namespaces['html'], 'var'), + (namespaces['html'], 'video'), + (namespaces['mathml'], 'maction'), + (namespaces['mathml'], 'math'), + (namespaces['mathml'], 'merror'), + (namespaces['mathml'], 'mfrac'), + (namespaces['mathml'], 'mi'), + (namespaces['mathml'], 'mmultiscripts'), + (namespaces['mathml'], 'mn'), + (namespaces['mathml'], 'mo'), + (namespaces['mathml'], 'mover'), + (namespaces['mathml'], 'mpadded'), + (namespaces['mathml'], 'mphantom'), + (namespaces['mathml'], 'mprescripts'), + (namespaces['mathml'], 'mroot'), + (namespaces['mathml'], 'mrow'), + (namespaces['mathml'], 'mspace'), + (namespaces['mathml'], 'msqrt'), + (namespaces['mathml'], 'mstyle'), + (namespaces['mathml'], 'msub'), + (namespaces['mathml'], 'msubsup'), + (namespaces['mathml'], 'msup'), + (namespaces['mathml'], 'mtable'), + (namespaces['mathml'], 'mtd'), + (namespaces['mathml'], 'mtext'), + (namespaces['mathml'], 'mtr'), + (namespaces['mathml'], 'munder'), + (namespaces['mathml'], 'munderover'), + (namespaces['mathml'], 'none'), + (namespaces['svg'], 'a'), + (namespaces['svg'], 'animate'), + (namespaces['svg'], 'animateColor'), + (namespaces['svg'], 'animateMotion'), + (namespaces['svg'], 'animateTransform'), + (namespaces['svg'], 'clipPath'), + (namespaces['svg'], 'circle'), + (namespaces['svg'], 'defs'), + (namespaces['svg'], 'desc'), + (namespaces['svg'], 'ellipse'), + (namespaces['svg'], 'font-face'), + (namespaces['svg'], 'font-face-name'), + (namespaces['svg'], 'font-face-src'), + (namespaces['svg'], 'g'), + (namespaces['svg'], 'glyph'), + (namespaces['svg'], 'hkern'), + (namespaces['svg'], 'linearGradient'), + (namespaces['svg'], 'line'), + (namespaces['svg'], 'marker'), + (namespaces['svg'], 'metadata'), + (namespaces['svg'], 'missing-glyph'), + (namespaces['svg'], 'mpath'), + (namespaces['svg'], 'path'), + (namespaces['svg'], 'polygon'), + (namespaces['svg'], 'polyline'), + (namespaces['svg'], 'radialGradient'), + (namespaces['svg'], 'rect'), + (namespaces['svg'], 'set'), + (namespaces['svg'], 'stop'), + (namespaces['svg'], 'svg'), + (namespaces['svg'], 'switch'), + (namespaces['svg'], 'text'), + (namespaces['svg'], 'title'), + (namespaces['svg'], 'tspan'), + (namespaces['svg'], 'use'), +)) + +acceptable_attributes = frozenset(( + # HTML attributes + (None, 'abbr'), + (None, 'accept'), + (None, 'accept-charset'), + (None, 'accesskey'), + (None, 'action'), + (None, 'align'), + (None, 'alt'), + (None, 'autocomplete'), + (None, 'autofocus'), + (None, 'axis'), + (None, 'background'), + (None, 'balance'), + (None, 'bgcolor'), + (None, 'bgproperties'), + (None, 'border'), + (None, 'bordercolor'), + (None, 'bordercolordark'), + (None, 'bordercolorlight'), + (None, 'bottompadding'), + (None, 'cellpadding'), + (None, 'cellspacing'), + (None, 'ch'), + (None, 'challenge'), + (None, 'char'), + (None, 'charoff'), + (None, 'choff'), + (None, 'charset'), + (None, 'checked'), + (None, 'cite'), + (None, 'class'), + (None, 'clear'), + (None, 'color'), + (None, 'cols'), + (None, 'colspan'), + (None, 'compact'), + (None, 'contenteditable'), + (None, 'controls'), + (None, 'coords'), + (None, 'data'), + (None, 'datafld'), + (None, 'datapagesize'), + (None, 'datasrc'), + (None, 'datetime'), + (None, 'default'), + (None, 'delay'), + (None, 'dir'), + (None, 'disabled'), + (None, 'draggable'), + (None, 'dynsrc'), + (None, 'enctype'), + (None, 'end'), + (None, 'face'), + (None, 'for'), + (None, 'form'), + (None, 'frame'), + (None, 'galleryimg'), + (None, 'gutter'), + (None, 'headers'), + (None, 'height'), + (None, 'hidefocus'), + (None, 'hidden'), + (None, 'high'), + (None, 'href'), + (None, 'hreflang'), + (None, 'hspace'), + (None, 'icon'), + (None, 'id'), + (None, 'inputmode'), + (None, 'ismap'), + (None, 'keytype'), + (None, 'label'), + (None, 'leftspacing'), + (None, 'lang'), + (None, 'list'), + (None, 'longdesc'), + (None, 'loop'), + (None, 'loopcount'), + (None, 'loopend'), + (None, 'loopstart'), + (None, 'low'), + (None, 'lowsrc'), + (None, 'max'), + (None, 'maxlength'), + (None, 'media'), + (None, 'method'), + (None, 'min'), + (None, 'multiple'), + (None, 'name'), + (None, 'nohref'), + (None, 'noshade'), + (None, 'nowrap'), + (None, 'open'), + (None, 'optimum'), + (None, 'pattern'), + (None, 'ping'), + (None, 'point-size'), + (None, 'poster'), + (None, 'pqg'), + (None, 'preload'), + (None, 'prompt'), + (None, 'radiogroup'), + (None, 'readonly'), + (None, 'rel'), + (None, 'repeat-max'), + (None, 'repeat-min'), + (None, 'replace'), + (None, 'required'), + (None, 'rev'), + (None, 'rightspacing'), + (None, 'rows'), + (None, 'rowspan'), + (None, 'rules'), + (None, 'scope'), + (None, 'selected'), + (None, 'shape'), + (None, 'size'), + (None, 'span'), + (None, 'src'), + (None, 'start'), + (None, 'step'), + (None, 'style'), + (None, 'summary'), + (None, 'suppress'), + (None, 'tabindex'), + (None, 'target'), + (None, 'template'), + (None, 'title'), + (None, 'toppadding'), + (None, 'type'), + (None, 'unselectable'), + (None, 'usemap'), + (None, 'urn'), + (None, 'valign'), + (None, 'value'), + (None, 'variable'), + (None, 'volume'), + (None, 'vspace'), + (None, 'vrml'), + (None, 'width'), + (None, 'wrap'), + (namespaces['xml'], 'lang'), + # MathML attributes + (None, 'actiontype'), + (None, 'align'), + (None, 'columnalign'), + (None, 'columnalign'), + (None, 'columnalign'), + (None, 'columnlines'), + (None, 'columnspacing'), + (None, 'columnspan'), + (None, 'depth'), + (None, 'display'), + (None, 'displaystyle'), + (None, 'equalcolumns'), + (None, 'equalrows'), + (None, 'fence'), + (None, 'fontstyle'), + (None, 'fontweight'), + (None, 'frame'), + (None, 'height'), + (None, 'linethickness'), + (None, 'lspace'), + (None, 'mathbackground'), + (None, 'mathcolor'), + (None, 'mathvariant'), + (None, 'mathvariant'), + (None, 'maxsize'), + (None, 'minsize'), + (None, 'other'), + (None, 'rowalign'), + (None, 'rowalign'), + (None, 'rowalign'), + (None, 'rowlines'), + (None, 'rowspacing'), + (None, 'rowspan'), + (None, 'rspace'), + (None, 'scriptlevel'), + (None, 'selection'), + (None, 'separator'), + (None, 'stretchy'), + (None, 'width'), + (None, 'width'), + (namespaces['xlink'], 'href'), + (namespaces['xlink'], 'show'), + (namespaces['xlink'], 'type'), + # SVG attributes + (None, 'accent-height'), + (None, 'accumulate'), + (None, 'additive'), + (None, 'alphabetic'), + (None, 'arabic-form'), + (None, 'ascent'), + (None, 'attributeName'), + (None, 'attributeType'), + (None, 'baseProfile'), + (None, 'bbox'), + (None, 'begin'), + (None, 'by'), + (None, 'calcMode'), + (None, 'cap-height'), + (None, 'class'), + (None, 'clip-path'), + (None, 'color'), + (None, 'color-rendering'), + (None, 'content'), + (None, 'cx'), + (None, 'cy'), + (None, 'd'), + (None, 'dx'), + (None, 'dy'), + (None, 'descent'), + (None, 'display'), + (None, 'dur'), + (None, 'end'), + (None, 'fill'), + (None, 'fill-opacity'), + (None, 'fill-rule'), + (None, 'font-family'), + (None, 'font-size'), + (None, 'font-stretch'), + (None, 'font-style'), + (None, 'font-variant'), + (None, 'font-weight'), + (None, 'from'), + (None, 'fx'), + (None, 'fy'), + (None, 'g1'), + (None, 'g2'), + (None, 'glyph-name'), + (None, 'gradientUnits'), + (None, 'hanging'), + (None, 'height'), + (None, 'horiz-adv-x'), + (None, 'horiz-origin-x'), + (None, 'id'), + (None, 'ideographic'), + (None, 'k'), + (None, 'keyPoints'), + (None, 'keySplines'), + (None, 'keyTimes'), + (None, 'lang'), + (None, 'marker-end'), + (None, 'marker-mid'), + (None, 'marker-start'), + (None, 'markerHeight'), + (None, 'markerUnits'), + (None, 'markerWidth'), + (None, 'mathematical'), + (None, 'max'), + (None, 'min'), + (None, 'name'), + (None, 'offset'), + (None, 'opacity'), + (None, 'orient'), + (None, 'origin'), + (None, 'overline-position'), + (None, 'overline-thickness'), + (None, 'panose-1'), + (None, 'path'), + (None, 'pathLength'), + (None, 'points'), + (None, 'preserveAspectRatio'), + (None, 'r'), + (None, 'refX'), + (None, 'refY'), + (None, 'repeatCount'), + (None, 'repeatDur'), + (None, 'requiredExtensions'), + (None, 'requiredFeatures'), + (None, 'restart'), + (None, 'rotate'), + (None, 'rx'), + (None, 'ry'), + (None, 'slope'), + (None, 'stemh'), + (None, 'stemv'), + (None, 'stop-color'), + (None, 'stop-opacity'), + (None, 'strikethrough-position'), + (None, 'strikethrough-thickness'), + (None, 'stroke'), + (None, 'stroke-dasharray'), + (None, 'stroke-dashoffset'), + (None, 'stroke-linecap'), + (None, 'stroke-linejoin'), + (None, 'stroke-miterlimit'), + (None, 'stroke-opacity'), + (None, 'stroke-width'), + (None, 'systemLanguage'), + (None, 'target'), + (None, 'text-anchor'), + (None, 'to'), + (None, 'transform'), + (None, 'type'), + (None, 'u1'), + (None, 'u2'), + (None, 'underline-position'), + (None, 'underline-thickness'), + (None, 'unicode'), + (None, 'unicode-range'), + (None, 'units-per-em'), + (None, 'values'), + (None, 'version'), + (None, 'viewBox'), + (None, 'visibility'), + (None, 'width'), + (None, 'widths'), + (None, 'x'), + (None, 'x-height'), + (None, 'x1'), + (None, 'x2'), + (namespaces['xlink'], 'actuate'), + (namespaces['xlink'], 'arcrole'), + (namespaces['xlink'], 'href'), + (namespaces['xlink'], 'role'), + (namespaces['xlink'], 'show'), + (namespaces['xlink'], 'title'), + (namespaces['xlink'], 'type'), + (namespaces['xml'], 'base'), + (namespaces['xml'], 'lang'), + (namespaces['xml'], 'space'), + (None, 'y'), + (None, 'y1'), + (None, 'y2'), + (None, 'zoomAndPan'), +)) + +attr_val_is_uri = frozenset(( + (None, 'href'), + (None, 'src'), + (None, 'cite'), + (None, 'action'), + (None, 'longdesc'), + (None, 'poster'), + (None, 'background'), + (None, 'datasrc'), + (None, 'dynsrc'), + (None, 'lowsrc'), + (None, 'ping'), + (namespaces['xlink'], 'href'), + (namespaces['xml'], 'base'), +)) + +svg_attr_val_allows_ref = frozenset(( + (None, 'clip-path'), + (None, 'color-profile'), + (None, 'cursor'), + (None, 'fill'), + (None, 'filter'), + (None, 'marker'), + (None, 'marker-start'), + (None, 'marker-mid'), + (None, 'marker-end'), + (None, 'mask'), + (None, 'stroke'), +)) + +svg_allow_local_href = frozenset(( + (None, 'altGlyph'), + (None, 'animate'), + (None, 'animateColor'), + (None, 'animateMotion'), + (None, 'animateTransform'), + (None, 'cursor'), + (None, 'feImage'), + (None, 'filter'), + (None, 'linearGradient'), + (None, 'pattern'), + (None, 'radialGradient'), + (None, 'textpath'), + (None, 'tref'), + (None, 'set'), + (None, 'use') +)) + +acceptable_css_properties = frozenset(( + 'azimuth', + 'background-color', + 'border-bottom-color', + 'border-collapse', + 'border-color', + 'border-left-color', + 'border-right-color', + 'border-top-color', + 'clear', + 'color', + 'cursor', + 'direction', + 'display', + 'elevation', + 'float', + 'font', + 'font-family', + 'font-size', + 'font-style', + 'font-variant', + 'font-weight', + 'height', + 'letter-spacing', + 'line-height', + 'overflow', + 'pause', + 'pause-after', + 'pause-before', + 'pitch', + 'pitch-range', + 'richness', + 'speak', + 'speak-header', + 'speak-numeral', + 'speak-punctuation', + 'speech-rate', + 'stress', + 'text-align', + 'text-decoration', + 'text-indent', + 'unicode-bidi', + 'vertical-align', + 'voice-family', + 'volume', + 'white-space', + 'width', +)) + +acceptable_css_keywords = frozenset(( + 'auto', + 'aqua', + 'black', + 'block', + 'blue', + 'bold', + 'both', + 'bottom', + 'brown', + 'center', + 'collapse', + 'dashed', + 'dotted', + 'fuchsia', + 'gray', + 'green', + '!important', + 'italic', + 'left', + 'lime', + 'maroon', + 'medium', + 'none', + 'navy', + 'normal', + 'nowrap', + 'olive', + 'pointer', + 'purple', + 'red', + 'right', + 'solid', + 'silver', + 'teal', + 'top', + 'transparent', + 'underline', + 'white', + 'yellow', +)) + +acceptable_svg_properties = frozenset(( + 'fill', + 'fill-opacity', + 'fill-rule', + 'stroke', + 'stroke-width', + 'stroke-linecap', + 'stroke-linejoin', + 'stroke-opacity', +)) + +acceptable_protocols = frozenset(( + 'ed2k', + 'ftp', + 'http', + 'https', + 'irc', + 'mailto', + 'news', + 'gopher', + 'nntp', + 'telnet', + 'webcal', + 'xmpp', + 'callto', + 'feed', + 'urn', + 'aim', + 'rsync', + 'tag', + 'ssh', + 'sftp', + 'rtsp', + 'afs', + 'data', +)) + +acceptable_content_types = frozenset(( + 'image/png', + 'image/jpeg', + 'image/gif', + 'image/webp', + 'image/bmp', + 'text/plain', +)) + +allowed_elements = acceptable_elements +allowed_attributes = acceptable_attributes +allowed_css_properties = acceptable_css_properties +allowed_css_keywords = acceptable_css_keywords +allowed_svg_properties = acceptable_svg_properties +allowed_protocols = acceptable_protocols +allowed_content_types = acceptable_content_types + +data_content_type = re.compile(r''' + ^ + # Match a content type / + (?P[-a-zA-Z0-9.]+/[-a-zA-Z0-9.]+) + # Match any character set and encoding + (?:(?:;charset=(?:[-a-zA-Z0-9]+)(?:;(?:base64))?) + |(?:;(?:base64))?(?:;charset=(?:[-a-zA-Z0-9]+))?) + # Assume the rest is data + ,.* + $ + ''', + re.VERBOSE) + + +class Filter(_base.Filter): + """ sanitization of XHTML+MathML+SVG and of inline style attributes.""" + def __init__(self, + source, + allowed_elements=allowed_elements, + allowed_attributes=allowed_attributes, + allowed_css_properties=allowed_css_properties, + allowed_css_keywords=allowed_css_keywords, + allowed_svg_properties=allowed_svg_properties, + allowed_protocols=allowed_protocols, + allowed_content_types=allowed_content_types, + attr_val_is_uri=attr_val_is_uri, + svg_attr_val_allows_ref=svg_attr_val_allows_ref, + svg_allow_local_href=svg_allow_local_href): + super(Filter, self).__init__(source) + self.allowed_elements = allowed_elements + self.allowed_attributes = allowed_attributes + self.allowed_css_properties = allowed_css_properties + self.allowed_css_keywords = allowed_css_keywords + self.allowed_svg_properties = allowed_svg_properties + self.allowed_protocols = allowed_protocols + self.allowed_content_types = allowed_content_types + self.attr_val_is_uri = attr_val_is_uri + self.svg_attr_val_allows_ref = svg_attr_val_allows_ref + self.svg_allow_local_href = svg_allow_local_href -class Filter(_base.Filter, HTMLSanitizerMixin): def __iter__(self): for token in _base.Filter.__iter__(self): token = self.sanitize_token(token) if token: yield token + + # Sanitize the +html+, escaping all elements not in ALLOWED_ELEMENTS, and + # stripping out all # attributes not in ALLOWED_ATTRIBUTES. Style + # attributes are parsed, and a restricted set, # specified by + # ALLOWED_CSS_PROPERTIES and ALLOWED_CSS_KEYWORDS, are allowed through. + # attributes in ATTR_VAL_IS_URI are scanned, and only URI schemes specified + # in ALLOWED_PROTOCOLS are allowed. + # + # sanitize_html('') + # => <script> do_nasty_stuff() </script> + # sanitize_html('Click here for $100') + # => Click here for $100 + def sanitize_token(self, token): + + # accommodate filters which use token_type differently + token_type = token["type"] + if token_type in ("StartTag", "EndTag", "EmptyTag"): + name = token["name"] + namespace = token["namespace"] + if ((namespace, name) in self.allowed_elements or + (namespace is None and + (namespaces["html"], name) in self.allowed_elements)): + return self.allowed_token(token, token_type) + else: + return self.disallowed_token(token, token_type) + elif token_type == "Comment": + pass + else: + return token + + def allowed_token(self, token, token_type): + if "data" in token: + attrs = token["data"] + attr_names = set(attrs.keys()) + + # Remove forbidden attributes + for to_remove in (attr_names - self.allowed_attributes): + del token["data"][to_remove] + attr_names.remove(to_remove) + + # Remove attributes with disallowed URL values + for attr in (attr_names & self.attr_val_is_uri): + assert attr in attrs + # I don't have a clue where this regexp comes from or why it matches those + # characters, nor why we call unescape. I just know it's always been here. + # Should you be worried by this comment in a sanitizer? Yes. On the other hand, all + # this will do is remove *more* than it otherwise would. + val_unescaped = re.sub("[`\x00-\x20\x7f-\xa0\s]+", '', + unescape(attrs[attr])).lower() + # remove replacement characters from unescaped characters + val_unescaped = val_unescaped.replace("\ufffd", "") + try: + uri = urlparse.urlparse(val_unescaped) + except ValueError: + uri = None + del attrs[attr] + if uri and uri.scheme: + if uri.scheme not in self.allowed_protocols: + del attrs[attr] + if uri.scheme == 'data': + m = data_content_type.match(uri.path) + if not m: + del attrs[attr] + elif m.group('content_type') not in self.allowed_content_types: + del attrs[attr] + + for attr in self.svg_attr_val_allows_ref: + if attr in attrs: + attrs[attr] = re.sub(r'url\s*\(\s*[^#\s][^)]+?\)', + ' ', + unescape(attrs[attr])) + if (token["name"] in self.svg_allow_local_href and + (namespaces['xlink'], 'href') in attrs and re.search('^\s*[^#\s].*', + attrs[(namespaces['xlink'], 'href')])): + del attrs[(namespaces['xlink'], 'href')] + if (None, 'style') in attrs: + attrs[(None, 'style')] = self.sanitize_css(attrs[(None, 'style')]) + token["data"] = attrs + return token + + def disallowed_token(self, token, token_type): + if token_type == "EndTag": + token["data"] = "" % token["name"] + elif token["data"]: + assert token_type in ("StartTag", "EmptyTag") + attrs = [] + for (ns, name), v in token["data"].items(): + attrs.append(' %s="%s"' % (name if ns is None else "%s:%s" % (prefixes[ns], name), escape(v))) + token["data"] = "<%s%s>" % (token["name"], ''.join(attrs)) + else: + token["data"] = "<%s>" % token["name"] + if token.get("selfClosing"): + token["data"] = token["data"][:-1] + "/>" + + token["type"] = "Characters" + + del token["name"] + return token + + def sanitize_css(self, style): + # disallow urls + style = re.compile('url\s*\(\s*[^\s)]+?\s*\)\s*').sub(' ', style) + + # gauntlet + if not re.match("""^([:,;#%.\sa-zA-Z0-9!]|\w-\w|'[\s\w]+'|"[\s\w]+"|\([\d,\s]+\))*$""", style): + return '' + if not re.match("^\s*([-\w]+\s*:[^:;]*(;\s*|$))*$", style): + return '' + + clean = [] + for prop, value in re.findall("([-\w]+)\s*:\s*([^:;]*)", style): + if not value: + continue + if prop.lower() in self.allowed_css_properties: + clean.append(prop + ': ' + value + ';') + elif prop.split('-')[0].lower() in ['background', 'border', 'margin', + 'padding']: + for keyword in value.split(): + if keyword not in self.allowed_css_keywords and \ + not re.match("^(#[0-9a-f]+|rgb\(\d+%?,\d*%?,?\d*%?\)?|\d{0,2}\.?\d{0,2}(cm|em|ex|in|mm|pc|pt|px|%|,|\))?)$", keyword): + break + else: + clean.append(prop + ': ' + value + ';') + elif prop.lower() in self.allowed_svg_properties: + clean.append(prop + ': ' + value + ';') + + return ' '.join(clean) diff --git a/html5lib/sanitizer.py b/html5lib/sanitizer.py deleted file mode 100644 index b714e8c9..00000000 --- a/html5lib/sanitizer.py +++ /dev/null @@ -1,300 +0,0 @@ -from __future__ import absolute_import, division, unicode_literals - -import re -from xml.sax.saxutils import escape, unescape -from six.moves import urllib_parse as urlparse - -from .tokenizer import HTMLTokenizer -from .constants import tokenTypes - - -content_type_rgx = re.compile(r''' - ^ - # Match a content type / - (?P[-a-zA-Z0-9.]+/[-a-zA-Z0-9.]+) - # Match any character set and encoding - (?:(?:;charset=(?:[-a-zA-Z0-9]+)(?:;(?:base64))?) - |(?:;(?:base64))?(?:;charset=(?:[-a-zA-Z0-9]+))?) - # Assume the rest is data - ,.* - $ - ''', - re.VERBOSE) - - -class HTMLSanitizerMixin(object): - """ sanitization of XHTML+MathML+SVG and of inline style attributes.""" - - acceptable_elements = ['a', 'abbr', 'acronym', 'address', 'area', - 'article', 'aside', 'audio', 'b', 'big', 'blockquote', 'br', 'button', - 'canvas', 'caption', 'center', 'cite', 'code', 'col', 'colgroup', - 'command', 'datagrid', 'datalist', 'dd', 'del', 'details', 'dfn', - 'dialog', 'dir', 'div', 'dl', 'dt', 'em', 'event-source', 'fieldset', - 'figcaption', 'figure', 'footer', 'font', 'form', 'header', 'h1', - 'h2', 'h3', 'h4', 'h5', 'h6', 'hr', 'i', 'img', 'input', 'ins', - 'keygen', 'kbd', 'label', 'legend', 'li', 'm', 'map', 'menu', 'meter', - 'multicol', 'nav', 'nextid', 'ol', 'output', 'optgroup', 'option', - 'p', 'pre', 'progress', 'q', 's', 'samp', 'section', 'select', - 'small', 'sound', 'source', 'spacer', 'span', 'strike', 'strong', - 'sub', 'sup', 'table', 'tbody', 'td', 'textarea', 'time', 'tfoot', - 'th', 'thead', 'tr', 'tt', 'u', 'ul', 'var', 'video'] - - mathml_elements = ['maction', 'math', 'merror', 'mfrac', 'mi', - 'mmultiscripts', 'mn', 'mo', 'mover', 'mpadded', 'mphantom', - 'mprescripts', 'mroot', 'mrow', 'mspace', 'msqrt', 'mstyle', 'msub', - 'msubsup', 'msup', 'mtable', 'mtd', 'mtext', 'mtr', 'munder', - 'munderover', 'none'] - - svg_elements = ['a', 'animate', 'animateColor', 'animateMotion', - 'animateTransform', 'clipPath', 'circle', 'defs', 'desc', 'ellipse', - 'font-face', 'font-face-name', 'font-face-src', 'g', 'glyph', 'hkern', - 'linearGradient', 'line', 'marker', 'metadata', 'missing-glyph', - 'mpath', 'path', 'polygon', 'polyline', 'radialGradient', 'rect', - 'set', 'stop', 'svg', 'switch', 'text', 'title', 'tspan', 'use'] - - acceptable_attributes = ['abbr', 'accept', 'accept-charset', 'accesskey', - 'action', 'align', 'alt', 'autocomplete', 'autofocus', 'axis', - 'background', 'balance', 'bgcolor', 'bgproperties', 'border', - 'bordercolor', 'bordercolordark', 'bordercolorlight', 'bottompadding', - 'cellpadding', 'cellspacing', 'ch', 'challenge', 'char', 'charoff', - 'choff', 'charset', 'checked', 'cite', 'class', 'clear', 'color', - 'cols', 'colspan', 'compact', 'contenteditable', 'controls', 'coords', - 'data', 'datafld', 'datapagesize', 'datasrc', 'datetime', 'default', - 'delay', 'dir', 'disabled', 'draggable', 'dynsrc', 'enctype', 'end', - 'face', 'for', 'form', 'frame', 'galleryimg', 'gutter', 'headers', - 'height', 'hidefocus', 'hidden', 'high', 'href', 'hreflang', 'hspace', - 'icon', 'id', 'inputmode', 'ismap', 'keytype', 'label', 'leftspacing', - 'lang', 'list', 'longdesc', 'loop', 'loopcount', 'loopend', - 'loopstart', 'low', 'lowsrc', 'max', 'maxlength', 'media', 'method', - 'min', 'multiple', 'name', 'nohref', 'noshade', 'nowrap', 'open', - 'optimum', 'pattern', 'ping', 'point-size', 'poster', 'pqg', 'preload', - 'prompt', 'radiogroup', 'readonly', 'rel', 'repeat-max', 'repeat-min', - 'replace', 'required', 'rev', 'rightspacing', 'rows', 'rowspan', - 'rules', 'scope', 'selected', 'shape', 'size', 'span', 'src', 'start', - 'step', 'style', 'summary', 'suppress', 'tabindex', 'target', - 'template', 'title', 'toppadding', 'type', 'unselectable', 'usemap', - 'urn', 'valign', 'value', 'variable', 'volume', 'vspace', 'vrml', - 'width', 'wrap', 'xml:lang'] - - mathml_attributes = ['actiontype', 'align', 'columnalign', 'columnalign', - 'columnalign', 'columnlines', 'columnspacing', 'columnspan', 'depth', - 'display', 'displaystyle', 'equalcolumns', 'equalrows', 'fence', - 'fontstyle', 'fontweight', 'frame', 'height', 'linethickness', 'lspace', - 'mathbackground', 'mathcolor', 'mathvariant', 'mathvariant', 'maxsize', - 'minsize', 'other', 'rowalign', 'rowalign', 'rowalign', 'rowlines', - 'rowspacing', 'rowspan', 'rspace', 'scriptlevel', 'selection', - 'separator', 'stretchy', 'width', 'width', 'xlink:href', 'xlink:show', - 'xlink:type', 'xmlns', 'xmlns:xlink'] - - svg_attributes = ['accent-height', 'accumulate', 'additive', 'alphabetic', - 'arabic-form', 'ascent', 'attributeName', 'attributeType', - 'baseProfile', 'bbox', 'begin', 'by', 'calcMode', 'cap-height', - 'class', 'clip-path', 'color', 'color-rendering', 'content', 'cx', - 'cy', 'd', 'dx', 'dy', 'descent', 'display', 'dur', 'end', 'fill', - 'fill-opacity', 'fill-rule', 'font-family', 'font-size', - 'font-stretch', 'font-style', 'font-variant', 'font-weight', 'from', - 'fx', 'fy', 'g1', 'g2', 'glyph-name', 'gradientUnits', 'hanging', - 'height', 'horiz-adv-x', 'horiz-origin-x', 'id', 'ideographic', 'k', - 'keyPoints', 'keySplines', 'keyTimes', 'lang', 'marker-end', - 'marker-mid', 'marker-start', 'markerHeight', 'markerUnits', - 'markerWidth', 'mathematical', 'max', 'min', 'name', 'offset', - 'opacity', 'orient', 'origin', 'overline-position', - 'overline-thickness', 'panose-1', 'path', 'pathLength', 'points', - 'preserveAspectRatio', 'r', 'refX', 'refY', 'repeatCount', - 'repeatDur', 'requiredExtensions', 'requiredFeatures', 'restart', - 'rotate', 'rx', 'ry', 'slope', 'stemh', 'stemv', 'stop-color', - 'stop-opacity', 'strikethrough-position', 'strikethrough-thickness', - 'stroke', 'stroke-dasharray', 'stroke-dashoffset', 'stroke-linecap', - 'stroke-linejoin', 'stroke-miterlimit', 'stroke-opacity', - 'stroke-width', 'systemLanguage', 'target', 'text-anchor', 'to', - 'transform', 'type', 'u1', 'u2', 'underline-position', - 'underline-thickness', 'unicode', 'unicode-range', 'units-per-em', - 'values', 'version', 'viewBox', 'visibility', 'width', 'widths', 'x', - 'x-height', 'x1', 'x2', 'xlink:actuate', 'xlink:arcrole', - 'xlink:href', 'xlink:role', 'xlink:show', 'xlink:title', 'xlink:type', - 'xml:base', 'xml:lang', 'xml:space', 'xmlns', 'xmlns:xlink', 'y', - 'y1', 'y2', 'zoomAndPan'] - - attr_val_is_uri = ['href', 'src', 'cite', 'action', 'longdesc', 'poster', 'background', 'datasrc', - 'dynsrc', 'lowsrc', 'ping', 'poster', 'xlink:href', 'xml:base'] - - svg_attr_val_allows_ref = ['clip-path', 'color-profile', 'cursor', 'fill', - 'filter', 'marker', 'marker-start', 'marker-mid', 'marker-end', - 'mask', 'stroke'] - - svg_allow_local_href = ['altGlyph', 'animate', 'animateColor', - 'animateMotion', 'animateTransform', 'cursor', 'feImage', 'filter', - 'linearGradient', 'pattern', 'radialGradient', 'textpath', 'tref', - 'set', 'use'] - - acceptable_css_properties = ['azimuth', 'background-color', - 'border-bottom-color', 'border-collapse', 'border-color', - 'border-left-color', 'border-right-color', 'border-top-color', 'clear', - 'color', 'cursor', 'direction', 'display', 'elevation', 'float', 'font', - 'font-family', 'font-size', 'font-style', 'font-variant', 'font-weight', - 'height', 'letter-spacing', 'line-height', 'overflow', 'pause', - 'pause-after', 'pause-before', 'pitch', 'pitch-range', 'richness', - 'speak', 'speak-header', 'speak-numeral', 'speak-punctuation', - 'speech-rate', 'stress', 'text-align', 'text-decoration', 'text-indent', - 'unicode-bidi', 'vertical-align', 'voice-family', 'volume', - 'white-space', 'width'] - - acceptable_css_keywords = ['auto', 'aqua', 'black', 'block', 'blue', - 'bold', 'both', 'bottom', 'brown', 'center', 'collapse', 'dashed', - 'dotted', 'fuchsia', 'gray', 'green', '!important', 'italic', 'left', - 'lime', 'maroon', 'medium', 'none', 'navy', 'normal', 'nowrap', 'olive', - 'pointer', 'purple', 'red', 'right', 'solid', 'silver', 'teal', 'top', - 'transparent', 'underline', 'white', 'yellow'] - - acceptable_svg_properties = ['fill', 'fill-opacity', 'fill-rule', - 'stroke', 'stroke-width', 'stroke-linecap', 'stroke-linejoin', - 'stroke-opacity'] - - acceptable_protocols = ['ed2k', 'ftp', 'http', 'https', 'irc', - 'mailto', 'news', 'gopher', 'nntp', 'telnet', 'webcal', - 'xmpp', 'callto', 'feed', 'urn', 'aim', 'rsync', 'tag', - 'ssh', 'sftp', 'rtsp', 'afs', 'data'] - - acceptable_content_types = ['image/png', 'image/jpeg', 'image/gif', 'image/webp', 'image/bmp', 'text/plain'] - - # subclasses may define their own versions of these constants - allowed_elements = acceptable_elements + mathml_elements + svg_elements - allowed_attributes = acceptable_attributes + mathml_attributes + svg_attributes - allowed_css_properties = acceptable_css_properties - allowed_css_keywords = acceptable_css_keywords - allowed_svg_properties = acceptable_svg_properties - allowed_protocols = acceptable_protocols - allowed_content_types = acceptable_content_types - - # Sanitize the +html+, escaping all elements not in ALLOWED_ELEMENTS, and - # stripping out all # attributes not in ALLOWED_ATTRIBUTES. Style - # attributes are parsed, and a restricted set, # specified by - # ALLOWED_CSS_PROPERTIES and ALLOWED_CSS_KEYWORDS, are allowed through. - # attributes in ATTR_VAL_IS_URI are scanned, and only URI schemes specified - # in ALLOWED_PROTOCOLS are allowed. - # - # sanitize_html('') - # => <script> do_nasty_stuff() </script> - # sanitize_html('Click here for $100') - # => Click here for $100 - def sanitize_token(self, token): - - # accommodate filters which use token_type differently - token_type = token["type"] - if token_type in list(tokenTypes.keys()): - token_type = tokenTypes[token_type] - - if token_type in (tokenTypes["StartTag"], tokenTypes["EndTag"], - tokenTypes["EmptyTag"]): - if token["name"] in self.allowed_elements: - return self.allowed_token(token, token_type) - else: - return self.disallowed_token(token, token_type) - elif token_type == tokenTypes["Comment"]: - pass - else: - return token - - def allowed_token(self, token, token_type): - if "data" in token: - attrs = dict([(name, val) for name, val in - token["data"][::-1] - if name in self.allowed_attributes]) - for attr in self.attr_val_is_uri: - if attr not in attrs: - continue - val_unescaped = re.sub("[`\000-\040\177-\240\s]+", '', - unescape(attrs[attr])).lower() - # remove replacement characters from unescaped characters - val_unescaped = val_unescaped.replace("\ufffd", "") - try: - uri = urlparse.urlparse(val_unescaped) - except ValueError: - uri = None - del attrs[attr] - if uri and uri.scheme: - if uri.scheme not in self.allowed_protocols: - del attrs[attr] - if uri.scheme == 'data': - m = content_type_rgx.match(uri.path) - if not m: - del attrs[attr] - elif m.group('content_type') not in self.allowed_content_types: - del attrs[attr] - - for attr in self.svg_attr_val_allows_ref: - if attr in attrs: - attrs[attr] = re.sub(r'url\s*\(\s*[^#\s][^)]+?\)', - ' ', - unescape(attrs[attr])) - if (token["name"] in self.svg_allow_local_href and - 'xlink:href' in attrs and re.search('^\s*[^#\s].*', - attrs['xlink:href'])): - del attrs['xlink:href'] - if 'style' in attrs: - attrs['style'] = self.sanitize_css(attrs['style']) - token["data"] = [[name, val] for name, val in list(attrs.items())] - return token - - def disallowed_token(self, token, token_type): - if token_type == tokenTypes["EndTag"]: - token["data"] = "" % token["name"] - elif token["data"]: - attrs = ''.join([' %s="%s"' % (k, escape(v)) for k, v in token["data"]]) - token["data"] = "<%s%s>" % (token["name"], attrs) - else: - token["data"] = "<%s>" % token["name"] - if token.get("selfClosing"): - token["data"] = token["data"][:-1] + "/>" - - if token["type"] in list(tokenTypes.keys()): - token["type"] = "Characters" - else: - token["type"] = tokenTypes["Characters"] - - del token["name"] - return token - - def sanitize_css(self, style): - # disallow urls - style = re.compile('url\s*\(\s*[^\s)]+?\s*\)\s*').sub(' ', style) - - # gauntlet - if not re.match("""^([:,;#%.\sa-zA-Z0-9!]|\w-\w|'[\s\w]+'|"[\s\w]+"|\([\d,\s]+\))*$""", style): - return '' - if not re.match("^\s*([-\w]+\s*:[^:;]*(;\s*|$))*$", style): - return '' - - clean = [] - for prop, value in re.findall("([-\w]+)\s*:\s*([^:;]*)", style): - if not value: - continue - if prop.lower() in self.allowed_css_properties: - clean.append(prop + ': ' + value + ';') - elif prop.split('-')[0].lower() in ['background', 'border', 'margin', - 'padding']: - for keyword in value.split(): - if keyword not in self.acceptable_css_keywords and \ - not re.match("^(#[0-9a-f]+|rgb\(\d+%?,\d*%?,?\d*%?\)?|\d{0,2}\.?\d{0,2}(cm|em|ex|in|mm|pc|pt|px|%|,|\))?)$", keyword): - break - else: - clean.append(prop + ': ' + value + ';') - elif prop.lower() in self.allowed_svg_properties: - clean.append(prop + ': ' + value + ';') - - return ' '.join(clean) - - -class HTMLSanitizer(HTMLTokenizer, HTMLSanitizerMixin): - def __init__(self, stream, encoding=None, parseMeta=True, useChardet=True, - lowercaseElementName=False, lowercaseAttrName=False, parser=None): - # Change case matching defaults as we only output lowercase html anyway - # This solution doesn't seem ideal... - HTMLTokenizer.__init__(self, stream, encoding, parseMeta, useChardet, - lowercaseElementName, lowercaseAttrName, parser=parser) - - def __iter__(self): - for token in HTMLTokenizer.__iter__(self): - token = self.sanitize_token(token) - if token: - yield token diff --git a/html5lib/tests/test_sanitizer.py b/html5lib/tests/test_sanitizer.py index e98c8c85..1f8a06f6 100644 --- a/html5lib/tests/test_sanitizer.py +++ b/html5lib/tests/test_sanitizer.py @@ -1,134 +1,114 @@ from __future__ import absolute_import, division, unicode_literals -try: - import json -except ImportError: - import simplejson as json - -from html5lib import html5parser, sanitizer, constants, treebuilders +from html5lib import constants, parseFragment, serialize +from html5lib.filters import sanitizer + + +def runSanitizerTest(name, expected, input): + parsed = parseFragment(expected) + expected = serialize(parsed, + omit_optional_tags=False, + use_trailing_solidus=True, + space_before_trailing_solidus=False, + quote_attr_values="always", + quote_char='"', + alphabetical_attributes=True) + assert expected == sanitize_html(input) -def toxmlFactory(): - tree = treebuilders.getTreeBuilder("etree") +def sanitize_html(stream): + parsed = parseFragment(stream) + serialized = serialize(parsed, + sanitize=True, + omit_optional_tags=False, + use_trailing_solidus=True, + space_before_trailing_solidus=False, + quote_attr_values="always", + quote_char='"', + alphabetical_attributes=True) + return serialized - def toxml(element): - # encode/decode roundtrip required for Python 2.6 compatibility - result_bytes = tree.implementation.tostring(element, encoding="utf-8") - return result_bytes.decode("utf-8") - return toxml +def test_should_handle_astral_plane_characters(): + sanitized = sanitize_html("

𝒵 𝔸

") + expected = '

\U0001d4b5 \U0001d538

' + assert expected == sanitized -def runSanitizerTest(name, expected, input, toxml=None): - if toxml is None: - toxml = toxmlFactory() - expected = ''.join([toxml(token) for token in html5parser.HTMLParser(). - parseFragment(expected)]) - expected = json.loads(json.dumps(expected)) - assert expected == sanitize_html(input) +def test_should_allow_relative_uris(): + sanitized = sanitize_html('

') + expected = '

' + assert expected == sanitized -def sanitize_html(stream, toxml=None): - if toxml is None: - toxml = toxmlFactory() - return ''.join([toxml(token) for token in - html5parser.HTMLParser(tokenizer=sanitizer.HTMLSanitizer). - parseFragment(stream)]) +def test_invalid_data_uri(): + sanitized = sanitize_html('') + expected = '' + assert expected == sanitized -def test_should_handle_astral_plane_characters(): - assert '\U0001d4b5 \U0001d538' == sanitize_html("

𝒵 𝔸

") +def test_invalid_ipv6_url(): + sanitized = sanitize_html('') + expected = "" + assert expected == sanitized -def test_should_allow_relative_uris(): - assert '' == sanitize_html('

') +def test_data_uri_disallowed_type(): + sanitized = sanitize_html('') + expected = "" + assert expected == sanitized def test_sanitizer(): - toxml = toxmlFactory() - for tag_name in sanitizer.HTMLSanitizer.allowed_elements: - if tag_name in ['caption', 'col', 'colgroup', 'optgroup', 'option', 'table', 'tbody', 'td', 'tfoot', 'th', 'thead', 'tr']: - continue # TODO - if tag_name != tag_name.lower(): + for ns, tag_name in sanitizer.allowed_elements: + if ns != constants.namespaces["html"]: + continue + if tag_name in ['caption', 'col', 'colgroup', 'optgroup', 'option', 'table', 'tbody', 'td', 'tfoot', 'th', 'thead', 'tr', 'select']: continue # TODO if tag_name == 'image': yield (runSanitizerTest, "test_should_allow_%s_tag" % tag_name, "foo <bad>bar</bad> baz", - "<%s title='1'>foo bar baz" % (tag_name, tag_name), - toxml) + "<%s title='1'>foo bar baz" % (tag_name, tag_name)) elif tag_name == 'br': yield (runSanitizerTest, "test_should_allow_%s_tag" % tag_name, "
foo <bad>bar</bad> baz
", - "<%s title='1'>foo bar baz" % (tag_name, tag_name), - toxml) + "<%s title='1'>foo bar baz" % (tag_name, tag_name)) elif tag_name in constants.voidElements: yield (runSanitizerTest, "test_should_allow_%s_tag" % tag_name, "<%s title=\"1\"/>foo <bad>bar</bad> baz" % tag_name, - "<%s title='1'>foo bar baz" % (tag_name, tag_name), - toxml) + "<%s title='1'>foo bar baz" % (tag_name, tag_name)) else: yield (runSanitizerTest, "test_should_allow_%s_tag" % tag_name, "<%s title=\"1\">foo <bad>bar</bad> baz" % (tag_name, tag_name), - "<%s title='1'>foo bar baz" % (tag_name, tag_name), - toxml) - - for tag_name in sanitizer.HTMLSanitizer.allowed_elements: - tag_name = tag_name.upper() - yield (runSanitizerTest, "test_should_forbid_%s_tag" % tag_name, - "<%s title=\"1\">foo <bad>bar</bad> baz</%s>" % (tag_name, tag_name), - "<%s title='1'>foo bar baz" % (tag_name, tag_name), - toxml) + "<%s title='1'>foo bar baz" % (tag_name, tag_name)) - for attribute_name in sanitizer.HTMLSanitizer.allowed_attributes: + for ns, attribute_name in sanitizer.allowed_attributes: + if ns is not None: + continue if attribute_name != attribute_name.lower(): continue # TODO if attribute_name == 'style': continue attribute_value = 'foo' - if attribute_name in sanitizer.HTMLSanitizer.attr_val_is_uri: - attribute_value = '%s://sub.domain.tld/path/object.ext' % sanitizer.HTMLSanitizer.allowed_protocols[0] + if attribute_name in sanitizer.attr_val_is_uri: + attribute_value = '%s://sub.domain.tld/path/object.ext' % sanitizer.allowed_protocols[0] yield (runSanitizerTest, "test_should_allow_%s_attribute" % attribute_name, "

foo <bad>bar</bad> baz

" % (attribute_name, attribute_value), - "

foo bar baz

" % (attribute_name, attribute_value), - toxml) - - for attribute_name in sanitizer.HTMLSanitizer.allowed_attributes: - attribute_name = attribute_name.upper() - yield (runSanitizerTest, "test_should_forbid_%s_attribute" % attribute_name, - "

foo <bad>bar</bad> baz

", - "

foo bar baz

" % attribute_name, - toxml) + "

foo bar baz

" % (attribute_name, attribute_value)) - for protocol in sanitizer.HTMLSanitizer.allowed_protocols: + for protocol in sanitizer.allowed_protocols: rest_of_uri = '//sub.domain.tld/path/object.ext' if protocol == 'data': rest_of_uri = 'image/png;base64,aGVsbG8gd29ybGQ=' yield (runSanitizerTest, "test_should_allow_uppercase_%s_uris" % protocol, "foo" % (protocol, rest_of_uri), - """foo""" % (protocol, rest_of_uri), - toxml) - - yield (runSanitizerTest, "test_invalid_data_uri", - "", - "", - toxml) - - yield (runSanitizerTest, "test_invalid_ipv6_url", - "", - "", - toxml) - - yield (runSanitizerTest, "test_data_uri_disallowed_type", - "", - "", - toxml) + """foo""" % (protocol, rest_of_uri)) - for protocol in sanitizer.HTMLSanitizer.allowed_protocols: + for protocol in sanitizer.allowed_protocols: rest_of_uri = '//sub.domain.tld/path/object.ext' if protocol == 'data': rest_of_uri = 'image/png;base64,aGVsbG8gd29ybGQ=' protocol = protocol.upper() yield (runSanitizerTest, "test_should_allow_uppercase_%s_uris" % protocol, "foo" % (protocol, rest_of_uri), - """foo""" % (protocol, rest_of_uri), - toxml) + """foo""" % (protocol, rest_of_uri)) diff --git a/parse.py b/parse.py index 2245060a..cceea84d 100755 --- a/parse.py +++ b/parse.py @@ -9,7 +9,7 @@ import traceback from optparse import OptionParser -from html5lib import html5parser, sanitizer +from html5lib import html5parser from html5lib.tokenizer import HTMLTokenizer from html5lib import treebuilders, serializer, treewalkers from html5lib import constants @@ -50,10 +50,7 @@ def parse(): treebuilder = treebuilders.getTreeBuilder(opts.treebuilder) - if opts.sanitize: - tokenizer = sanitizer.HTMLSanitizer - else: - tokenizer = HTMLTokenizer + tokenizer = HTMLTokenizer p = html5parser.HTMLParser(tree=treebuilder, tokenizer=tokenizer, debug=opts.log) @@ -135,6 +132,9 @@ def printOutput(parser, document, opts): if not kwargs['quote_char']: del kwargs['quote_char'] + if opts.sanitize: + kwargs["sanitize"] = True + tokens = treewalkers.getTreeWalker(opts.treebuilder)(document) if sys.version_info[0] >= 3: encoding = None From 75cf69711f9706b84ce6259f81c6640089e78a5f Mon Sep 17 00:00:00 2001 From: Geoffrey Sneddon Date: Sun, 8 May 2016 19:13:08 +0100 Subject: [PATCH 006/223] Reintroduce the old sanitizer testsuite from html5lib-tests This is imported into this repo as its expectations are very much implementation dependent, with expectations amended to match our actual behaviour. --- html5lib/serializer/htmlserializer.py | 11 +- html5lib/tests/conftest.py | 5 + html5lib/tests/sanitizer-testdata/tests1.dat | 433 +++++++++++++++++++ html5lib/tests/sanitizer.py | 50 +++ 4 files changed, 494 insertions(+), 5 deletions(-) create mode 100644 html5lib/tests/sanitizer-testdata/tests1.dat create mode 100644 html5lib/tests/sanitizer.py diff --git a/html5lib/serializer/htmlserializer.py b/html5lib/serializer/htmlserializer.py index afe2e0e2..23f6befe 100644 --- a/html5lib/serializer/htmlserializer.py +++ b/html5lib/serializer/htmlserializer.py @@ -184,6 +184,12 @@ def serialize(self, treewalker, encoding=None): if encoding and self.inject_meta_charset: from ..filters.inject_meta_charset import Filter treewalker = Filter(treewalker, encoding) + # Alphabetical attributes is here under the assumption that none of + # the later filters add or change order of attributes; it needs to be + # before the sanitizer so escaped elements come out correctly + if self.alphabetical_attributes: + from ..filters.alphabeticalattributes import Filter + treewalker = Filter(treewalker) # WhitespaceFilter should be used before OptionalTagFilter # for maximum efficiently of this latter filter if self.strip_whitespace: @@ -195,11 +201,6 @@ def serialize(self, treewalker, encoding=None): if self.omit_optional_tags: from ..filters.optionaltags import Filter treewalker = Filter(treewalker) - # Alphabetical attributes must be last, as other filters - # could add attributes and alter the order - if self.alphabetical_attributes: - from ..filters.alphabeticalattributes import Filter - treewalker = Filter(treewalker) for token in treewalker: type = token["type"] diff --git a/html5lib/tests/conftest.py b/html5lib/tests/conftest.py index 811aebbf..dceb94cc 100644 --- a/html5lib/tests/conftest.py +++ b/html5lib/tests/conftest.py @@ -2,11 +2,13 @@ from .tree_construction import TreeConstructionFile from .tokenizer import TokenizerFile +from .sanitizer import SanitizerFile _dir = os.path.abspath(os.path.dirname(__file__)) _testdata = os.path.join(_dir, "testdata") _tree_construction = os.path.join(_testdata, "tree-construction") _tokenizer = os.path.join(_testdata, "tokenizer") +_sanitizer_testdata = os.path.join(_dir, "sanitizer-testdata") def pytest_collectstart(): @@ -24,3 +26,6 @@ def pytest_collect_file(path, parent): elif dir == _tokenizer: if path.ext == ".test": return TokenizerFile(path, parent) + elif dir == _sanitizer_testdata: + if path.ext == ".dat": + return SanitizerFile(path, parent) diff --git a/html5lib/tests/sanitizer-testdata/tests1.dat b/html5lib/tests/sanitizer-testdata/tests1.dat new file mode 100644 index 00000000..74e88336 --- /dev/null +++ b/html5lib/tests/sanitizer-testdata/tests1.dat @@ -0,0 +1,433 @@ +[ + { + "name": "IE_Comments", + "input": "", + "output": "" + }, + + { + "name": "IE_Comments_2", + "input": "", + "output": "<script>alert('XSS');</script>" + }, + + { + "name": "allow_colons_in_path_component", + "input": "foo", + "output": "foo" + }, + + { + "name": "background_attribute", + "input": "
", + "output": "
" + }, + + { + "name": "bgsound", + "input": "", + "output": "<bgsound src=\"javascript:alert('XSS');\"></bgsound>" + }, + + { + "name": "div_background_image_unicode_encoded", + "input": "
foo
", + "output": "
foo
" + }, + + { + "name": "div_expression", + "input": "
foo
", + "output": "
foo
" + }, + + { + "name": "double_open_angle_brackets", + "input": "", + "output": "" + }, + + { + "name": "img_dynsrc_lowsrc", + "input": "", + "output": "" + }, + + { + "name": "img_vbscript", + "input": "", + "output": "" + }, + + { + "name": "input_image", + "input": "", + "output": "" + }, + + { + "name": "link_stylesheets", + "input": "", + "output": "<link href=\"javascript:alert('XSS');\" rel=\"stylesheet\">" + }, + + { + "name": "link_stylesheets_2", + "input": "", + "output": "<link href=\"http://ha.ckers.org/xss.css\" rel=\"stylesheet\">" + }, + + { + "name": "list_style_image", + "input": "
  • foo
    • ", + "output": "
  • foo
    • " + }, + + { + "name": "no_closing_script_tags", + "input": "", + "output": "<script src=\"http://ha.ckers.org/xss.js\" xss=\"\"></script>" + }, + + { + "name": "non_alpha_non_digit_2", + "input": "foo", + "output": "foo" + }, + + { + "name": "non_alpha_non_digit_3", + "input": "", + "output": "" + }, + + { + "name": "non_alpha_non_digit_II", + "input": "foo", + "output": "foo" + }, + + { + "name": "non_alpha_non_digit_III", + "input": "foo", + "output": "foo" + }, + + { + "name": "platypus", + "input": "never trust your upstream platypus", + "output": "never trust your upstream platypus" + }, + + { + "name": "protocol_resolution_in_script_tag", + "input": "", + "output": "<script src=\"//ha.ckers.org/.j\"></script>" + }, + + { + "name": "should_allow_anchors", + "input": "", + "output": "<script>baz</script>" + }, + + { + "name": "should_allow_image_alt_attribute", + "input": "foo", + "output": "foo" + }, + + { + "name": "should_allow_image_height_attribute", + "input": "", + "output": "" + }, + + { + "name": "should_allow_image_src_attribute", + "input": "", + "output": "" + }, + + { + "name": "should_allow_image_width_attribute", + "input": "", + "output": "" + }, + + { + "name": "should_handle_blank_text", + "input": "", + "output": "" + }, + + { + "name": "should_handle_malformed_image_tags", + "input": "\">", + "output": "<script>alert(\"XSS\")</script>\">" + }, + + { + "name": "should_handle_non_html", + "input": "abc", + "output": "abc" + }, + + { + "name": "should_not_fall_for_ridiculous_hack", + "input": "", + "output": "" + }, + + { + "name": "should_not_fall_for_xss_image_hack_0", + "input": "", + "output": "" + }, + + { + "name": "should_not_fall_for_xss_image_hack_1", + "input": "", + "output": "" + }, + + { + "name": "should_not_fall_for_xss_image_hack_10", + "input": "", + "output": "" + }, + + { + "name": "should_not_fall_for_xss_image_hack_11", + "input": "", + "output": "" + }, + + { + "name": "should_not_fall_for_xss_image_hack_12", + "input": "", + "output": "" + }, + + { + "name": "should_not_fall_for_xss_image_hack_13", + "input": "", + "output": "" + }, + + { + "name": "should_not_fall_for_xss_image_hack_14", + "input": "", + "output": "" + }, + + { + "name": "should_not_fall_for_xss_image_hack_2", + "input": "", + "output": "" + }, + + { + "name": "should_not_fall_for_xss_image_hack_3", + "input": "", + "output": "" + }, + + { + "name": "should_not_fall_for_xss_image_hack_4", + "input": "", + "output": "" + }, + + { + "name": "should_not_fall_for_xss_image_hack_5", + "input": "", + "output": "" + }, + + { + "name": "should_not_fall_for_xss_image_hack_6", + "input": "", + "output": "" + }, + + { + "name": "should_not_fall_for_xss_image_hack_7", + "input": "", + "output": "" + }, + + { + "name": "should_not_fall_for_xss_image_hack_8", + "input": "", + "output": "" + }, + + { + "name": "should_not_fall_for_xss_image_hack_9", + "input": "", + "output": "" + }, + + { + "name": "should_sanitize_half_open_scripts", + "input": "", + "output": "<script src=\"http://ha.ckers.org/xss.js\" xss=\"\"></script>" + }, + + { + "name": "should_sanitize_script_tag_with_multiple_open_brackets", + "input": "<", + "output": "<<script>alert(\"XSS\");//<</script>" + }, + + { + "name": "should_sanitize_script_tag_with_multiple_open_brackets_2", + "input": "