Enable the use of Camo to remove mixed content warnings

docs/configuration.rst

@@ -117,6 +117,25 @@ search.hosts
     This is a list of elasticsearch hosts that Warehouse should attempt to use.
     Each list entry should be a dictionary with a ``host`` and ``port`` key.
 
+camo.url
+~~~~~~~~
+
+:Type: URL
+:Default: ``None``
+:Required: No
+:Descritpion:
+    The base url of the camo instance. This *must* end with a trailing slash.
+
+camo.key
+~~~~~~~~
+
+:Type: String
+:Default: ``None``
+:Required: No
+:Description:
+    The secret key used by camo to identify that the URL was generated by an
+    approved application.
+
 logging
 ~~~~~~~
 
@@ -159,6 +178,10 @@ Example Configuration
             - host: 127.0.0.1
               port: 9200
 
+    camo:
+        url: https://camo.example.com/
+        key: asecretkey
+
     logging:
         version: 1
         formatters:

setup.cfg

@@ -14,6 +14,7 @@ requires-dist =
     elasticsearch
     enum34
     guard
+    html5lib
     Jinja2
     PyYAML
     raven

setup.py

@@ -43,6 +43,7 @@ def recursive_glob(path, pattern, cutdirs=0):
     "elasticsearch",
     "enum34",
     "guard",
+    "html5lib",
     "Jinja2",
     "PyYAML",
     "raven",

tests/packaging/test_views.py

@@ -157,7 +157,7 @@ def test_project_detail_invalid_version():
     ]
 
 
-@pytest.mark.parametrize(("version", "description"), [
+@pytest.mark.parametrize(("version", "description", "camo"), [
     (
         None,
         textwrap.dedent("""
@@ -166,6 +166,31 @@ def test_project_detail_invalid_version():
 
             This is a test project
         """),
+        None,
+    ),
+    (
+        "1.0",
+        textwrap.dedent("""
+            Test Project
+            ============
+
+            This is a test project
+        """),
+        None,
+    ),
+    (None, ".. code-fail::\n    wat", None),
+    ("1.0", ".. code-fail::\n    wat", None),
+    (None, None, None),
+    ("1.0", None, None),
+    (
+        None,
+        textwrap.dedent("""
+            Test Project
+            ============
+
+            This is a test project
+        """),
+        pretend.stub(url="https://camo.example.com/", key="secret key"),
     ),
     (
         "1.0",
@@ -175,13 +200,30 @@ def test_project_detail_invalid_version():
 
             This is a test project
         """),
+        pretend.stub(url="https://camo.example.com/", key="secret key"),
+    ),
+    (
+        None,
+        ".. code-fail::\n    wat",
+        pretend.stub(url="https://camo.example.com/", key="secret key"),
+    ),
+    (
+        "1.0",
+        ".. code-fail::\n    wat",
+        pretend.stub(url="https://camo.example.com/", key="secret key"),
+    ),
+    (
+        None,
+        None,
+        pretend.stub(url="https://camo.example.com/", key="secret key"),
+    ),
+    (
+        "1.0",
+        None,
+        pretend.stub(url="https://camo.example.com/", key="secret key"),
     ),
-    (None, ".. code-fail::\n    wat"),
-    ("1.0", ".. code-fail::\n    wat"),
-    (None, None),
-    ("1.0", None),
 ])
-def test_project_detail_valid(version, description):
+def test_project_detail_valid(version, description, camo):
     release = {
         "description": description,
     }
@@ -196,6 +238,7 @@ def test_project_detail_valid(version, description):
                 browser=False,
                 varnish=False,
             ),
+            camo=camo,
         ),
         db=pretend.stub(
             packaging=pretend.stub(

tests/test_application.py

@@ -44,6 +44,7 @@ def test_basic_instantiation():
             "index": "warehouse",
             "hosts": [],
         },
+        "camo": None,
         "logging": {
             "version": 1,
         },
@@ -219,3 +220,24 @@ def test_guard_middleware(monkeypatch):
     )
 
     assert ContentSecurityPolicy.calls == [pretend.call(mock.ANY, mock.ANY)]
+
+
+def test_camo_settings(monkeypatch):
+    ContentSecurityPolicy = pretend.call_recorder(lambda app, policy: app)
+
+    monkeypatch.setattr(guard, "ContentSecurityPolicy", ContentSecurityPolicy)
+
+    Warehouse.from_yaml(
+        os.path.abspath(os.path.join(
+            os.path.dirname(__file__),
+            "test_config.yml",
+        )),
+        override={"camo": {"url": "https://camo.example.com/", "key": "skey"}},
+    )
+
+    assert ContentSecurityPolicy.calls == [pretend.call(mock.ANY, mock.ANY)]
+    assert set(ContentSecurityPolicy.calls[0].args[1]["img-src"]) == {
+        "'self'",
+        "https://camo.example.com",
+        "https://secure.gravatar.com",
+    }

tests/test_utils.py

@@ -22,7 +22,8 @@
 from warehouse.utils import (
     AttributeDict, FastlyFormatter, convert_to_attr_dict, merge_dict,
     render_response, cache, get_wsgi_application, get_mimetype, redirect,
-    SearchPagination, is_valid_json_callback_name,
+    SearchPagination, is_valid_json_callback_name, generate_camouflage_url,
+    camouflage_images,
 )
 
 
@@ -227,3 +228,35 @@ def test_next_url(self):
 ])
 def test_is_valid_json_callback_name(callback, expected):
     assert is_valid_json_callback_name(callback) == expected
+
+
+@pytest.mark.parametrize(("camo_url", "camo_key", "url", "expected"), [
+    (
+        "https://camo.example.com/",
+        "123",
+        "https://example.com/fake.png",
+        "https://camo.example.com/dec25c03d21dc84f233f39c6107d305120746ca0/"
+        "68747470733a2f2f6578616d706c652e636f6d2f66616b652e706e67",
+    )
+])
+def test_generate_camouflage_url(camo_url, camo_key, url, expected):
+    assert generate_camouflage_url(camo_url, camo_key, url) == expected
+
+
+@pytest.mark.parametrize(("camo_url", "camo_key", "html", "expected"), [
+    (
+        "https://camo.example.com/",
+        "123",
+        '<html><body><img src="http://example.com/fake.png"></body></html>',
+        '<img src=https://camo.example.com/d59e450f25b4dad6ef4bc4bd71fef1f10d1'
+        '74273/687474703a2f2f6578616d706c652e636f6d2f66616b652e706e67>',
+    ),
+    (
+        "https://camo.example.com/",
+        "123",
+        '<html><body><img alt="whatever"></body></html>',
+        "<img alt=whatever>",
+    ),
+])
+def test_camouflage_images(camo_url, camo_key, html, expected):
+    assert camouflage_images(camo_url, camo_key, html) == expected

warehouse/application.py

@@ -35,6 +35,7 @@
 
 from raven import Client
 from raven.middleware import Sentry
+from six.moves import urllib_parse
 from werkzeug.exceptions import HTTPException
 from werkzeug.wsgi import SharedDataMiddleware, responder
 
@@ -135,12 +136,22 @@ def __init__(self, config, engine=None, redis=None):
         ))
 
         # Add our Content Security Policy Middleware
+        img_src = ["'self'"]
+        if self.config.camo:
+            camo_parsed = urllib_parse.urlparse(self.config.camo.url)
+            img_src += [
+                "{}://{}".format(camo_parsed.scheme, camo_parsed.netloc),
+                "https://secure.gravatar.com",
+            ]
+        else:
+            img_src += ["*"]
+
         self.wsgi_app = guard.ContentSecurityPolicy(
             self.wsgi_app,
             {
                 "default-src": ["'self'"],
                 "font-src": ["'self'", "data:"],
-                "img-src": ["*"],
+                "img-src": img_src,
                 "style-src": ["'self'", "cloud.typography.com"],
             },
         )

warehouse/config.yml

@@ -7,6 +7,8 @@ site:
 search:
     index: warehouse
 
+camo: null
+
 paths:
     documentation: data/packagedocs
 

warehouse/packaging/views.py

@@ -20,7 +20,9 @@
 from werkzeug.exceptions import NotFound
 
 from warehouse.helpers import url_for
-from warehouse.utils import cache, fastly, redirect, render_response
+from warehouse.utils import (
+    cache, fastly, redirect, render_response, camouflage_images,
+)
 
 
 @cache(browser=1, varnish=120)
@@ -77,6 +79,13 @@ def project_detail(app, request, project_name, version=None):
     if release.get("description"):
         # Render the project description
         description_html = readme.rst.render(release["description"])
+
+        if app.config.camo:
+            description_html = camouflage_images(
+                app.config.camo.url,
+                app.config.camo.key,
+                description_html,
+            )
     else:
         description_html = ""
 

warehouse/utils.py

@@ -14,12 +14,19 @@
 from __future__ import absolute_import, division, print_function
 from __future__ import unicode_literals
 
+import binascii
 import collections
 import functools
+import hashlib
+import hmac
 import mimetypes
 import re
 import string
 
+import html5lib
+import html5lib.serializer
+import html5lib.treewalkers
+
 from werkzeug.urls import iri_to_uri
 from werkzeug.utils import escape
 
@@ -250,3 +257,37 @@ def is_valid_json_callback_name(callback_name):
         return False
 
     return True
+
+
+def generate_camouflage_url(camo_url, camo_key, url):
+    digest = hmac.new(
+        camo_key.encode("utf8"),
+        url.encode("utf8"),
+        digestmod=hashlib.sha1,
+    ).hexdigest()
+    return "".join([
+        camo_url,
+        "/".join([
+            digest,
+            binascii.hexlify(url.encode("utf8")).decode("utf8")
+        ]),
+    ])
+
+
+def camouflage_images(camo_url, camo_key, html):
+    # Parse SRC as HTML.
+    tree_builder = html5lib.treebuilders.getTreeBuilder("dom")
+    parser = html5lib.html5parser.HTMLParser(tree=tree_builder)
+    dom = parser.parse(html)
+
+    for e in dom.getElementsByTagName("img"):
+        u = e.getAttribute("src")
+        if u:
+            e.setAttribute(
+                "src",
+                generate_camouflage_url(camo_url, camo_key, u),
+            )
+
+    tree_walker = html5lib.treewalkers.getTreeWalker("dom")
+    html_serializer = html5lib.serializer.htmlserializer.HTMLSerializer()
+    return "".join(html_serializer.serialize(tree_walker(dom)))