Enable Warehouse to run on Python 3.3+, Python 2.7, and PyPy

.coveragerc

@@ -24,3 +24,6 @@ omit =
     # Migrations don't make sense to include in coverage
     warehouse/migrations/versions/*
     warehouse/migrations/env.py
+
+    # The compat module contains things that are only run on one Python version
+    warehouse/compat.py

.travis.yml

@@ -1,5 +1,5 @@
 language: python
-python: 2.7
+python: 3.3
 
 addons:
   postgresql: 9.3
@@ -9,8 +9,10 @@ env:
         - WAREHOUSE_DATABASE_URL=postgresql://postgres@localhost/warehouse
     matrix:
         - TOXENV=py27
+        - TOXENV=py33
         - TOXENV=pypy
         - TOXENV=pep8
+        - TOXENV=py2pep8
         - TOXENV=docs
         - TOXENV=packaging
 

setup.cfg

@@ -5,3 +5,24 @@ universal = 1
 ignore =
     .travis.yml
     tasks.py
+
+[metadata]
+requires-dist =
+    alembic
+    arrow
+    babel
+    elasticsearch
+    enum34
+    guard
+    Jinja2
+    PyYAML
+    raven
+    readme>=0.1.1
+    redis
+    six
+    SQLAlchemy
+    sqlalchemy-citext>=1.2.0
+    Werkzeug
+
+    psycopg2; platform_python_implementation == 'CPython'
+    psycopg2cffi; platform_python_implementation == 'PyPy'

setup.py

@@ -16,6 +16,7 @@
 
 import fnmatch
 import os
+import platform
 
 from setuptools import setup, find_packages
 
@@ -34,6 +35,30 @@ def recursive_glob(path, pattern, cutdirs=0):
             matches.append(filepath)
     return matches
 
+# Note: Adjusting these requires adjusting setup.cfg as well
+install_requires = [
+    "alembic",
+    "arrow",
+    "babel",
+    "elasticsearch",
+    "enum34",
+    "guard",
+    "Jinja2",
+    "PyYAML",
+    "raven",
+    "readme>=0.1.1",
+    "redis",
+    "six",
+    "SQLAlchemy",
+    "sqlalchemy-citext>=1.2.0",
+    "Werkzeug",
+]
+
+if platform.python_implementation() == "PyPy":
+    install_requires += ["psycopg2cffi"]
+else:
+    install_requires += ["psycopg2"]
+
 
 setup(
     name=about["__title__"],
@@ -64,23 +89,7 @@ def recursive_glob(path, pattern, cutdirs=0):
         "warehouse.migrations": ["*.mako", "versions/*.py"],
     },
 
-    install_requires=[
-        "alembic",
-        "arrow",
-        "babel",
-        "elasticsearch",
-        "enum34",
-        "guard",
-        "Jinja2",
-        "psycopg2cffi",
-        "PyYAML",
-        "raven",
-        "readme>=0.1.1",
-        "redis",
-        "SQLAlchemy",
-        "sqlalchemy-citext>=1.2.0",
-        "Werkzeug",
-    ],
+    install_requires=install_requires,
 
     entry_points={
         "console_scripts": [

tests/legacy/test_pypi.py

@@ -88,7 +88,7 @@ def test_daytime(monkeypatch):
 
     resp = pypi.daytime(app, request)
 
-    assert resp.response[0] == '19700101T00:00:00\n'
+    assert resp.response[0] == b'19700101T00:00:00\n'
 
 
 @pytest.mark.parametrize("callback", [None, 'yes'])
@@ -130,7 +130,7 @@ def test_json(monkeypatch, callback):
         '"upload_time": "1970-01-01T00:00:00"}]}'
     if callback:
         expected = '/**/ %s(%s);' % (callback, expected)
-    assert resp.data == expected
+    assert resp.data == expected.encode("utf8")
 
 
 def test_jsonp_invalid():
@@ -208,7 +208,7 @@ def test_rss(monkeypatch):
             'summary': u'hai spam v2',
             'created': u'now',
         }]
-    assert resp.data == "<xml>dummy</xml>"
+    assert resp.data == b"<xml>dummy</xml>"
 
 
 def test_packages_rss(monkeypatch):
@@ -261,7 +261,7 @@ def test_packages_rss(monkeypatch):
             'summary': u'hai eggs!',
             'created': u'now',
         }]
-    assert resp.data == "<xml>dummy</xml>"
+    assert resp.data == b"<xml>dummy</xml>"
 
 
 def test_rss_xml_template(monkeypatch):

tests/legacy/test_xmlrpc.py

@@ -56,7 +56,7 @@ def test_xmlrpc_handler(monkeypatch):
     assert interface.list_packages.calls == [pretend.call()]
 
     response_xml = Response.calls[0].args[0]
-    assert response_xml == '''<?xml version='1.0'?>
+    assert response_xml == b'''<?xml version='1.0'?>
 <methodResponse>
 <params>
 <param>

tests/test_helpers.py

@@ -40,7 +40,7 @@
         {},
         ("https://secure.gravatar.com/avatar/d41d8cd98f00b204e9800998ecf8427e"
          "?size=80"),
-    )
+    ),
 ])
 def test_gravatar_url(email, kwargs, expected):
     assert gravatar_url(email, **kwargs) == expected

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py27,pypy,pep8,docs,packaging
+envlist = py27,py33,pypy,py2pep8,pep8,docs,packaging
 
 [testenv]
 deps =
@@ -26,7 +26,13 @@ deps = check-manifest
 commands =
     check-manifest
 
+[testenv:py2pep8]
+basepython=python2.7
+deps = flake8
+commands = flake8 .
+
 [testenv:pep8]
+basepython=python3.3
 deps = flake8
 commands = flake8 .
 

warehouse/__init__.py

@@ -14,17 +14,15 @@
 from __future__ import absolute_import, division, print_function
 from __future__ import unicode_literals
 
-import psycopg2cffi.compat
-
 from warehouse.__about__ import (
     __title__, __summary__, __uri__, __version__, __author__, __email__,
     __license__, __copyright__, __build__,
 )
+from warehouse.compat import psycopg2_register
 
 __all__ = [
     "__title__", "__summary__", "__uri__", "__version__", "__author__",
     "__email__", "__license__", "__copyright__", "__build__",
 ]
 
-
-psycopg2cffi.compat.register()
+psycopg2_register()

warehouse/compat.py

@@ -0,0 +1,30 @@
+# Copyright 2013 Donald Stufft
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import absolute_import, division, print_function
+from __future__ import unicode_literals
+
+# flake8: noqa
+
+import platform
+
+try:
+    from xmlrpc.server import SimpleXMLRPCDispatcher
+except ImportError:
+    from SimpleXMLRPCServer import SimpleXMLRPCDispatcher
+
+
+def psycopg2_register():
+    if platform.python_implementation() == "PyPy":
+        import psycopg2cffi.compat
+        psycopg2cffi.compat.register()

warehouse/helpers.py

@@ -17,8 +17,8 @@
 import hashlib
 import json
 import os.path
-import urllib
-import urlparse
+
+from six.moves import urllib_parse
 
 import warehouse
 
@@ -35,14 +35,14 @@ def gravatar_url(email, size=80):
     if email is None:
         email = ""
 
-    email_hash = hashlib.md5(email.strip().lower()).hexdigest()
+    email_hash = hashlib.md5(email.strip().lower().encode("utf8")).hexdigest()
 
     url = "https://secure.gravatar.com/avatar/{}".format(email_hash)
     params = {
         "size": size,
     }
 
-    return "?".join([url, urllib.urlencode(params)])
+    return "?".join([url, urllib_parse.urlencode(params)])
 
 
 def static_url(app, filename):
@@ -70,4 +70,4 @@ def static_url(app, filename):
         if basename is not None:
             filename = os.path.join(os.path.dirname(filename), basename)
 
-    return urlparse.urljoin("/static/", filename)
+    return urllib_parse.urljoin("/static/", filename)

warehouse/legacy/xmlrpc.py

@@ -14,11 +14,10 @@
 from __future__ import absolute_import, division, print_function
 from __future__ import unicode_literals
 
-from SimpleXMLRPCServer import SimpleXMLRPCDispatcher
-
 import arrow
 from werkzeug.exceptions import BadRequest
 
+from warehouse.compat import SimpleXMLRPCDispatcher
 from warehouse.http import Response
 
 

warehouse/packaging/db.py

@@ -16,9 +16,10 @@
 
 import datetime
 import os.path
-import urlparse
 import logging
 
+from six.moves import urllib_parse
+
 from warehouse import db
 from warehouse.packaging.tables import ReleaseDependencyKind
 
@@ -199,7 +200,7 @@ def get_top_projects(self, num=None):
         """,
         lambda r: {
             "filename": r["filename"],
-            "url": urlparse.urljoin(
+            "url": urllib_parse.urljoin(
                 "/".join([
                     "../../packages",
                     r["python_version"],
@@ -477,7 +478,7 @@ def search_by_classifier(self, selected_classifiers):
         releases = []
         with self.engine.connect() as conn:
             for name, version in conn.execute(query):
-                releases.append((name.decode('utf-8'), version))
+                releases.append((name, version))
 
         return releases
 
@@ -488,7 +489,7 @@ def get_documentation_url(self, project):
             "index.html",
         ]
         if os.path.exists(os.path.join(*path_parts)):
-            return urlparse.urljoin(
+            return urllib_parse.urljoin(
                 self.app.config.urls.documentation,
                 project
             ) + "/"

warehouse/search/indexes.py

@@ -45,7 +45,7 @@ def reindex(self, alias=True, keep_old=False):
         # Generate an Index Name for Warehouse
         index = "".join([
             self._index,
-            binascii.hexlify(os.urandom(4)),
+            binascii.hexlify(os.urandom(4)).decode("ascii"),
         ])
 
         # Create this index
@@ -67,7 +67,7 @@ def reindex(self, alias=True, keep_old=False):
     def update_alias(self, alias, index, keep_old=False):
         # Get the old index from ElasticSearch
         try:
-            old_index = self.es.indices.get_alias(self._index).keys()[0]
+            old_index = list(self.es.indices.get_alias(self._index))[0]
         except TransportError as exc:
             if not exc.status_code == 404:
                 raise