From ef2f1bf2b836a3a3793d03044b7c8301a26165cb Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Sun, 2 Mar 2014 22:07:15 -0500 Subject: [PATCH 1/3] Switch Warehouse to be a Python 3.3+ Project --- .travis.yml | 5 ++--- setup.py | 2 +- tests/legacy/test_pypi.py | 8 ++++---- tests/legacy/test_xmlrpc.py | 2 +- tests/test_helpers.py | 2 +- tox.ini | 2 +- warehouse/__init__.py | 5 ----- warehouse/helpers.py | 8 ++++---- warehouse/legacy/xmlrpc.py | 2 +- warehouse/packaging/db.py | 8 ++++---- warehouse/search/indexes.py | 4 ++-- 11 files changed, 21 insertions(+), 27 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3cc7451e8273..31ceee146a18 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ language: python -python: 2.7 +python: 3.3 addons: postgresql: 9.3 @@ -8,8 +8,7 @@ env: global: - WAREHOUSE_DATABASE_URL=postgresql://postgres@localhost/warehouse matrix: - - TOXENV=py27 - - TOXENV=pypy + - TOXENV=py33 - TOXENV=pep8 - TOXENV=docs - TOXENV=packaging diff --git a/setup.py b/setup.py index ef172d4f19d7..a9f5a76b1e4f 100644 --- a/setup.py +++ b/setup.py @@ -72,7 +72,7 @@ def recursive_glob(path, pattern, cutdirs=0): "enum34", "guard", "Jinja2", - "psycopg2cffi", + "psycopg2", "PyYAML", "raven", "readme>=0.1.1", diff --git a/tests/legacy/test_pypi.py b/tests/legacy/test_pypi.py index 37587d0d9b22..09ad4dab0297 100644 --- a/tests/legacy/test_pypi.py +++ b/tests/legacy/test_pypi.py @@ -88,7 +88,7 @@ def test_daytime(monkeypatch): resp = pypi.daytime(app, request) - assert resp.response[0] == '19700101T00:00:00\n' + assert resp.response[0] == b'19700101T00:00:00\n' @pytest.mark.parametrize("callback", [None, 'yes']) @@ -130,7 +130,7 @@ def test_json(monkeypatch, callback): '"upload_time": "1970-01-01T00:00:00"}]}' if callback: expected = '/**/ %s(%s);' % (callback, expected) - assert resp.data == expected + assert resp.data == expected.encode("utf8") def test_jsonp_invalid(): @@ -208,7 +208,7 @@ def test_rss(monkeypatch): 'summary': u'hai spam v2', 'created': u'now', }] - assert resp.data == "dummy" + assert resp.data == b"dummy" def test_packages_rss(monkeypatch): @@ -261,7 +261,7 @@ def test_packages_rss(monkeypatch): 'summary': u'hai eggs!', 'created': u'now', }] - assert resp.data == "dummy" + assert resp.data == b"dummy" def test_rss_xml_template(monkeypatch): diff --git a/tests/legacy/test_xmlrpc.py b/tests/legacy/test_xmlrpc.py index 9d418dfcec23..c12c2958cfb5 100644 --- a/tests/legacy/test_xmlrpc.py +++ b/tests/legacy/test_xmlrpc.py @@ -56,7 +56,7 @@ def test_xmlrpc_handler(monkeypatch): assert interface.list_packages.calls == [pretend.call()] response_xml = Response.calls[0].args[0] - assert response_xml == ''' + assert response_xml == b''' diff --git a/tests/test_helpers.py b/tests/test_helpers.py index 34f1737a8a39..fea17888dda9 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -40,7 +40,7 @@ {}, ("https://secure.gravatar.com/avatar/d41d8cd98f00b204e9800998ecf8427e" "?size=80"), - ) + ), ]) def test_gravatar_url(email, kwargs, expected): assert gravatar_url(email, **kwargs) == expected diff --git a/tox.ini b/tox.ini index a9c9b84a1b56..2a94b7bdcd79 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27,pypy,pep8,docs,packaging +envlist = py33,pep8,docs,packaging [testenv] deps = diff --git a/warehouse/__init__.py b/warehouse/__init__.py index a69a9f0dbce8..6ebc4fb29bb3 100644 --- a/warehouse/__init__.py +++ b/warehouse/__init__.py @@ -14,8 +14,6 @@ from __future__ import absolute_import, division, print_function from __future__ import unicode_literals -import psycopg2cffi.compat - from warehouse.__about__ import ( __title__, __summary__, __uri__, __version__, __author__, __email__, __license__, __copyright__, __build__, @@ -25,6 +23,3 @@ "__title__", "__summary__", "__uri__", "__version__", "__author__", "__email__", "__license__", "__copyright__", "__build__", ] - - -psycopg2cffi.compat.register() diff --git a/warehouse/helpers.py b/warehouse/helpers.py index e2c48580d690..a9570634d5dc 100644 --- a/warehouse/helpers.py +++ b/warehouse/helpers.py @@ -18,7 +18,7 @@ import json import os.path import urllib -import urlparse +import urllib.parse import warehouse @@ -35,14 +35,14 @@ def gravatar_url(email, size=80): if email is None: email = "" - email_hash = hashlib.md5(email.strip().lower()).hexdigest() + email_hash = hashlib.md5(email.strip().lower().encode("utf8")).hexdigest() url = "https://secure.gravatar.com/avatar/{}".format(email_hash) params = { "size": size, } - return "?".join([url, urllib.urlencode(params)]) + return "?".join([url, urllib.parse.urlencode(params)]) def static_url(app, filename): @@ -70,4 +70,4 @@ def static_url(app, filename): if basename is not None: filename = os.path.join(os.path.dirname(filename), basename) - return urlparse.urljoin("/static/", filename) + return urllib.parse.urljoin("/static/", filename) diff --git a/warehouse/legacy/xmlrpc.py b/warehouse/legacy/xmlrpc.py index 31185bbdd417..38a0c12f964e 100644 --- a/warehouse/legacy/xmlrpc.py +++ b/warehouse/legacy/xmlrpc.py @@ -14,7 +14,7 @@ from __future__ import absolute_import, division, print_function from __future__ import unicode_literals -from SimpleXMLRPCServer import SimpleXMLRPCDispatcher +from xmlrpc.server import SimpleXMLRPCDispatcher import arrow from werkzeug.exceptions import BadRequest diff --git a/warehouse/packaging/db.py b/warehouse/packaging/db.py index 7c723edfcec0..31c1ebb33b3d 100644 --- a/warehouse/packaging/db.py +++ b/warehouse/packaging/db.py @@ -16,7 +16,7 @@ import datetime import os.path -import urlparse +import urllib.parse import logging from warehouse import db @@ -199,7 +199,7 @@ def get_top_projects(self, num=None): """, lambda r: { "filename": r["filename"], - "url": urlparse.urljoin( + "url": urllib.parse.urljoin( "/".join([ "../../packages", r["python_version"], @@ -477,7 +477,7 @@ def search_by_classifier(self, selected_classifiers): releases = [] with self.engine.connect() as conn: for name, version in conn.execute(query): - releases.append((name.decode('utf-8'), version)) + releases.append((name, version)) return releases @@ -488,7 +488,7 @@ def get_documentation_url(self, project): "index.html", ] if os.path.exists(os.path.join(*path_parts)): - return urlparse.urljoin( + return urllib.parse.urljoin( self.app.config.urls.documentation, project ) + "/" diff --git a/warehouse/search/indexes.py b/warehouse/search/indexes.py index d69e0cd27a57..2816114d68c9 100644 --- a/warehouse/search/indexes.py +++ b/warehouse/search/indexes.py @@ -45,7 +45,7 @@ def reindex(self, alias=True, keep_old=False): # Generate an Index Name for Warehouse index = "".join([ self._index, - binascii.hexlify(os.urandom(4)), + binascii.hexlify(os.urandom(4)).decode("ascii"), ]) # Create this index @@ -67,7 +67,7 @@ def reindex(self, alias=True, keep_old=False): def update_alias(self, alias, index, keep_old=False): # Get the old index from ElasticSearch try: - old_index = self.es.indices.get_alias(self._index).keys()[0] + old_index = list(self.es.indices.get_alias(self._index))[0] except TransportError as exc: if not exc.status_code == 404: raise From dbd7f94cb17d3eaa8214ffd1af3adcc09aba2ca7 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Sun, 2 Mar 2014 22:26:24 -0500 Subject: [PATCH 2/3] Restore compatability with CPython2.7 --- .coveragerc | 3 +++ .travis.yml | 2 ++ setup.py | 1 + tox.ini | 8 +++++++- warehouse/compat.py | 22 ++++++++++++++++++++++ warehouse/helpers.py | 8 ++++---- warehouse/legacy/xmlrpc.py | 3 +-- warehouse/packaging/db.py | 7 ++++--- 8 files changed, 44 insertions(+), 10 deletions(-) create mode 100644 warehouse/compat.py diff --git a/.coveragerc b/.coveragerc index 8bee424f8c4e..baab0e2bedf1 100644 --- a/.coveragerc +++ b/.coveragerc @@ -24,3 +24,6 @@ omit = # Migrations don't make sense to include in coverage warehouse/migrations/versions/* warehouse/migrations/env.py + + # The compat module contains things that are only run on one Python version + warehouse/compat.py diff --git a/.travis.yml b/.travis.yml index 31ceee146a18..50c8a6c1809f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,8 +8,10 @@ env: global: - WAREHOUSE_DATABASE_URL=postgresql://postgres@localhost/warehouse matrix: + - TOXENV=py27 - TOXENV=py33 - TOXENV=pep8 + - TOXENV=py2pep8 - TOXENV=docs - TOXENV=packaging diff --git a/setup.py b/setup.py index a9f5a76b1e4f..ead8adb7549b 100644 --- a/setup.py +++ b/setup.py @@ -77,6 +77,7 @@ def recursive_glob(path, pattern, cutdirs=0): "raven", "readme>=0.1.1", "redis", + "six", "SQLAlchemy", "sqlalchemy-citext>=1.2.0", "Werkzeug", diff --git a/tox.ini b/tox.ini index 2a94b7bdcd79..bd045af3554e 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py33,pep8,docs,packaging +envlist = py27,py33,py2pep8,pep8,docs,packaging [testenv] deps = @@ -26,7 +26,13 @@ deps = check-manifest commands = check-manifest +[testenv:py2pep8] +basepython=python2.7 +deps = flake8 +commands = flake8 . + [testenv:pep8] +basepython=python3.3 deps = flake8 commands = flake8 . diff --git a/warehouse/compat.py b/warehouse/compat.py new file mode 100644 index 000000000000..216534dd0be7 --- /dev/null +++ b/warehouse/compat.py @@ -0,0 +1,22 @@ +# Copyright 2013 Donald Stufft +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import absolute_import, division, print_function +from __future__ import unicode_literals + +# flake8: noqa + +try: + from xmlrpc.server import SimpleXMLRPCDispatcher +except ImportError: + from SimpleXMLRPCServer import SimpleXMLRPCDispatcher diff --git a/warehouse/helpers.py b/warehouse/helpers.py index a9570634d5dc..f3b060054978 100644 --- a/warehouse/helpers.py +++ b/warehouse/helpers.py @@ -17,8 +17,8 @@ import hashlib import json import os.path -import urllib -import urllib.parse + +from six.moves import urllib_parse import warehouse @@ -42,7 +42,7 @@ def gravatar_url(email, size=80): "size": size, } - return "?".join([url, urllib.parse.urlencode(params)]) + return "?".join([url, urllib_parse.urlencode(params)]) def static_url(app, filename): @@ -70,4 +70,4 @@ def static_url(app, filename): if basename is not None: filename = os.path.join(os.path.dirname(filename), basename) - return urllib.parse.urljoin("/static/", filename) + return urllib_parse.urljoin("/static/", filename) diff --git a/warehouse/legacy/xmlrpc.py b/warehouse/legacy/xmlrpc.py index 38a0c12f964e..4c85e08848b6 100644 --- a/warehouse/legacy/xmlrpc.py +++ b/warehouse/legacy/xmlrpc.py @@ -14,11 +14,10 @@ from __future__ import absolute_import, division, print_function from __future__ import unicode_literals -from xmlrpc.server import SimpleXMLRPCDispatcher - import arrow from werkzeug.exceptions import BadRequest +from warehouse.compat import SimpleXMLRPCDispatcher from warehouse.http import Response diff --git a/warehouse/packaging/db.py b/warehouse/packaging/db.py index 31c1ebb33b3d..253559a4f383 100644 --- a/warehouse/packaging/db.py +++ b/warehouse/packaging/db.py @@ -16,9 +16,10 @@ import datetime import os.path -import urllib.parse import logging +from six.moves import urllib_parse + from warehouse import db from warehouse.packaging.tables import ReleaseDependencyKind @@ -199,7 +200,7 @@ def get_top_projects(self, num=None): """, lambda r: { "filename": r["filename"], - "url": urllib.parse.urljoin( + "url": urllib_parse.urljoin( "/".join([ "../../packages", r["python_version"], @@ -488,7 +489,7 @@ def get_documentation_url(self, project): "index.html", ] if os.path.exists(os.path.join(*path_parts)): - return urllib.parse.urljoin( + return urllib_parse.urljoin( self.app.config.urls.documentation, project ) + "/" From facded517490f6849f96f775ebd1ce7221411c23 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Sun, 2 Mar 2014 22:50:54 -0500 Subject: [PATCH 3/3] Restore Compatability with PyPy --- .travis.yml | 1 + setup.cfg | 21 +++++++++++++++++++++ setup.py | 44 +++++++++++++++++++++++++------------------ tox.ini | 2 +- warehouse/__init__.py | 3 +++ warehouse/compat.py | 8 ++++++++ 6 files changed, 60 insertions(+), 19 deletions(-) diff --git a/.travis.yml b/.travis.yml index 50c8a6c1809f..eb5c97111286 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,7 @@ env: matrix: - TOXENV=py27 - TOXENV=py33 + - TOXENV=pypy - TOXENV=pep8 - TOXENV=py2pep8 - TOXENV=docs diff --git a/setup.cfg b/setup.cfg index ead0ee9efa77..00e309cefa2e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,3 +5,24 @@ universal = 1 ignore = .travis.yml tasks.py + +[metadata] +requires-dist = + alembic + arrow + babel + elasticsearch + enum34 + guard + Jinja2 + PyYAML + raven + readme>=0.1.1 + redis + six + SQLAlchemy + sqlalchemy-citext>=1.2.0 + Werkzeug + + psycopg2; platform_python_implementation == 'CPython' + psycopg2cffi; platform_python_implementation == 'PyPy' diff --git a/setup.py b/setup.py index ead8adb7549b..5b66c608e1a8 100644 --- a/setup.py +++ b/setup.py @@ -16,6 +16,7 @@ import fnmatch import os +import platform from setuptools import setup, find_packages @@ -34,6 +35,30 @@ def recursive_glob(path, pattern, cutdirs=0): matches.append(filepath) return matches +# Note: Adjusting these requires adjusting setup.cfg as well +install_requires = [ + "alembic", + "arrow", + "babel", + "elasticsearch", + "enum34", + "guard", + "Jinja2", + "PyYAML", + "raven", + "readme>=0.1.1", + "redis", + "six", + "SQLAlchemy", + "sqlalchemy-citext>=1.2.0", + "Werkzeug", +] + +if platform.python_implementation() == "PyPy": + install_requires += ["psycopg2cffi"] +else: + install_requires += ["psycopg2"] + setup( name=about["__title__"], @@ -64,24 +89,7 @@ def recursive_glob(path, pattern, cutdirs=0): "warehouse.migrations": ["*.mako", "versions/*.py"], }, - install_requires=[ - "alembic", - "arrow", - "babel", - "elasticsearch", - "enum34", - "guard", - "Jinja2", - "psycopg2", - "PyYAML", - "raven", - "readme>=0.1.1", - "redis", - "six", - "SQLAlchemy", - "sqlalchemy-citext>=1.2.0", - "Werkzeug", - ], + install_requires=install_requires, entry_points={ "console_scripts": [ diff --git a/tox.ini b/tox.ini index bd045af3554e..fe927625cafb 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27,py33,py2pep8,pep8,docs,packaging +envlist = py27,py33,pypy,py2pep8,pep8,docs,packaging [testenv] deps = diff --git a/warehouse/__init__.py b/warehouse/__init__.py index 6ebc4fb29bb3..12b1dfa47957 100644 --- a/warehouse/__init__.py +++ b/warehouse/__init__.py @@ -18,8 +18,11 @@ __title__, __summary__, __uri__, __version__, __author__, __email__, __license__, __copyright__, __build__, ) +from warehouse.compat import psycopg2_register __all__ = [ "__title__", "__summary__", "__uri__", "__version__", "__author__", "__email__", "__license__", "__copyright__", "__build__", ] + +psycopg2_register() diff --git a/warehouse/compat.py b/warehouse/compat.py index 216534dd0be7..a747cd6ae6bc 100644 --- a/warehouse/compat.py +++ b/warehouse/compat.py @@ -16,7 +16,15 @@ # flake8: noqa +import platform + try: from xmlrpc.server import SimpleXMLRPCDispatcher except ImportError: from SimpleXMLRPCServer import SimpleXMLRPCDispatcher + + +def psycopg2_register(): + if platform.python_implementation() == "PyPy": + import psycopg2cffi.compat + psycopg2cffi.compat.register()