From d1e010cd797fc1f80eaea1959aef45e20118f4f7 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Tue, 7 Feb 2012 22:12:34 -0500 Subject: [PATCH 01/11] add the url we downloaded from to the InstallRequirement, and then record it in info.ini --- pip/req.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/pip/req.py b/pip/req.py index ff423dfa2c9..c9bd56cde0e 100644 --- a/pip/req.py +++ b/pip/req.py @@ -601,8 +601,20 @@ def install(self, install_options, global_options=()): filename += os.path.sep new_lines.append(make_path_relative(filename, egg_info_dir)) f.close() + + # FIXME: Should The Generation Of This File Go Higher? + info = ConfigParser.RawConfigParser() + info.add_section('download') + info.set('download', 'url', self.url.url) + + f = open(os.path.join(egg_info_dir, 'info.ini'), 'w') + info.write(f) + f.close() + + new_lines.append("info.ini") + f = open(os.path.join(egg_info_dir, 'installed-files.txt'), 'w') - f.write('\n'.join(new_lines)+'\n') + f.write('\n'.join(new_lines) + '\n') f.close() finally: if os.path.exists(record_filename): @@ -981,6 +993,7 @@ def prepare_files(self, finder, force_root_egg_info=False, bundle=False): url = Link(req_to_install.url) assert url if url: + req_to_install.url = url try: self.unpack_url(url, location, self.is_download) except HTTPError: From dbfa932929c5af563e6c2f8b35e82580f5d54edc Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Tue, 7 Feb 2012 22:28:31 -0500 Subject: [PATCH 02/11] record the requirements line used to cause this package to install --- pip/req.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pip/req.py b/pip/req.py index c9bd56cde0e..d49c4cfc648 100644 --- a/pip/req.py +++ b/pip/req.py @@ -606,6 +606,7 @@ def install(self, install_options, global_options=()): info = ConfigParser.RawConfigParser() info.add_section('download') info.set('download', 'url', self.url.url) + info.set('download', 'requirement', str(self.req)) f = open(os.path.join(egg_info_dir, 'info.ini'), 'w') info.write(f) From 14502cc6af6a29303fcb5df94fe3bafdac2e1acc Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Tue, 7 Feb 2012 23:00:03 -0500 Subject: [PATCH 03/11] make requirement_line the line that was passed to it and add support for editables --- pip/req.py | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/pip/req.py b/pip/req.py index d49c4cfc648..d9e02c0f1d4 100644 --- a/pip/req.py +++ b/pip/req.py @@ -34,7 +34,7 @@ class InstallRequirement(object): def __init__(self, req, comes_from, source_dir=None, editable=False, - url=None, update=True): + url=None, update=True, requirement_line=None): self.extras = () if isinstance(req, string_types): req = pkg_resources.Requirement.parse(req) @@ -44,6 +44,7 @@ def __init__(self, req, comes_from, source_dir=None, editable=False, self.source_dir = source_dir self.editable = editable self.url = url + self.requirement_line = requirement_line self._egg_info_path = None # This holds the pkg_resources.Distribution object if this requirement # is already available: @@ -62,12 +63,13 @@ def __init__(self, req, comes_from, source_dir=None, editable=False, @classmethod def from_editable(cls, editable_req, comes_from=None, default_vcs=None): + requirement_line = "--editable=%s" % editable_req name, url = parse_editable(editable_req, default_vcs) if url.startswith('file:'): source_dir = url_to_path(url) else: source_dir = None - return cls(name, comes_from, source_dir=source_dir, editable=True, url=url) + return cls(name, comes_from, source_dir=source_dir, editable=True, url=url, requirement_line=requirement_line) @classmethod def from_line(cls, name, comes_from=None): @@ -104,7 +106,7 @@ def from_line(cls, name, comes_from=None): else: req = name - return cls(req, comes_from, url=url) + return cls(req, comes_from, url=url, requirement_line=name) def __str__(self): if self.req: @@ -606,7 +608,8 @@ def install(self, install_options, global_options=()): info = ConfigParser.RawConfigParser() info.add_section('download') info.set('download', 'url', self.url.url) - info.set('download', 'requirement', str(self.req)) + if self.requirement_line: + info.set('download', 'requirement_line', self.requirement_line) f = open(os.path.join(egg_info_dir, 'info.ini'), 'w') info.write(f) @@ -650,6 +653,23 @@ def install_editable(self, install_options, global_options=()): logger.indent -= 2 self.install_succeeded = True + # FIXME: Is there a better way to get the egg-info dir? + egg_info_dir = None + for fname in os.listdir(self.source_dir): + if fname.endswith(".egg-info"): + egg_info_dir = os.path.join(self.source_dir, fname) + + if egg_info_dir: + info = ConfigParser.RawConfigParser() + info.add_section('download') + info.set('download', 'url', self.url) + if self.requirement_line: + info.set('download', 'requirement_line', self.requirement_line) + + f = open(os.path.join(egg_info_dir, 'info.ini'), 'w') + info.write(f) + f.close() + def _filter_install(self, line): level = logger.NOTIFY for regex in [r'^running .*', r'^writing .*', '^creating .*', '^[Cc]opying .*', From 25fcbd95e541e5cf9187e84943956fc2edea08e8 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Tue, 7 Feb 2012 23:19:01 -0500 Subject: [PATCH 04/11] refactor handling of the info.ini file into a method that is passed a dict --- pip/req.py | 53 +++++++++++++++++++++++++++++++++++------------------ 1 file changed, 35 insertions(+), 18 deletions(-) diff --git a/pip/req.py b/pip/req.py index d9e02c0f1d4..8740411e790 100644 --- a/pip/req.py +++ b/pip/req.py @@ -550,6 +550,19 @@ def _clean_zip_name(self, name, prefix): name = name.replace(os.path.sep, '/') return name + def _write_info_ini(self, egg_info_dir, options): + info = ConfigParser.RawConfigParser() + + for section, values in options.iteritems(): + info.add_section(section) + + for key, value in values.iteritems(): + info.set(section, key, value) + + f = open(os.path.join(egg_info_dir, 'info.ini'), 'w') + info.write(f) + f.close() + def install(self, install_options, global_options=()): if self.editable: self.install_editable(install_options, global_options) @@ -604,18 +617,20 @@ def install(self, install_options, global_options=()): new_lines.append(make_path_relative(filename, egg_info_dir)) f.close() - # FIXME: Should The Generation Of This File Go Higher? - info = ConfigParser.RawConfigParser() - info.add_section('download') - info.set('download', 'url', self.url.url) - if self.requirement_line: - info.set('download', 'requirement_line', self.requirement_line) + info_data = { + 'download': { + 'url': self.url.url, + } + } - f = open(os.path.join(egg_info_dir, 'info.ini'), 'w') - info.write(f) - f.close() + if self.requirement_line: + info_data['download']['requirement'] = self.requirement_line + else: + logger.warn("No requirement line recorded for %s" % self.name) + self._write_info_ini(egg_info_dir, info_data) - new_lines.append("info.ini") + # Add the info.ini to the installed-files.txt + new_lines.append('info.ini') f = open(os.path.join(egg_info_dir, 'installed-files.txt'), 'w') f.write('\n'.join(new_lines) + '\n') @@ -660,15 +675,17 @@ def install_editable(self, install_options, global_options=()): egg_info_dir = os.path.join(self.source_dir, fname) if egg_info_dir: - info = ConfigParser.RawConfigParser() - info.add_section('download') - info.set('download', 'url', self.url) - if self.requirement_line: - info.set('download', 'requirement_line', self.requirement_line) + info_data = { + 'download': { + 'url': self.url, + } + } - f = open(os.path.join(egg_info_dir, 'info.ini'), 'w') - info.write(f) - f.close() + if self.requirement_line: + info_data['download']['requirement'] = self.requirement_line + else: + logger.warn("No requirement line recorded for %s" % self.name) + self._write_info_ini(egg_info_dir, info_data) def _filter_install(self, line): level = logger.NOTIFY From 43ea7d05e5505dcfa3c72b55d5085d88f593244c Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Wed, 8 Feb 2012 00:41:59 -0500 Subject: [PATCH 05/11] add tests to test info.ini on different types of installs --- tests/test_info_file.py | 79 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 tests/test_info_file.py diff --git a/tests/test_info_file.py b/tests/test_info_file.py new file mode 100644 index 00000000000..906eff5de46 --- /dev/null +++ b/tests/test_info_file.py @@ -0,0 +1,79 @@ +import os + +from pip.backwardcompat import ConfigParser +from pip.download import path_to_url2 +from tests.test_pip import here, reset_env, run_pip +from tests.path import Path + + +def test_index(): + """ + Test that the info.ini is written and works from an index (PyPI). + """ + env = reset_env() + run_pip('install', '-i http://pypi.python.org/simple/', 'INITools') + + egg_info_dir = None + for x in os.listdir(os.path.join(os.path.dirname(env.venv_path), env.site_packages)): + if x.startswith("INITools-") and x.endswith(".egg-info"): + egg_info_dir = os.path.join(os.path.dirname(env.venv_path), env.site_packages, x) + break + assert egg_info_dir is not None + + infofp = open(os.path.join(os.path.dirname(env.venv_path), env.site_packages, egg_info_dir, "info.ini")) + info = ConfigParser.RawConfigParser() + info.readfp(infofp) + infofp.close() + + assert info.has_section("download") + assert info.get("download", "url").startswith("http://pypi.python.org/packages/source/I/INITools/INITools") + assert info.get("download", "requirement") == "INITools" + + +def test_tarball(): + """ + Test that the info.ini is written and works from an tarball. + """ + env = reset_env() + run_pip('install', 'http://pypi.python.org/packages/source/I/INITools/INITools-0.3.1.tar.gz') + + egg_info_dir = None + for x in os.listdir(os.path.join(os.path.dirname(env.venv_path), env.site_packages)): + if x.startswith("INITools-") and x.endswith(".egg-info"): + egg_info_dir = os.path.join(os.path.dirname(env.venv_path), env.site_packages, x) + break + assert egg_info_dir is not None + + infofp = open(os.path.join(egg_info_dir, "info.ini")) + info = ConfigParser.RawConfigParser() + info.readfp(infofp) + infofp.close() + + assert info.has_section("download") + assert info.get("download", "url") == "http://pypi.python.org/packages/source/I/INITools/INITools-0.3.1.tar.gz" + assert info.get("download", "requirement") == "http://pypi.python.org/packages/source/I/INITools/INITools-0.3.1.tar.gz" + + +def test_editable(): + """ + Test that the info.ini is written and works from an editable. + """ + env = reset_env() + fspkg = path_to_url2(Path(here) / 'packages' / 'FSPkg') + run_pip('install', '-e', fspkg) + + egg_info_dir = None + for x in os.listdir(Path(here) / 'packages' / 'FSPkg'): + if x.startswith("FSPkg") and x.endswith(".egg-info"): + egg_info_dir = os.path.join(Path(here) / 'packages' / 'FSPkg', x) + break + assert egg_info_dir is not None + + infofp = open(os.path.join(egg_info_dir, "info.ini")) + info = ConfigParser.RawConfigParser() + info.readfp(infofp) + infofp.close() + + assert info.has_section("download") + assert info.get("download", "url") == "file:///Users/dstufft/projects/pip/tests/packages/FSPkg" + assert info.get("download", "requirement") == "--editable=file:///Users/dstufft/projects/pip/tests/packages/FSPkg" From 7c187ca9eca040b84c926701b0c37189fda3c19a Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Mon, 12 Mar 2012 14:49:52 -0400 Subject: [PATCH 06/11] change the name from info.ini to pip.ini --- pip/req.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pip/req.py b/pip/req.py index 8740411e790..801a047a4e8 100644 --- a/pip/req.py +++ b/pip/req.py @@ -559,7 +559,7 @@ def _write_info_ini(self, egg_info_dir, options): for key, value in values.iteritems(): info.set(section, key, value) - f = open(os.path.join(egg_info_dir, 'info.ini'), 'w') + f = open(os.path.join(egg_info_dir, 'pip.ini'), 'w') info.write(f) f.close() @@ -629,8 +629,8 @@ def install(self, install_options, global_options=()): logger.warn("No requirement line recorded for %s" % self.name) self._write_info_ini(egg_info_dir, info_data) - # Add the info.ini to the installed-files.txt - new_lines.append('info.ini') + # Add the pip.ini to the installed-files.txt + new_lines.append('pip.ini') f = open(os.path.join(egg_info_dir, 'installed-files.txt'), 'w') f.write('\n'.join(new_lines) + '\n') From a706574ac5c7c960aba6affdfdf227c7d88b5fe4 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Mon, 12 Mar 2012 14:56:53 -0400 Subject: [PATCH 07/11] use the available methods to get the egg info dir --- pip/req.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pip/req.py b/pip/req.py index 801a047a4e8..9a51dd59155 100644 --- a/pip/req.py +++ b/pip/req.py @@ -668,11 +668,7 @@ def install_editable(self, install_options, global_options=()): logger.indent -= 2 self.install_succeeded = True - # FIXME: Is there a better way to get the egg-info dir? - egg_info_dir = None - for fname in os.listdir(self.source_dir): - if fname.endswith(".egg-info"): - egg_info_dir = os.path.join(self.source_dir, fname) + egg_info_dir = os.path.dirname(self.egg_info_path("pip.ini")) if egg_info_dir: info_data = { From efaf1066d2e279d34d19c4b9634484e2a690ea4f Mon Sep 17 00:00:00 2001 From: Erik Simmler Date: Tue, 10 Apr 2012 10:36:12 -0400 Subject: [PATCH 08/11] Cleaned up info_file tests based on discussion in https://github.com/pypa/pip/pull/453 --- tests/test_info_file.py | 44 ++++++++++++++--------------------------- 1 file changed, 15 insertions(+), 29 deletions(-) diff --git a/tests/test_info_file.py b/tests/test_info_file.py index 906eff5de46..f5c63e13fcd 100644 --- a/tests/test_info_file.py +++ b/tests/test_info_file.py @@ -2,49 +2,40 @@ from pip.backwardcompat import ConfigParser from pip.download import path_to_url2 -from tests.test_pip import here, reset_env, run_pip +from tests.test_pip import here, reset_env, run_pip, pyversion + from tests.path import Path def test_index(): """ - Test that the info.ini is written and works from an index (PyPI). + Test that the pip.ini is written and works from an index (PyPI). """ env = reset_env() - run_pip('install', '-i http://pypi.python.org/simple/', 'INITools') + run_pip('install', '-i http://pypi.python.org/simple/', 'INITools==0.3.1') - egg_info_dir = None - for x in os.listdir(os.path.join(os.path.dirname(env.venv_path), env.site_packages)): - if x.startswith("INITools-") and x.endswith(".egg-info"): - egg_info_dir = os.path.join(os.path.dirname(env.venv_path), env.site_packages, x) - break - assert egg_info_dir is not None + egg_info_dir = env.base_path / env.site_packages / 'INITools-0.3.1-py%s.egg-info' % pyversion - infofp = open(os.path.join(os.path.dirname(env.venv_path), env.site_packages, egg_info_dir, "info.ini")) + infofp = open(os.path.join(egg_info_dir, "pip.ini")) info = ConfigParser.RawConfigParser() info.readfp(infofp) infofp.close() assert info.has_section("download") assert info.get("download", "url").startswith("http://pypi.python.org/packages/source/I/INITools/INITools") - assert info.get("download", "requirement") == "INITools" + assert info.get("download", "requirement") == "INITools==0.3.1" def test_tarball(): """ - Test that the info.ini is written and works from an tarball. + Test that the pip.ini is written and works from an tarball. """ env = reset_env() run_pip('install', 'http://pypi.python.org/packages/source/I/INITools/INITools-0.3.1.tar.gz') - egg_info_dir = None - for x in os.listdir(os.path.join(os.path.dirname(env.venv_path), env.site_packages)): - if x.startswith("INITools-") and x.endswith(".egg-info"): - egg_info_dir = os.path.join(os.path.dirname(env.venv_path), env.site_packages, x) - break - assert egg_info_dir is not None + egg_info_dir = env.base_path / env.site_packages / 'INITools-0.3.1-py%s.egg-info' % pyversion - infofp = open(os.path.join(egg_info_dir, "info.ini")) + infofp = open(os.path.join(egg_info_dir, "pip.ini")) info = ConfigParser.RawConfigParser() info.readfp(infofp) infofp.close() @@ -56,24 +47,19 @@ def test_tarball(): def test_editable(): """ - Test that the info.ini is written and works from an editable. + Test that the pip.ini is written and works from an editable. """ env = reset_env() fspkg = path_to_url2(Path(here) / 'packages' / 'FSPkg') run_pip('install', '-e', fspkg) - egg_info_dir = None - for x in os.listdir(Path(here) / 'packages' / 'FSPkg'): - if x.startswith("FSPkg") and x.endswith(".egg-info"): - egg_info_dir = os.path.join(Path(here) / 'packages' / 'FSPkg', x) - break - assert egg_info_dir is not None + egg_info_dir = Path(here) / 'packages' / 'FSPkg' / 'FSPkg.egg-info' - infofp = open(os.path.join(egg_info_dir, "info.ini")) + infofp = open(os.path.join(egg_info_dir, "pip.ini")) info = ConfigParser.RawConfigParser() info.readfp(infofp) infofp.close() assert info.has_section("download") - assert info.get("download", "url") == "file:///Users/dstufft/projects/pip/tests/packages/FSPkg" - assert info.get("download", "requirement") == "--editable=file:///Users/dstufft/projects/pip/tests/packages/FSPkg" + assert info.get("download", "url") == fspkg + assert info.get("download", "requirement") == "--editable=" + fspkg From 9990c264dc34556af5adfaccd734800f0245ba95 Mon Sep 17 00:00:00 2001 From: Erik Simmler Date: Tue, 10 Apr 2012 13:58:55 -0400 Subject: [PATCH 09/11] Add branch to travis ci --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index afcf7d63a7d..165b24351be 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,5 +12,6 @@ notifications: branches: only: - develop + - feature/record-download-info env: - PIP_USE_MIRRORS=true From 5c6c0f9d18e7bef8d485b4afc3043f17540dfd6e Mon Sep 17 00:00:00 2001 From: Erik Simmler Date: Tue, 10 Apr 2012 15:29:46 -0400 Subject: [PATCH 10/11] Consolidated some of the testing code. --- tests/test_info_file.py | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/tests/test_info_file.py b/tests/test_info_file.py index f5c63e13fcd..1d51d5a7c80 100644 --- a/tests/test_info_file.py +++ b/tests/test_info_file.py @@ -7,6 +7,14 @@ from tests.path import Path +def get_pip_ini(egg_info_dir): + infofp = open(os.path.join(egg_info_dir, "pip.ini")) + info = ConfigParser.RawConfigParser() + info.readfp(infofp) + infofp.close() + return info + + def test_index(): """ Test that the pip.ini is written and works from an index (PyPI). @@ -15,11 +23,7 @@ def test_index(): run_pip('install', '-i http://pypi.python.org/simple/', 'INITools==0.3.1') egg_info_dir = env.base_path / env.site_packages / 'INITools-0.3.1-py%s.egg-info' % pyversion - - infofp = open(os.path.join(egg_info_dir, "pip.ini")) - info = ConfigParser.RawConfigParser() - info.readfp(infofp) - infofp.close() + info = get_pip_ini(egg_info_dir) assert info.has_section("download") assert info.get("download", "url").startswith("http://pypi.python.org/packages/source/I/INITools/INITools") @@ -34,11 +38,7 @@ def test_tarball(): run_pip('install', 'http://pypi.python.org/packages/source/I/INITools/INITools-0.3.1.tar.gz') egg_info_dir = env.base_path / env.site_packages / 'INITools-0.3.1-py%s.egg-info' % pyversion - - infofp = open(os.path.join(egg_info_dir, "pip.ini")) - info = ConfigParser.RawConfigParser() - info.readfp(infofp) - infofp.close() + info = get_pip_ini(egg_info_dir) assert info.has_section("download") assert info.get("download", "url") == "http://pypi.python.org/packages/source/I/INITools/INITools-0.3.1.tar.gz" @@ -54,11 +54,7 @@ def test_editable(): run_pip('install', '-e', fspkg) egg_info_dir = Path(here) / 'packages' / 'FSPkg' / 'FSPkg.egg-info' - - infofp = open(os.path.join(egg_info_dir, "pip.ini")) - info = ConfigParser.RawConfigParser() - info.readfp(infofp) - infofp.close() + info = get_pip_ini(egg_info_dir) assert info.has_section("download") assert info.get("download", "url") == fspkg From 79b6114f5844ea96707fc240eaae7eb017a93791 Mon Sep 17 00:00:00 2001 From: Erik Simmler Date: Tue, 10 Apr 2012 17:14:04 -0400 Subject: [PATCH 11/11] Remove iteritems() for py3 compatibility. --- pip/req.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pip/req.py b/pip/req.py index f45b8e85869..40df18b6445 100644 --- a/pip/req.py +++ b/pip/req.py @@ -554,10 +554,10 @@ def _clean_zip_name(self, name, prefix): def _write_info_ini(self, egg_info_dir, options): info = ConfigParser.RawConfigParser() - for section, values in options.iteritems(): + for section, values in options.items(): info.add_section(section) - for key, value in values.iteritems(): + for key, value in values.items(): info.set(section, key, value) f = open(os.path.join(egg_info_dir, 'pip.ini'), 'w')