From bfaf2304d59306b20bd8fdfcfa3210812e416fcd Mon Sep 17 00:00:00 2001 From: Tieg Zaharia Date: Mon, 1 Mar 2021 15:07:20 -0500 Subject: [PATCH 01/10] Adds a 'before' parameter to /rss/updates.xml endpoint. --- warehouse/rss/views.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/warehouse/rss/views.py b/warehouse/rss/views.py index 7f62c8bd37e6..562ad5f92781 100644 --- a/warehouse/rss/views.py +++ b/warehouse/rss/views.py @@ -10,6 +10,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from datetime import datetime + from email.utils import getaddresses from pyramid.view import view_config @@ -59,6 +61,12 @@ def _format_author(release): ], ) def rss_updates(request): + try: + before_cursor_timestamp = int(request.params.get("before", datetime.now().timestamp())) + except ValueError: + raise HTTPBadRequest("'before' must be a UTC timestamp integer in milliseconds.") from None + before_cursor = datetime.utcfromtimestamp(before_cursor_timestamp) + request.response.content_type = "text/xml" request.find_service(name="csp").merge(XML_CSP) @@ -66,6 +74,7 @@ def rss_updates(request): latest_releases = ( request.db.query(Release) .options(joinedload(Release.project)) + .filter(Release.created < before_cursor) .order_by(Release.created.desc()) .limit(40) .all() From 4da1c1f90eed847f8a37e07da8099e337dee03a9 Mon Sep 17 00:00:00 2001 From: Tieg Zaharia Date: Mon, 1 Mar 2021 17:22:46 -0500 Subject: [PATCH 02/10] Add a test for 'before' cursor param. --- tests/unit/rss/test_views.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/unit/rss/test_views.py b/tests/unit/rss/test_views.py index 5465b8ad6720..86636b8435cb 100644 --- a/tests/unit/rss/test_views.py +++ b/tests/unit/rss/test_views.py @@ -46,6 +46,13 @@ def test_rss_updates(db_request): } assert db_request.response.content_type == "text/xml" + db_request.params["before"] = release3.created.timestamp() + assert rss.rss_updates(db_request) == { + "latest_releases": tuple( + zip((release2, release1), ("noreply@pypi.org", None)) + ) + } + assert db_request.response.content_type == "text/xml" def test_rss_packages(db_request): db_request.find_service = pretend.call_recorder( From accbbb1237b29e0aa81532813b367ca745e3029f Mon Sep 17 00:00:00 2001 From: Tieg Zaharia Date: Mon, 1 Mar 2021 18:36:08 -0500 Subject: [PATCH 03/10] Docs --- docs/api-reference/feeds.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/api-reference/feeds.rst b/docs/api-reference/feeds.rst index 77770389f12d..e3660fca1b31 100644 --- a/docs/api-reference/feeds.rst +++ b/docs/api-reference/feeds.rst @@ -21,6 +21,10 @@ Available at https://pypi.org/rss/updates.xml, this feed provides the latest newly created releases for individual projects on PyPI, including the project name and description, release version, and a link to the release page. +Accepts an optional `before` cursor, which limits results to the latest +releases before the given UTC integer seconds since the epoch (e.g., the +``timestamp`` method to a ``datetime.datetime`` object). + Project Releases Feed --------------------- From 2c6d66721e8ee3719bec9e95db951bdddd7b5276 Mon Sep 17 00:00:00 2001 From: Tieg Zaharia Date: Mon, 1 Mar 2021 18:37:09 -0500 Subject: [PATCH 04/10] Backticks --- docs/api-reference/feeds.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/api-reference/feeds.rst b/docs/api-reference/feeds.rst index e3660fca1b31..e57da9371817 100644 --- a/docs/api-reference/feeds.rst +++ b/docs/api-reference/feeds.rst @@ -21,7 +21,7 @@ Available at https://pypi.org/rss/updates.xml, this feed provides the latest newly created releases for individual projects on PyPI, including the project name and description, release version, and a link to the release page. -Accepts an optional `before` cursor, which limits results to the latest +Accepts an optional ``before`` cursor, which limits results to the latest releases before the given UTC integer seconds since the epoch (e.g., the ``timestamp`` method to a ``datetime.datetime`` object). From 0a204359923ca8be9b045a3e704c4bf05c9be99c Mon Sep 17 00:00:00 2001 From: Tieg Zaharia Date: Mon, 1 Mar 2021 18:37:51 -0500 Subject: [PATCH 05/10] 40 -> 100 page size. --- warehouse/rss/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/warehouse/rss/views.py b/warehouse/rss/views.py index 562ad5f92781..ccf1d7864742 100644 --- a/warehouse/rss/views.py +++ b/warehouse/rss/views.py @@ -76,7 +76,7 @@ def rss_updates(request): .options(joinedload(Release.project)) .filter(Release.created < before_cursor) .order_by(Release.created.desc()) - .limit(40) + .limit(100) .all() ) release_authors = [_format_author(release) for release in latest_releases] From 3a1f2c4acee94714f29514eddb043b22b705aefc Mon Sep 17 00:00:00 2001 From: Tieg Zaharia Date: Mon, 1 Mar 2021 19:05:38 -0500 Subject: [PATCH 06/10] Fix test. --- tests/unit/rss/test_views.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit/rss/test_views.py b/tests/unit/rss/test_views.py index 86636b8435cb..237de3294f4a 100644 --- a/tests/unit/rss/test_views.py +++ b/tests/unit/rss/test_views.py @@ -46,7 +46,8 @@ def test_rss_updates(db_request): } assert db_request.response.content_type == "text/xml" - db_request.params["before"] = release3.created.timestamp() + before_cursor = datetime.datetime(release3.created.year, release3.created.month, release3.created.day) + db_request.params["before"] = before_cursor.timestamp() assert rss.rss_updates(db_request) == { "latest_releases": tuple( zip((release2, release1), ("noreply@pypi.org", None)) From 897fc6542f9865234bd4148c5b464bf0c588797f Mon Sep 17 00:00:00 2001 From: Tieg Zaharia Date: Mon, 1 Mar 2021 19:26:23 -0500 Subject: [PATCH 07/10] Lint. --- tests/unit/rss/test_views.py | 9 +++++---- warehouse/rss/views.py | 7 ++++--- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/unit/rss/test_views.py b/tests/unit/rss/test_views.py index 237de3294f4a..1659a3183353 100644 --- a/tests/unit/rss/test_views.py +++ b/tests/unit/rss/test_views.py @@ -46,15 +46,16 @@ def test_rss_updates(db_request): } assert db_request.response.content_type == "text/xml" - before_cursor = datetime.datetime(release3.created.year, release3.created.month, release3.created.day) + before_cursor = datetime.datetime( + release3.created.year, release3.created.month, release3.created.day + ) db_request.params["before"] = before_cursor.timestamp() assert rss.rss_updates(db_request) == { - "latest_releases": tuple( - zip((release2, release1), ("noreply@pypi.org", None)) - ) + "latest_releases": tuple(zip((release2, release1), ("noreply@pypi.org", None))) } assert db_request.response.content_type == "text/xml" + def test_rss_packages(db_request): db_request.find_service = pretend.call_recorder( lambda *args, **kwargs: pretend.stub( diff --git a/warehouse/rss/views.py b/warehouse/rss/views.py index ccf1d7864742..3afac871fa69 100644 --- a/warehouse/rss/views.py +++ b/warehouse/rss/views.py @@ -11,9 +11,9 @@ # limitations under the License. from datetime import datetime - from email.utils import getaddresses +from pyramid.httpexceptions import HTTPBadRequest from pyramid.view import view_config from sqlalchemy.orm import joinedload @@ -62,9 +62,10 @@ def _format_author(release): ) def rss_updates(request): try: - before_cursor_timestamp = int(request.params.get("before", datetime.now().timestamp())) + now = datetime.now().timestamp() + before_cursor_timestamp = int(request.params.get("before", now)) except ValueError: - raise HTTPBadRequest("'before' must be a UTC timestamp integer in milliseconds.") from None + raise HTTPBadRequest("'before' must be an integer") from None before_cursor = datetime.utcfromtimestamp(before_cursor_timestamp) request.response.content_type = "text/xml" From 84f8736ed53003c04899f8f65c18af7caad9aa04 Mon Sep 17 00:00:00 2001 From: Tieg Zaharia Date: Tue, 2 Mar 2021 17:20:15 -0500 Subject: [PATCH 08/10] before -> after --- docs/api-reference/feeds.rst | 7 ++++--- tests/unit/rss/test_views.py | 8 ++++---- warehouse/rss/views.py | 30 ++++++++++++++++-------------- 3 files changed, 24 insertions(+), 21 deletions(-) diff --git a/docs/api-reference/feeds.rst b/docs/api-reference/feeds.rst index e57da9371817..37e8637be76c 100644 --- a/docs/api-reference/feeds.rst +++ b/docs/api-reference/feeds.rst @@ -21,9 +21,10 @@ Available at https://pypi.org/rss/updates.xml, this feed provides the latest newly created releases for individual projects on PyPI, including the project name and description, release version, and a link to the release page. -Accepts an optional ``before`` cursor, which limits results to the latest -releases before the given UTC integer seconds since the epoch (e.g., the -``timestamp`` method to a ``datetime.datetime`` object). +Accepts an optional ``after`` integer cursor, which limits results to the +releases after the given seconds since the UTC epoch (e.g., the +``timestamp`` method to a ``datetime.datetime`` object). Note that this changes +the ordering of the results from newest->oldest to oldest->newest. Project Releases Feed diff --git a/tests/unit/rss/test_views.py b/tests/unit/rss/test_views.py index 1659a3183353..1ab7cbfaba89 100644 --- a/tests/unit/rss/test_views.py +++ b/tests/unit/rss/test_views.py @@ -46,12 +46,12 @@ def test_rss_updates(db_request): } assert db_request.response.content_type == "text/xml" - before_cursor = datetime.datetime( - release3.created.year, release3.created.month, release3.created.day + after_cursor = datetime.datetime( + release1.created.year, release1.created.month, release1.created.day ) - db_request.params["before"] = before_cursor.timestamp() + db_request.params["after"] = after_cursor.timestamp() assert rss.rss_updates(db_request) == { - "latest_releases": tuple(zip((release2, release1), ("noreply@pypi.org", None))) + "latest_releases": tuple(zip((release2, release3), ("noreply@pypi.org", None))) } assert db_request.response.content_type == "text/xml" diff --git a/warehouse/rss/views.py b/warehouse/rss/views.py index 3afac871fa69..cc231725b0d7 100644 --- a/warehouse/rss/views.py +++ b/warehouse/rss/views.py @@ -61,25 +61,27 @@ def _format_author(release): ], ) def rss_updates(request): - try: - now = datetime.now().timestamp() - before_cursor_timestamp = int(request.params.get("before", now)) - except ValueError: - raise HTTPBadRequest("'before' must be an integer") from None - before_cursor = datetime.utcfromtimestamp(before_cursor_timestamp) + query = request.db.query(Release).options(joinedload(Release.project)) + + after_timestamp = request.params.get("after") + if after_timestamp: + try: + after_timestamp = datetime.utcfromtimestamp(int(after_timestamp)) + except ValueError: + raise HTTPBadRequest("'after' must be an integer") from None + query = ( + query + .filter(Release.created > after_timestamp) + .order_by(Release.created.asc()) + ) + else: + query = query.order_by(Release.created.desc()) request.response.content_type = "text/xml" request.find_service(name="csp").merge(XML_CSP) - latest_releases = ( - request.db.query(Release) - .options(joinedload(Release.project)) - .filter(Release.created < before_cursor) - .order_by(Release.created.desc()) - .limit(100) - .all() - ) + latest_releases = query.limit(100).all() release_authors = [_format_author(release) for release in latest_releases] return {"latest_releases": tuple(zip(latest_releases, release_authors))} From df5bc9e954c4f097c6f20086ea2c4051aeac3254 Mon Sep 17 00:00:00 2001 From: Tieg Zaharia Date: Tue, 2 Mar 2021 17:28:26 -0500 Subject: [PATCH 09/10] make reformat --- warehouse/rss/views.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/warehouse/rss/views.py b/warehouse/rss/views.py index cc231725b0d7..f2d986ee97a0 100644 --- a/warehouse/rss/views.py +++ b/warehouse/rss/views.py @@ -69,10 +69,8 @@ def rss_updates(request): after_timestamp = datetime.utcfromtimestamp(int(after_timestamp)) except ValueError: raise HTTPBadRequest("'after' must be an integer") from None - query = ( - query - .filter(Release.created > after_timestamp) - .order_by(Release.created.asc()) + query = query.filter(Release.created > after_timestamp).order_by( + Release.created.asc() ) else: query = query.order_by(Release.created.desc()) From 89f97a4f71d4823a66d4ff5dabb7d64ff794a536 Mon Sep 17 00:00:00 2001 From: Tieg Zaharia Date: Tue, 2 Mar 2021 18:44:14 -0500 Subject: [PATCH 10/10] Break the queries into each branch to make it more readable. --- warehouse/rss/views.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/warehouse/rss/views.py b/warehouse/rss/views.py index f2d986ee97a0..d4e2ae1e228d 100644 --- a/warehouse/rss/views.py +++ b/warehouse/rss/views.py @@ -61,7 +61,6 @@ def _format_author(release): ], ) def rss_updates(request): - query = request.db.query(Release).options(joinedload(Release.project)) after_timestamp = request.params.get("after") if after_timestamp: @@ -69,17 +68,27 @@ def rss_updates(request): after_timestamp = datetime.utcfromtimestamp(int(after_timestamp)) except ValueError: raise HTTPBadRequest("'after' must be an integer") from None - query = query.filter(Release.created > after_timestamp).order_by( - Release.created.asc() + latest_releases = ( + request.db.query(Release) + .options(joinedload(Release.project)) + .filter(Release.created > after_timestamp) + .order_by(Release.created.asc()) + .limit(100) + .all() ) else: - query = query.order_by(Release.created.desc()) + latest_releases = ( + request.db.query(Release) + .options(joinedload(Release.project)) + .order_by(Release.created.desc()) + .limit(100) + .all() + ) request.response.content_type = "text/xml" request.find_service(name="csp").merge(XML_CSP) - latest_releases = query.limit(100).all() release_authors = [_format_author(release) for release in latest_releases] return {"latest_releases": tuple(zip(latest_releases, release_authors))}