From 1354e45b55413a6f6707a333451925cdb6ffc3c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Thu, 22 May 2025 15:33:40 +0200 Subject: [PATCH 1/5] patches: Simplify patch grouping by submitter and delegate As a desired side effect this allows to let api.patch_list() return a generator instead of a list. --- pwclient/patches.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pwclient/patches.py b/pwclient/patches.py index 67e9fc9..eb4d1db 100644 --- a/pwclient/patches.py +++ b/pwclient/patches.py @@ -56,6 +56,10 @@ def patch_field(matchobj): ) +def sortgroupby(iterable, key=None): + return itertools.groupby(sorted(iterable, key=key), key=key) + + def action_list( api, project=None, @@ -89,9 +93,8 @@ def action_list( filters['submitter'] = submitter patches = api.patch_list(**filters) - patches.sort(key=lambda x: x['submitter']) - for person, person_patches in itertools.groupby( + for person, person_patches in sortgroupby( patches, key=lambda x: x['submitter'] ): print(f'Patches submitted by {person}:') @@ -103,9 +106,8 @@ def action_list( filters['delegate'] = delegate patches = api.patch_list(**filters) - patches.sort(key=lambda x: x['delegate']) - for delegate, delegate_patches in itertools.groupby( + for delegate, delegate_patches in sortgroupby( patches, key=lambda x: x['delegate'] ): print(f'Patches delegated to {delegate}:') From 3893689f514882ca8236ce614849d316de817ebe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Thu, 22 May 2025 16:42:34 +0200 Subject: [PATCH 2/5] patches: Make use of operator.itemgetter() instead of open-coding it --- pwclient/patches.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pwclient/patches.py b/pwclient/patches.py index eb4d1db..8127843 100644 --- a/pwclient/patches.py +++ b/pwclient/patches.py @@ -6,6 +6,7 @@ import io import itertools +import operator import os import re import subprocess @@ -95,7 +96,7 @@ def action_list( patches = api.patch_list(**filters) for person, person_patches in sortgroupby( - patches, key=lambda x: x['submitter'] + patches, key=operator.itemgetter('submitter') ): print(f'Patches submitted by {person}:') _list_patches(list(person_patches), format_str) @@ -108,7 +109,7 @@ def action_list( patches = api.patch_list(**filters) for delegate, delegate_patches in sortgroupby( - patches, key=lambda x: x['delegate'] + patches, key=operator.itemgetter('delegate') ): print(f'Patches delegated to {delegate}:') _list_patches(list(delegate_patches), format_str) From a149e35cfd5ae29be16ea4360a1afca4c293e061 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Thu, 22 May 2025 16:49:41 +0200 Subject: [PATCH 3/5] api/rest: Handle patchwork pagination If there are many patches (or other objects) to be listed, patchwork only returns the first 30 by default. In such a case continue fetching objects using the "next" link provided in the Link: header. Note this changes REST._list() to return a generator instead of a list. All callers only use it to generate a list, so this isn't a problem. --- pwclient/api.py | 21 +++++++++++++++++++-- requirements.txt | 2 ++ 2 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 requirements.txt diff --git a/pwclient/api.py b/pwclient/api.py index 15bceb8..8d55ddb 100644 --- a/pwclient/api.py +++ b/pwclient/api.py @@ -19,6 +19,8 @@ from . import __version__ from .xmlrpc import xmlrpclib +from requests.utils import parse_header_links + class API(metaclass=abc.ABCMeta): @abc.abstractmethod @@ -594,8 +596,23 @@ def _list( url = f'{url}{resource_id}/{subresource_type}/' if params: url = f'{url}?{urllib.parse.urlencode(params)}' - data, _ = self._get(url) - return json.loads(data) + + while url: + data, headers = self._get(url) + ret = json.loads(data) + + yield from ret + + try: + link = next(value for name, value in headers if name == 'Link') + url = next( + a['url'] + for a in parse_header_links(link) + if a['rel'] == "next" + ) + + except StopIteration: + url = None # project diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..8dee024 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +requests +types-requests From dc8c6ba1bba9bcf84cdea70b87db1c56701c102d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Thu, 22 May 2025 16:54:39 +0200 Subject: [PATCH 4/5] api/rest: Add support for the submitter filter Similar to commit 70f96f971088 ("support the delegate filter for REST API") which added the delegate filter this implements an exact match only and not (as promised by `pwclient list -h`) as substring match. But for the same reason as for the delegate filter this is better than ignoring a passed submitter. --- pwclient/api.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pwclient/api.py b/pwclient/api.py index 8d55ddb..3a36302 100644 --- a/pwclient/api.py +++ b/pwclient/api.py @@ -764,6 +764,9 @@ def patch_list( if archived is not None: filters['archived'] = archived + if submitter is not None: + filters['submitter'] = submitter + if delegate is not None: filters['delegate'] = delegate From 7cb9da6d6ed4e04c8db0c04ca98cb74db750ba8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Thu, 22 May 2025 17:06:22 +0200 Subject: [PATCH 5/5] api/rest: Let p{roject,eople,atches}_list return a generator This has the advantage that with pagination the first entries are already provided before later entries are queried from the server. Also if the generator isn't walked through completely, only the needed queries are actually executed. All callers of these functions can cope with the returned object being a generator now instead of a list as they just iterate once over it. --- pwclient/api.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pwclient/api.py b/pwclient/api.py index 3a36302..cc8f53c 100644 --- a/pwclient/api.py +++ b/pwclient/api.py @@ -644,7 +644,7 @@ def project_list(self, search_str=None, max_count=0): ) projects = self._list('projects') - return [self._project_to_dict(project) for project in projects] + return (self._project_to_dict(project) for project in projects) def project_get(self, project_id): project = self._detail('projects', project_id) @@ -679,7 +679,7 @@ def person_list(self, search_str=None, max_count=0): ) people = self._list('people') - return [self._person_to_dict(person) for person in people] + return (self._person_to_dict(person) for person in people) def person_get(self, person_id): person = self._detail('people', person_id) @@ -771,7 +771,7 @@ def patch_list( filters['delegate'] = delegate patches = self._list('patches', params=filters) - return [self._patch_to_dict(patch) for patch in patches] + return (self._patch_to_dict(patch) for patch in patches) def patch_get(self, patch_id): patch = self._detail('patches', patch_id)