From adbb892bf483ef01ae11cb8a5eb1a89f84b46a4e Mon Sep 17 00:00:00 2001 From: jmahlik <38999128+jmahlik@users.noreply.github.com> Date: Thu, 13 Jan 2022 20:15:18 -0600 Subject: [PATCH 1/3] fix(s3.list): type hinting for chunked arg --- awswrangler/s3/_list.py | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/awswrangler/s3/_list.py b/awswrangler/s3/_list.py index c2513ced6..12b586ba2 100644 --- a/awswrangler/s3/_list.py +++ b/awswrangler/s3/_list.py @@ -3,7 +3,8 @@ import datetime import fnmatch import logging -from typing import Any, Dict, Iterator, List, Optional, Sequence, Union +from typing import Any, Dict, Iterator, List, Optional, overload, Sequence, Union +from typing_extensions import Literal import boto3 import botocore.exceptions @@ -267,6 +268,36 @@ def list_directories( return [path for paths in result_iterator for path in paths] +@overload +def list_objects( + path: str, + suffix: Union[str, List[str], None] = None, + ignore_suffix: Union[str, List[str], None] = None, + last_modified_begin: Optional[datetime.datetime] = None, + last_modified_end: Optional[datetime.datetime] = None, + ignore_empty: bool = False, + chunked: Literal[False] = False, + s3_additional_kwargs: Optional[Dict[str, Any]] = None, + boto3_session: Optional[boto3.Session] = None, +) -> List[str]: + ... + + +@overload +def list_objects( + path: str, + suffix: Union[str, List[str], None] = None, + ignore_suffix: Union[str, List[str], None] = None, + last_modified_begin: Optional[datetime.datetime] = None, + last_modified_end: Optional[datetime.datetime] = None, + ignore_empty: bool = False, + chunked: bool = False, + s3_additional_kwargs: Optional[Dict[str, Any]] = None, + boto3_session: Optional[boto3.Session] = None, +) -> Union[List[str], Iterator[List[str]]]: + ... + + def list_objects( path: str, suffix: Union[str, List[str], None] = None, From 804bb0f5d1f250679100aa9d06221a6671b9ad9e Mon Sep 17 00:00:00 2001 From: Anton Kukushkin <3997468+kukushking@users.noreply.github.com> Date: Mon, 17 Jan 2022 16:58:31 +0000 Subject: [PATCH 2/3] Fix isort --- awswrangler/s3/_list.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/awswrangler/s3/_list.py b/awswrangler/s3/_list.py index 12b586ba2..90306d58b 100644 --- a/awswrangler/s3/_list.py +++ b/awswrangler/s3/_list.py @@ -3,11 +3,11 @@ import datetime import fnmatch import logging -from typing import Any, Dict, Iterator, List, Optional, overload, Sequence, Union -from typing_extensions import Literal +from typing import Any, Dict, Iterator, List, Optional, Sequence, Union, overload import boto3 import botocore.exceptions +from typing_extensions import Literal from awswrangler import _utils, exceptions from awswrangler.s3 import _fs From df49487dce2f054b16c0a3f7bedc888d85e4dfc1 Mon Sep 17 00:00:00 2001 From: jmahlik <38999128+jmahlik@users.noreply.github.com> Date: Fri, 21 Jan 2022 12:27:40 -0600 Subject: [PATCH 3/3] fix(s3.list): type hinting for chunked arg to ... --- awswrangler/s3/_list.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/awswrangler/s3/_list.py b/awswrangler/s3/_list.py index 90306d58b..7e61bd64e 100644 --- a/awswrangler/s3/_list.py +++ b/awswrangler/s3/_list.py @@ -271,14 +271,14 @@ def list_directories( @overload def list_objects( path: str, - suffix: Union[str, List[str], None] = None, - ignore_suffix: Union[str, List[str], None] = None, - last_modified_begin: Optional[datetime.datetime] = None, - last_modified_end: Optional[datetime.datetime] = None, - ignore_empty: bool = False, + suffix: Union[str, List[str], None] = ..., + ignore_suffix: Union[str, List[str], None] = ..., + last_modified_begin: Optional[datetime.datetime] = ..., + last_modified_end: Optional[datetime.datetime] = ..., + ignore_empty: bool = ..., chunked: Literal[False] = False, - s3_additional_kwargs: Optional[Dict[str, Any]] = None, - boto3_session: Optional[boto3.Session] = None, + s3_additional_kwargs: Optional[Dict[str, Any]] = ..., + boto3_session: Optional[boto3.Session] = ..., ) -> List[str]: ... @@ -286,14 +286,14 @@ def list_objects( @overload def list_objects( path: str, - suffix: Union[str, List[str], None] = None, - ignore_suffix: Union[str, List[str], None] = None, - last_modified_begin: Optional[datetime.datetime] = None, - last_modified_end: Optional[datetime.datetime] = None, - ignore_empty: bool = False, - chunked: bool = False, - s3_additional_kwargs: Optional[Dict[str, Any]] = None, - boto3_session: Optional[boto3.Session] = None, + suffix: Union[str, List[str], None] = ..., + ignore_suffix: Union[str, List[str], None] = ..., + last_modified_begin: Optional[datetime.datetime] = ..., + last_modified_end: Optional[datetime.datetime] = ..., + ignore_empty: bool = ..., + chunked: Literal[True] = True, + s3_additional_kwargs: Optional[Dict[str, Any]] = ..., + boto3_session: Optional[boto3.Session] = ..., ) -> Union[List[str], Iterator[List[str]]]: ...