From 87b7e7a5cb1a3148f87b9b590d23ae227ad5c53b Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Mon, 2 Oct 2023 15:54:06 +0200 Subject: [PATCH 1/4] Catch warning in PyLint tests --- pyiceberg/manifest.py | 10 +++++----- pyproject.toml | 5 +++++ tests/catalog/test_sql.py | 1 + tests/conftest.py | 1 + tests/io/test_io.py | 2 ++ tests/io/test_pyarrow_stats.py | 1 + tests/utils/test_manifest.py | 4 ++-- 7 files changed, 17 insertions(+), 7 deletions(-) diff --git a/pyiceberg/manifest.py b/pyiceberg/manifest.py index 8bdbfd3524..54bd951db5 100644 --- a/pyiceberg/manifest.py +++ b/pyiceberg/manifest.py @@ -668,8 +668,8 @@ def __init__(self, spec: PartitionSpec, schema: Schema, output_file: OutputFile, output_file, snapshot_id, { - "schema": schema.json(), - "partition-spec": spec.json(), + "schema": schema.model_dump_json(), + "partition-spec": spec.model_dump_json(), "partition-spec-id": str(spec.spec_id), "format-version": "1", }, @@ -696,9 +696,9 @@ def __init__(self, spec: PartitionSpec, schema: Schema, output_file: OutputFile, schema, output_file, snapshot_id, - { - "schema": schema.json(), - "partition-spec": spec.json(), + meta={ + "schema": schema.model_dump_json(), + "partition-spec": spec.model_dump_json(), "partition-spec-id": str(spec.spec_id), "format-version": "2", "content": "data", diff --git a/pyproject.toml b/pyproject.toml index 5090e29cbb..0735de2fc6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -122,6 +122,11 @@ markers = [ "gcs: marks a test as requiring access to gcs compliant storage (use with --gs.token, --gs.project, and --gs.endpoint)" ] +# Turns a warning into an error +filterwarnings = [ + "error" +] + [tool.black] line-length = 130 target-version = ['py38'] diff --git a/tests/catalog/test_sql.py b/tests/catalog/test_sql.py index 41bb5c76b8..4277845633 100644 --- a/tests/catalog/test_sql.py +++ b/tests/catalog/test_sql.py @@ -304,6 +304,7 @@ def test_create_namespace_with_comment_and_location(test_catalog: SqlCatalog, da assert properties["location"] == test_location +@pytest.mark.filterwarnings("ignore") def test_create_namespace_with_null_properties(test_catalog: SqlCatalog, database_name: str) -> None: with pytest.raises(IntegrityError): test_catalog.create_namespace(namespace=database_name, properties={None: "value"}) # type: ignore diff --git a/tests/conftest.py b/tests/conftest.py index ed7f1caa21..fdc6fb451c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1544,6 +1544,7 @@ def adlfs_fsspec_fileio(request: pytest.FixtureRequest) -> Generator[FsspecFileI bbs.create_container("tests") yield fsspec.FsspecFileIO(properties=properties) bbs.delete_container("tests") + bbs.close() @pytest.fixture(scope="session") diff --git a/tests/io/test_io.py b/tests/io/test_io.py index c4dc3d45a5..d27274e17e 100644 --- a/tests/io/test_io.py +++ b/tests/io/test_io.py @@ -288,12 +288,14 @@ def test_load_file_io_location_no_schema() -> None: assert isinstance(load_file_io({"location": "/no-schema/"}), PyArrowFileIO) +@pytest.mark.filterwarnings("ignore") def test_mock_warehouse_location_file_io() -> None: # For testing the selection logic io = load_file_io({"warehouse": "test://some-path/"}) assert io.properties["warehouse"] == "test://some-path/" +@pytest.mark.filterwarnings("ignore") def test_mock_table_location_file_io() -> None: # For testing the selection logic io = load_file_io({}, "test://some-path/") diff --git a/tests/io/test_pyarrow_stats.py b/tests/io/test_pyarrow_stats.py index 74297fe526..6f00061174 100644 --- a/tests/io/test_pyarrow_stats.py +++ b/tests/io/test_pyarrow_stats.py @@ -76,6 +76,7 @@ @dataclass(frozen=True) class TestStruct: + __test__ = False x: Optional[int] y: Optional[float] diff --git a/tests/utils/test_manifest.py b/tests/utils/test_manifest.py index 41af844bba..b2c972b8c2 100644 --- a/tests/utils/test_manifest.py +++ b/tests/utils/test_manifest.py @@ -340,8 +340,8 @@ def test_write_manifest(generated_manifest_file_file_v1: str, generated_manifest writer.add_entry(manifest_entries[0]) expected_metadata = { - "schema": test_schema.json(), - "partition-spec": test_spec.json(), + "schema": test_schema.model_dump_json(), + "partition-spec": test_spec.model_dump_json(), "partition-spec-id": str(test_spec.spec_id), "format-version": str(format_version), } From 3c1cc1e0943575fe85d2102fedd560253ec34d9a Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Thu, 5 Oct 2023 10:31:46 +0200 Subject: [PATCH 2/4] Update Setuptools --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 80e6f4dee7..b3af1a4008 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,8 @@ # under the License. install-poetry: - pip install poetry==1.6.1 + pip3 install setuptools --upgrade + pip3 install poetry==1.6.1 install-dependencies: poetry install -E pyarrow -E hive -E s3fs -E glue -E adlfs -E duckdb -E ray -E sql-postgres -E gcsfs From 1a06e8f822ffcfab8b28448ec7dfec483193bfbf Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Thu, 5 Oct 2023 10:34:01 +0200 Subject: [PATCH 3/4] Change spaces into a tab --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index b3af1a4008..353011d511 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ # under the License. install-poetry: - pip3 install setuptools --upgrade + pip3 install setuptools --upgrade pip3 install poetry==1.6.1 install-dependencies: From c2c45046abe6ea3bf854b1dc54142490803c3b2b Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Mon, 16 Oct 2023 23:03:47 +0200 Subject: [PATCH 4/4] Ignore Ray warning --- Makefile | 3 +-- tests/test_integration.py | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 353011d511..80e6f4dee7 100644 --- a/Makefile +++ b/Makefile @@ -16,8 +16,7 @@ # under the License. install-poetry: - pip3 install setuptools --upgrade - pip3 install poetry==1.6.1 + pip install poetry==1.6.1 install-dependencies: poetry install -E pyarrow -E hive -E s3fs -E glue -E adlfs -E duckdb -E ray -E sql-postgres -E gcsfs diff --git a/tests/test_integration.py b/tests/test_integration.py index 6e874b68fa..8771eb3488 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -189,6 +189,7 @@ def test_pyarrow_limit(table_test_limit: Table) -> None: assert len(full_result) == 10 +@pytest.mark.filterwarnings("ignore") @pytest.mark.integration def test_ray_nan(table_test_null_nan_rewritten: Table) -> None: ray_dataset = table_test_null_nan_rewritten.scan().to_ray()