Merge branch 'main' of github.com:apache/iceberg-python into fd-warnings

Author: Fokko

mkdocs/docs/how-to-release.md

@@ -57,22 +57,24 @@ Before committing the files to the Apache SVN artifact distribution SVN hashes n
 Go to [Github Actions and run the `Python release` action](https://github.com/apache/iceberg/actions/workflows/python-release.yml). **Set the version to master, since we cannot modify the source**. Download the zip, and sign the files:
 
 ```bash
-for name in $(ls release-master/pyiceberg-*.whl release-master/pyiceberg-*.tar.gz)
+cd release-master/
+
+for name in $(ls pyiceberg-*.whl pyiceberg-*.tar.gz)
 do
     gpg --yes --armor --local-user [email protected] --output "${name}.asc" --detach-sig "${name}"
-    shasum -a 512 "${name}.asc" > "${name}.asc.sha512"
+    shasum -a 512 "${name}" > "${name}.sha512"
 done
 ```
 
-Now we can upload the files
+Now we can upload the files from the same directory:
 
 ```bash
 export SVN_TMP_DIR=/tmp/iceberg-${VERSION_BRANCH}/
 svn checkout https://dist.apache.org/repos/dist/dev/iceberg $SVN_TMP_DIR
 
 export SVN_TMP_DIR_VERSIONED=${SVN_TMP_DIR}pyiceberg-$VERSION/
 mkdir -p $SVN_TMP_DIR_VERSIONED
-cp release-master/* $SVN_TMP_DIR_VERSIONED
+cp * $SVN_TMP_DIR_VERSIONED
 svn add $SVN_TMP_DIR_VERSIONED
 svn ci -m "PyIceberg ${VERSION}" ${SVN_TMP_DIR_VERSIONED}
 ```
@@ -92,11 +94,15 @@ Final step is to generate the email to the dev mail list:
 ```bash
 cat << EOF > release-announcement-email.txt
 To: [email protected]
-Subject: [VOTE] Release Apache PyIceberg $VERSION_WITHOUT_RC
+Subject: [VOTE] Release Apache PyIceberg $VERSION
 Hi Everyone,
 
 I propose that we release the following RC as the official PyIceberg $VERSION_WITHOUT_RC release.
 
+A summary of the high level features:
+
+* <Add summary by hand>
+
 The commit ID is $LAST_COMMIT_ID
 
 * This corresponds to the tag: $GIT_TAG ($GIT_TAG_HASH)

mkdocs/docs/verify-release.md

@@ -47,20 +47,29 @@ gpg --import KEYS
 Next, verify the `.asc` file.
 
 ```sh
-gpg --verify pyiceberg-0.4.0-py3-none-any.whl.asc pyiceberg-0.4.0-py3-none-any.whl
+svn checkout https://dist.apache.org/repos/dist/dev/iceberg/pyiceberg-0.5.0rc1/ /tmp/pyiceberg/
+
+for name in $(ls /tmp/pyiceberg/pyiceberg-*.whl /tmp/pyiceberg/pyiceberg-*.tar.gz)
+do
+    gpg --verify ${name}.asc ${name}
+done
 ```
 
 ## Verifying checksums
 
 ```sh
-shasum -a 512 --check pyiceberg-0.4.0-py3-none-any.whl.sha512
+cd  /tmp/pyiceberg/
+for name in $(ls /tmp/pyiceberg/pyiceberg-*.whl.asc.sha512 /tmp/pyiceberg/pyiceberg-*.tar.gz.asc.sha512)
+do
+    shasum -a 512 --check ${name}
+done
 ```
 
 ## Verifying License Documentation
 
 ```sh
-tar xzf pyiceberg-0.4.0.tar.gz
-cd pyiceberg-0.4.0
+tar xzf pyiceberg-0.5.0.tar.gz
+cd pyiceberg-0.5.0
 ```
 
 Run RAT checks to validate license header:

mkdocs/requirements.txt

@@ -16,13 +16,13 @@
 # under the License.
 
 mkdocs==1.5.3
-griffe==0.36.2
+griffe==0.36.4
 jinja2==3.1.2
 mkdocstrings==0.23.0
 mkdocstrings-python==1.7.1
 mkdocs-literate-nav==0.6.1
 mkdocs-autorefs==0.5.0
 mkdocs-gen-files==0.5.0
-mkdocs-material==9.4.1
+mkdocs-material==9.4.2
 mkdocs-material-extensions==1.2
 mkdocs-section-index==0.3.8

poetry.lock

@@ -17,6 +17,7 @@
 from __future__ import annotations
 
 import itertools
+import uuid
 from abc import ABC, abstractmethod
 from copy import copy
 from dataclasses import dataclass
@@ -498,6 +499,14 @@ def location(self) -> str:
         """Return the table's base location."""
         return self.metadata.location
 
+    def new_snapshot_id(self) -> int:
+        """Generate a new snapshot-id that's not in use."""
+        snapshot_id = _generate_snapshot_id()
+        while self.snapshot_by_id(snapshot_id) is not None:
+            snapshot_id = _generate_snapshot_id()
+
+        return snapshot_id
+
     def current_snapshot(self) -> Optional[Snapshot]:
         """Get the current snapshot for this table, or None if there is no current snapshot."""
         if snapshot_id := self.metadata.current_snapshot_id:
@@ -1566,3 +1575,17 @@ def _add_and_move_fields(
     elif len(moves) > 0:
         return _move_fields(fields, moves)
     return None if len(adds) == 0 else tuple(*fields, *adds)
+
+
+def _generate_snapshot_id() -> int:
+    """Generate a new Snapshot ID from a UUID.
+
+    Returns: An 64 bit long
+    """
+    rnd_uuid = uuid.uuid4()
+    snapshot_id = int.from_bytes(
+        bytes(lhs ^ rhs for lhs, rhs in zip(rnd_uuid.bytes[0:8], rnd_uuid.bytes[8:16])), byteorder='little', signed=True
+    )
+    snapshot_id = snapshot_id if snapshot_id >= 0 else snapshot_id * -1
+
+    return snapshot_id

pyiceberg/table/__init__.py

@@ -52,7 +52,7 @@ requests = ">=2.20.0,<3.0.0"
 click = ">=7.1.1,<9.0.0"
 rich = ">=10.11.0,<14.0.0"
 strictyaml = ">=1.7.0,<2.0.0" # CVE-2020-14343 was fixed in 5.4.
-pydantic = ">=2.0,<3.0"
+pydantic = ">=2.0,<3.0,!=2.4.0,!=2.4.1" # 2.4.0, 2.4.1 has a critical bug
 sortedcontainers = "2.4.0"
 fsspec = ">=2023.1.0,<2024.1.0"
 pyparsing = ">=3.1.0,<4.0.0"
@@ -78,8 +78,8 @@ pre-commit = "3.4.0"
 fastavro = "1.8.3"
 coverage = { version = "^7.3.1", extras = ["toml"] }
 requests-mock = "1.11.0"
-moto = "^4.2.4"
-typing-extensions = "4.7.1"
+moto = "^4.2.5"
+typing-extensions = "4.8.0"
 pytest-mock = "3.11.1"
 cython = "3.0.2"
 

pyproject.toml

@@ -41,6 +41,7 @@
     StaticTable,
     Table,
     UpdateSchema,
+    _generate_snapshot_id,
     _match_deletes_to_datafile,
 )
 from pyiceberg.table.metadata import INITIAL_SEQUENCE_NUMBER
@@ -506,3 +507,8 @@ def test_add_nested_list_type_column(table: Table) -> None:
         element_required=False,
     )
     assert new_schema.highest_field_id == 7
+
+
+def test_generate_snapshot_id(table: Table) -> None:
+    assert isinstance(_generate_snapshot_id(), int)
+    assert isinstance(table.new_snapshot_id(), int)