-
Notifications
You must be signed in to change notification settings - Fork 6
fix: support converting empty time
Series to pyarrow Array
#11
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
e793ec6
e08f7fc
281afb9
1613bb6
63b946c
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,163 @@ | ||
# Copyright 2021 Google LLC | ||
# | ||
# Licensed under the Apache License, Version 2.0 (the "License"); | ||
# you may not use this file except in compliance with the License. | ||
# You may obtain a copy of the License at | ||
# | ||
# http://www.apache.org/licenses/LICENSE-2.0 | ||
# | ||
# Unless required by applicable law or agreed to in writing, software | ||
# distributed under the License is distributed on an "AS IS" BASIS, | ||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
# See the License for the specific language governing permissions and | ||
# limitations under the License. | ||
|
||
import datetime as dt | ||
|
||
import pandas | ||
import pyarrow | ||
import pytest | ||
|
||
# To register the types. | ||
import db_dtypes # noqa | ||
|
||
|
||
@pytest.mark.parametrize( | ||
("series", "expected"), | ||
( | ||
(pandas.Series([], dtype="date"), pyarrow.array([], type=pyarrow.date32())), | ||
( | ||
pandas.Series([None, None, None], dtype="date"), | ||
pyarrow.array([None, None, None], type=pyarrow.date32()), | ||
), | ||
( | ||
pandas.Series( | ||
[dt.date(2021, 9, 27), None, dt.date(2011, 9, 27)], dtype="date" | ||
), | ||
pyarrow.array( | ||
[dt.date(2021, 9, 27), None, dt.date(2011, 9, 27)], | ||
type=pyarrow.date32(), | ||
), | ||
), | ||
( | ||
pandas.Series( | ||
[dt.date(1677, 9, 22), dt.date(1970, 1, 1), dt.date(2262, 4, 11)], | ||
dtype="date", | ||
), | ||
pyarrow.array( | ||
[dt.date(1677, 9, 22), dt.date(1970, 1, 1), dt.date(2262, 4, 11)], | ||
type=pyarrow.date32(), | ||
), | ||
), | ||
(pandas.Series([], dtype="time"), pyarrow.array([], type=pyarrow.time64("ns"))), | ||
( | ||
pandas.Series([None, None, None], dtype="time"), | ||
pyarrow.array([None, None, None], type=pyarrow.time64("ns")), | ||
), | ||
( | ||
pandas.Series( | ||
[dt.time(0, 0, 0, 0), None, dt.time(23, 59, 59, 999_999)], dtype="time" | ||
), | ||
pyarrow.array( | ||
[dt.time(0, 0, 0, 0), None, dt.time(23, 59, 59, 999_999)], | ||
type=pyarrow.time64("ns"), | ||
), | ||
), | ||
( | ||
pandas.Series( | ||
[ | ||
dt.time(0, 0, 0, 0), | ||
dt.time(12, 30, 15, 125_000), | ||
dt.time(23, 59, 59, 999_999), | ||
], | ||
dtype="time", | ||
), | ||
pyarrow.array( | ||
[ | ||
dt.time(0, 0, 0, 0), | ||
dt.time(12, 30, 15, 125_000), | ||
dt.time(23, 59, 59, 999_999), | ||
], | ||
type=pyarrow.time64("ns"), | ||
), | ||
), | ||
), | ||
) | ||
def test_to_arrow(series, expected): | ||
array = pyarrow.array(series) | ||
assert array.equals(expected) | ||
|
||
|
||
@pytest.mark.parametrize( | ||
("series", "expected"), | ||
( | ||
(pandas.Series([], dtype="date"), pyarrow.array([], type=pyarrow.date64())), | ||
( | ||
pandas.Series([None, None, None], dtype="date"), | ||
pyarrow.array([None, None, None], type=pyarrow.date64()), | ||
), | ||
( | ||
pandas.Series( | ||
[dt.date(2021, 9, 27), None, dt.date(2011, 9, 27)], dtype="date" | ||
), | ||
pyarrow.array( | ||
[dt.date(2021, 9, 27), None, dt.date(2011, 9, 27)], | ||
type=pyarrow.date64(), | ||
), | ||
), | ||
( | ||
pandas.Series( | ||
[dt.date(1677, 9, 22), dt.date(1970, 1, 1), dt.date(2262, 4, 11)], | ||
dtype="date", | ||
), | ||
pyarrow.array( | ||
[dt.date(1677, 9, 22), dt.date(1970, 1, 1), dt.date(2262, 4, 11)], | ||
type=pyarrow.date64(), | ||
), | ||
), | ||
(pandas.Series([], dtype="time"), pyarrow.array([], type=pyarrow.time32("ms"))), | ||
( | ||
pandas.Series([None, None, None], dtype="time"), | ||
pyarrow.array([None, None, None], type=pyarrow.time32("ms")), | ||
), | ||
( | ||
pandas.Series( | ||
[dt.time(0, 0, 0, 0), None, dt.time(23, 59, 59, 999_000)], dtype="time" | ||
), | ||
pyarrow.array( | ||
[dt.time(0, 0, 0, 0), None, dt.time(23, 59, 59, 999_000)], | ||
type=pyarrow.time32("ms"), | ||
), | ||
), | ||
( | ||
pandas.Series( | ||
[dt.time(0, 0, 0, 0), None, dt.time(23, 59, 59, 999_999)], dtype="time" | ||
), | ||
pyarrow.array( | ||
[dt.time(0, 0, 0, 0), None, dt.time(23, 59, 59, 999_999)], | ||
type=pyarrow.time64("us"), | ||
), | ||
), | ||
( | ||
pandas.Series( | ||
[ | ||
dt.time(0, 0, 0, 0), | ||
dt.time(12, 30, 15, 125_000), | ||
dt.time(23, 59, 59, 999_999), | ||
], | ||
dtype="time", | ||
), | ||
pyarrow.array( | ||
[ | ||
dt.time(0, 0, 0, 0), | ||
dt.time(12, 30, 15, 125_000), | ||
dt.time(23, 59, 59, 999_999), | ||
], | ||
type=pyarrow.time64("us"), | ||
), | ||
), | ||
), | ||
) | ||
def test_to_arrow_w_arrow_type(series, expected): | ||
array = pyarrow.array(series, type=expected.type) | ||
assert array.equals(expected) | ||
Comment on lines
+1
to
+163
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Your tests are nicer than mine. :) There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Thanks! They did catch a bug with empty arrays, so I'm glad I wrote them |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -15,7 +15,6 @@ | |
import datetime | ||
|
||
import packaging.version | ||
import pyarrow.lib | ||
import pytest | ||
|
||
pd = pytest.importorskip("pandas") | ||
|
@@ -670,13 +669,3 @@ def test_bad_time_parsing(value, error): | |
def test_bad_date_parsing(value, error): | ||
with pytest.raises(ValueError, match=error): | ||
_cls("date")([value]) | ||
|
||
|
||
@for_date_and_time | ||
def test_date___arrow__array__(dtype): | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Removed because it's redundant with the new |
||
a = _make_one(dtype) | ||
ar = a.__arrow_array__() | ||
assert isinstance( | ||
ar, pyarrow.Date32Array if dtype == "date" else pyarrow.Time64Array, | ||
) | ||
assert [v.as_py() for v in ar] == list(a) |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This might have performance implications, but it does seem to prevent the cast to float64 for empty arrays. Also, the
dtype
seems to beobject
whenever there are any values in the array, anyway.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I approached this by adding the missing
to_numpy()
for pandas <1, that just usesastype('object')
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Sweet. That seems to have done the trick. I copied the implementation from your PR https://github.com/googleapis/python-db-dtypes-pandas/pull/9/files#diff-1956943f14005805ef968dfc37c26fe3eee995786f62a1c66dee5a29d9b1a251R111