Skip to content

DEPR: Remove Panel-specific parts of io.pytables #25233

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Feb 8, 2019
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion doc/source/whatsnew/v0.25.0.rst
Original file line number Diff line number Diff line change
@@ -51,7 +51,7 @@ Deprecations

Removal of prior version deprecations/changes
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
- Removed (parts of) :class:`Panel` (:issue:`25047`)
- Removed (parts of) :class:`Panel` (:issue:`25047`,:issue:`25191`,:issue:`25231`)
-
-
-
2 changes: 1 addition & 1 deletion pandas/core/internals/__init__.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
from .blocks import ( # noqa:F401
_block2d_to_blocknd, _factor_indexer, _block_shape, # io.pytables
_block_shape, # io.pytables
_safe_reshape, # io.packers
make_block, # io.pytables, io.packers
FloatBlock, IntBlock, ComplexBlock, BoolBlock, ObjectBlock,
35 changes: 0 additions & 35 deletions pandas/core/internals/blocks.py
Original file line number Diff line number Diff line change
@@ -3134,31 +3134,6 @@ def _merge_blocks(blocks, dtype=None, _can_consolidate=True):
return blocks


def _block2d_to_blocknd(values, placement, shape, labels, ref_items):
""" pivot to the labels shape """
panel_shape = (len(placement),) + shape

# TODO: lexsort depth needs to be 2!!

# Create observation selection vector using major and minor
# labels, for converting to panel format.
selector = _factor_indexer(shape[1:], labels)
mask = np.zeros(np.prod(shape), dtype=bool)
mask.put(selector, True)

if mask.all():
pvalues = np.empty(panel_shape, dtype=values.dtype)
else:
dtype, fill_value = maybe_promote(values.dtype)
pvalues = np.empty(panel_shape, dtype=dtype)
pvalues.fill(fill_value)

for i in range(len(placement)):
pvalues[i].flat[mask] = values[:, i]

return make_block(pvalues, placement=placement)


def _safe_reshape(arr, new_shape):
"""
If possible, reshape `arr` to have shape `new_shape`,
@@ -3181,16 +3156,6 @@ def _safe_reshape(arr, new_shape):
return arr


def _factor_indexer(shape, labels):
"""
given a tuple of shape and a list of Categorical labels, return the
expanded label indexer
"""
mult = np.array(shape)[::-1].cumprod()[::-1]
return ensure_platform_int(
np.sum(np.array(labels).T * np.append(mult, [1]), axis=1).T)


def _putmask_smart(v, m, n):
"""
Return a new ndarray, try to preserve dtype if possible.
121 changes: 13 additions & 108 deletions pandas/io/pytables.py
Original file line number Diff line number Diff line change
@@ -15,34 +15,29 @@

import numpy as np

from pandas._libs import algos, lib, writers as libwriters
from pandas._libs import lib, writers as libwriters
from pandas._libs.tslibs import timezones
from pandas.compat import PY3, filter, lrange, range, string_types
from pandas.errors import PerformanceWarning

from pandas.core.dtypes.common import (
ensure_int64, ensure_object, ensure_platform_int, is_categorical_dtype,
is_datetime64_dtype, is_datetime64tz_dtype, is_list_like,
is_timedelta64_dtype)
ensure_object, is_categorical_dtype, is_datetime64_dtype,
is_datetime64tz_dtype, is_list_like, is_timedelta64_dtype)
from pandas.core.dtypes.missing import array_equivalent

from pandas import (
DataFrame, DatetimeIndex, Index, Int64Index, MultiIndex, Panel,
PeriodIndex, Series, SparseDataFrame, SparseSeries, TimedeltaIndex, compat,
concat, isna, to_datetime)
DataFrame, DatetimeIndex, Index, Int64Index, MultiIndex, PeriodIndex,
Series, SparseDataFrame, SparseSeries, TimedeltaIndex, compat, concat,
isna, to_datetime)
from pandas.core import config
from pandas.core.algorithms import unique
from pandas.core.arrays.categorical import (
Categorical, _factorize_from_iterables)
from pandas.core.arrays.categorical import Categorical
from pandas.core.arrays.sparse import BlockIndex, IntIndex
from pandas.core.base import StringMixin
import pandas.core.common as com
from pandas.core.computation.pytables import Expr, maybe_expression
from pandas.core.config import get_option
from pandas.core.index import ensure_index
from pandas.core.internals import (
BlockManager, _block2d_to_blocknd, _block_shape, _factor_indexer,
make_block)
from pandas.core.internals import BlockManager, _block_shape, make_block

from pandas.io.common import _stringify_path
from pandas.io.formats.printing import adjoin, pprint_thing
@@ -175,7 +170,6 @@ class DuplicateWarning(Warning):
SparseSeries: u'sparse_series',
DataFrame: u'frame',
SparseDataFrame: u'sparse_frame',
Panel: u'wide',
}

# storer class map
@@ -187,7 +181,6 @@ class DuplicateWarning(Warning):
u'sparse_series': 'SparseSeriesFixed',
u'frame': 'FrameFixed',
u'sparse_frame': 'SparseFrameFixed',
u'wide': 'PanelFixed',
}

# table class map
@@ -198,14 +191,11 @@ class DuplicateWarning(Warning):
u'appendable_frame': 'AppendableFrameTable',
u'appendable_multiframe': 'AppendableMultiFrameTable',
u'worm': 'WORMTable',
u'legacy_frame': 'LegacyFrameTable',
u'legacy_panel': 'LegacyPanelTable',
}

# axes map
_AXES_MAP = {
DataFrame: [0],
Panel: [1, 2]
}

# register our configuration options
@@ -864,7 +854,7 @@ def put(self, key, value, format=None, append=False, **kwargs):
Parameters
----------
key : object
value : {Series, DataFrame, Panel}
value : {Series, DataFrame}
format : 'fixed(f)|table(t)', default is 'fixed'
fixed(f) : Fixed format
Fast writing/reading. Not-appendable, nor searchable
@@ -946,7 +936,7 @@ def append(self, key, value, format=None, append=True, columns=None,
Parameters
----------
key : object
value : {Series, DataFrame, Panel}
value : {Series, DataFrame}
format : 'table' is the default
table(t) : table format
Write as a PyTables Table structure which may perform
@@ -3027,16 +3017,6 @@ class FrameFixed(BlockManagerFixed):
obj_type = DataFrame


class PanelFixed(BlockManagerFixed):
pandas_kind = u'wide'
obj_type = Panel
is_shape_reversed = True

def write(self, obj, **kwargs):
obj._consolidate_inplace()
return super(PanelFixed, self).write(obj, **kwargs)


class Table(Fixed):

""" represent a table:
@@ -3899,85 +3879,11 @@ def read(self, where=None, columns=None, **kwargs):
if not self.read_axes(where=where, **kwargs):
return None

lst_vals = [a.values for a in self.index_axes]
labels, levels = _factorize_from_iterables(lst_vals)
# labels and levels are tuples but lists are expected
labels = list(labels)
levels = list(levels)
N = [len(lvl) for lvl in levels]

# compute the key
key = _factor_indexer(N[1:], labels)

objs = []
if len(unique(key)) == len(key):

sorter, _ = algos.groupsort_indexer(
ensure_int64(key), np.prod(N))
sorter = ensure_platform_int(sorter)

# create the objs
for c in self.values_axes:

# the data need to be sorted
sorted_values = c.take_data().take(sorter, axis=0)
if sorted_values.ndim == 1:
sorted_values = sorted_values.reshape(
(sorted_values.shape[0], 1))

take_labels = [l.take(sorter) for l in labels]
items = Index(c.values)
block = _block2d_to_blocknd(
values=sorted_values, placement=np.arange(len(items)),
shape=tuple(N), labels=take_labels, ref_items=items)

# create the object
mgr = BlockManager([block], [items] + levels)
obj = self.obj_type(mgr)

# permute if needed
if self.is_transposed:
obj = obj.transpose(
*tuple(Series(self.data_orientation).argsort()))

objs.append(obj)

else:
raise NotImplementedError("Panel is removed in pandas 0.25.0")

# create the composite object
if len(objs) == 1:
wp = objs[0]
else:
wp = concat(objs, axis=0, verify_integrity=False)._consolidate()

# apply the selection filters & axis orderings
wp = self.process_axes(wp, columns=columns)

return wp


class LegacyFrameTable(LegacyTable):

""" support the legacy frame table """
pandas_kind = u'frame_table'
table_type = u'legacy_frame'
obj_type = Panel

def read(self, *args, **kwargs):
return super(LegacyFrameTable, self).read(*args, **kwargs)['value']


class LegacyPanelTable(LegacyTable):

""" support the legacy panel table """
table_type = u'legacy_panel'
obj_type = Panel
raise NotImplementedError("Panel is removed in pandas 0.25.0")


class AppendableTable(LegacyTable):

""" suppor the new appendable table formats """
""" support the new appendable table formats """
_indexables = None
table_type = u'appendable'

@@ -4209,8 +4115,7 @@ def delete(self, where=None, start=None, stop=None, **kwargs):


class AppendableFrameTable(AppendableTable):

""" suppor the new appendable table formats """
""" support the new appendable table formats """
pandas_kind = u'frame_table'
table_type = u'appendable_frame'
ndim = 2
Binary file removed pandas/tests/io/data/legacy_hdf/legacy_table.h5
Binary file not shown.
25 changes: 0 additions & 25 deletions pandas/tests/io/test_pytables.py
Original file line number Diff line number Diff line change
@@ -141,7 +141,6 @@ def teardown_method(self, method):


@pytest.mark.single
@pytest.mark.filterwarnings("ignore:\\nPanel:FutureWarning")
class TestHDFStore(Base):

def test_format_kwarg_in_constructor(self):
@@ -3984,30 +3983,6 @@ def test_legacy_table_read_py2(self, datapath):
})
assert_frame_equal(expected, result)

def test_legacy_table_read(self, datapath):
# legacy table types
with ensure_clean_store(
datapath('io', 'data', 'legacy_hdf', 'legacy_table.h5'),
mode='r') as store:

with catch_warnings():
simplefilter("ignore", pd.io.pytables.IncompatibilityWarning)
store.select('df1')
store.select('df2')
store.select('wp1')

# force the frame
store.select('df2', typ='legacy_frame')

# old version warning
pytest.raises(
Exception, store.select, 'wp1', 'minor_axis=B')

df2 = store.select('df2')
result = store.select('df2', 'index>df2.index[2]')
expected = df2[df2.index > df2.index[2]]
assert_frame_equal(expected, result)

def test_copy(self):

with catch_warnings(record=True):