allowed to save and load Axis and Group objects in HDF files (#578) :

- added to_hdf method to Axis and Group
- updated read_hdf (inout/hdf.py)
- updated documentation of Session's methods
- updated doctests of Session.load and Session.save
- added context manager LHDFStore (utils/misc.py)

refactored package inout: created one module per file extension or external object type like in pandas/io/:

new modules:
  - common.py
  - pandas.py
  - csv.py
  - excel.py
  - hdf.py
  - sas.py
  - misc.py
  - pickle.py

renamed modules:
  - excel.py --> xw_excel.py

deleted modules:
  - array.py

Author: alixdamman

doc/source/api.rst

@@ -77,6 +77,14 @@ Testing
    Axis.iscompatible
    Axis.equals
 
+Save
+----
+
+.. autosummary::
+   :toctree: _generated/
+
+   Axis.to_hdf
+
 .. _api-group:
 
 Group
@@ -104,6 +112,7 @@ IGroup
    IGroup.startingwith
    IGroup.endingwith
    IGroup.matching
+   IGroup.to_hdf
 
 LGroup
 ------
@@ -127,6 +136,7 @@ LGroup
    LGroup.startingwith
    LGroup.endingwith
    LGroup.matching
+   LGroup.to_hdf
 
 .. _api-set:
 

doc/source/changes/version_0_29.rst.inc

@@ -23,7 +23,25 @@ New features
 Miscellaneous improvements
 --------------------------
 
-* improved something.
+* saving or loading a session from a file now includes `Axis` and `Group` objects in addition to arrays
+  (closes :issue:`578`):
+
+    Create a session containing axes, groups and arrays
+
+    >>> a, b = Axis("a=a0..a2"), Axis("b=b0..b2")
+    >>> a01 = a['a0,a1'] >> 'a01'
+    >>> arr1, arr2 = ndtest((a, b)), ndtest(a)
+    >>> s = Session([('a', a), ('b', b), ('a01', a01), ('arr1', arr1), ('arr2', arr2)])
+
+    Saving a session will save axes, groups and arrays
+
+    >>> s.save('session.h5')
+
+    Loading a session will load axes, groups and arrays
+
+    >>> s2 = s.load('session.h5')
+    >>> s2
+    Session(arr1, arr2, a, b, a01)
 
 
 Fixes

larray/core/array.py

@@ -64,11 +64,11 @@
 from larray.core.abstractbases import ABCLArray
 from larray.core.expr import ExprNode
 from larray.core.group import (Group, IGroup, LGroup, remove_nested_groups, _to_key, _to_keys,
-                               _range_to_slice, _translate_sheet_name, _translate_key_hdf)
+                               _range_to_slice, _translate_sheet_name, _translate_group_key_hdf)
 from larray.core.axis import Axis, AxisReference, AxisCollection, X, _make_axis
 from larray.util.misc import (table2str, size2str, basestring, izip, rproduct, ReprString, duplicates,
                               float_error_handler_factory, _isnoneslice, light_product, unique_list, common_type,
-                              renamed_to, deprecate_kwarg)
+                              renamed_to, deprecate_kwarg, LHDFStore)
 
 
 nan = np.nan
@@ -5997,7 +5997,7 @@ def to_csv(self, filepath, sep=',', na_rep='', wide=True, value_name='value', dr
             series = self.to_series(value_name, dropna is not None)
             series.to_csv(filepath, sep=sep, na_rep=na_rep, header=True, **kwargs)
 
-    def to_hdf(self, filepath, key, *args, **kwargs):
+    def to_hdf(self, filepath, key):
         """
         Writes array to a HDF file.
 
@@ -6009,17 +6009,31 @@ def to_hdf(self, filepath, key, *args, **kwargs):
         filepath : str
             Path where the hdf file has to be written.
         key : str or Group
-            Name of the array within the HDF file.
-        *args
-        **kargs
+            Key (path) of the array within the HDF file (see Notes below).
+
+        Notes
+        -----
+        Objects stored in a HDF file can be grouped together in `HDF groups`.
+        If an object 'my_obj' is stored in a HDF group 'my_group',
+        the key associated with this object is then 'my_group/my_obj'.
+        Be aware that a HDF group can have subgroups.
 
         Examples
         --------
         >>> a = ndtest((2, 3))
-        >>> a.to_hdf('test.h5', 'a')  # doctest: +SKIP
+
+        Save an array
+
+        >>> a.to_hdf('test.h5', 'a')          # doctest: +SKIP
+
+        Save an array in a specific HDF group
+
+        >>> a.to_hdf('test.h5', 'arrays/a')  # doctest: +SKIP
         """
-        key = _translate_key_hdf(key)
-        self.to_frame().to_hdf(filepath, key, *args, **kwargs)
+        key = _translate_group_key_hdf(key)
+        with LHDFStore(filepath) as store:
+            store.put(key, self.to_frame())
+            store.get_storer(key).attrs.type = 'Array'
 
     @deprecate_kwarg('sheet_name', 'sheet') 
     def to_excel(self, filepath=None, sheet=None, position='A1', overwrite_file=False, clear_sheet=False,
@@ -6085,7 +6099,7 @@ def to_excel(self, filepath=None, sheet=None, position='A1', overwrite_file=Fals
             engine = 'xlwings' if xw is not None else None
 
         if engine == 'xlwings':
-            from larray.inout.excel import open_excel
+            from larray.inout.xw_excel import open_excel
 
             close = False
             new_workbook = False
@@ -7022,7 +7036,7 @@ def aslarray(a):
     elif hasattr(a, '__larray__'):
         return a.__larray__()
     elif isinstance(a, pd.DataFrame):
-        from larray.inout.array import from_frame
+        from larray.inout.pandas import from_frame
         return from_frame(a)
     else:
         return LArray(a)

larray/core/axis.py

@@ -7,14 +7,15 @@
 from itertools import product
 
 import numpy as np
+import pandas as pd
 
 from larray.core.abstractbases import ABCAxis, ABCAxisReference, ABCLArray
 from larray.core.expr import ExprNode
 from larray.core.group import (Group, LGroup, IGroup, IGroupMaker, _to_tick, _to_ticks, _to_key, _seq_summary,
-                               _contain_group_ticks, _seq_group_to_name)
+                               _contain_group_ticks, _seq_group_to_name, _translate_group_key_hdf)
 from larray.util.oset import *
 from larray.util.misc import (basestring, PY2, unicode, long, duplicates, array_lookup2, ReprString, index_by_id,
-                              renamed_to, common_type)
+                              renamed_to, common_type, LHDFStore)
 
 __all__ = ['Axis', 'AxisCollection', 'X', 'x']
 
@@ -1194,6 +1195,57 @@ def align(self, other, join='outer'):
                 other = Axis(other)
             return other
 
+    def to_hdf(self, filepath, key=None):
+        """
+        Writes axis to a HDF file.
+
+        A HDF file can contain multiple axes.
+        The 'key' parameter is a unique identifier for the axis.
+
+        Parameters
+        ----------
+        filepath : str
+            Path where the hdf file has to be written.
+        key : str or Group, optional
+            Key (path) of the axis within the HDF file (see Notes below).
+            If None, the name of the axis is used.
+            Defaults to None.
+
+        Notes
+        -----
+        Objects stored in a HDF file can be grouped together in `HDF groups`.
+        If an object 'my_obj' is stored in a HDF group 'my_group',
+        the key associated with this object is then 'my_group/my_obj'.
+        Be aware that a HDF group can have subgroups.
+
+        Examples
+        --------
+        >>> a = Axis("a=a0..a2")
+
+        Save axis
+
+        >>> # by default, the key is the name of the axis
+        >>> a.to_hdf('test.h5')            # doctest: +SKIP
+
+        Save axis with a specific key
+
+        >>> a.to_hdf('test.h5', 'a')       # doctest: +SKIP
+
+        Save axis in a specific HDF group
+
+        >>> a.to_hdf('test.h5', 'axes/a')  # doctest: +SKIP
+        """
+        if key is None:
+            if self.name is None:
+                raise ValueError("Argument key must be provided explicitly in case of anonymous axis")
+            key = self.name
+        key = _translate_group_key_hdf(key)
+        s = pd.Series(data=self.labels, name=self.name)
+        with LHDFStore(filepath) as store:
+            store.put(key, s)
+            store.get_storer(key).attrs.type = 'Axis'
+            store.get_storer(key).attrs.wildcard = self.iswildcard
+
 
 def _make_axis(obj):
     if isinstance(obj, Axis):

larray/core/group.py

@@ -11,7 +11,8 @@
 
 from larray.core.abstractbases import ABCAxis, ABCAxisReference, ABCLArray
 from larray.util.oset import *
-from larray.util.misc import basestring, PY2, unique, find_closing_chr, _parse_bound, _seq_summary, renamed_to
+from larray.util.misc import (basestring, PY2, unique, find_closing_chr, _parse_bound, _seq_summary,
+                              renamed_to, LHDFStore)
 
 __all__ = ['Group', 'LGroup', 'LSet', 'IGroup', 'union']
 
@@ -652,7 +653,7 @@ def _translate_sheet_name(sheet_name):
 _key_hdf_pattern = re.compile('[\\\/]')
 
 
-def _translate_key_hdf(key):
+def _translate_group_key_hdf(key):
     if isinstance(key, Group):
         key = _key_hdf_pattern.sub('_', str(_to_tick(key)))
     return key
@@ -1275,6 +1276,80 @@ def containing(self, substring):
             substring = substring.eval()
         return LGroup([v for v in self.eval() if substring in v], axis=self.axis)
 
+    def to_hdf(self, filepath, key=None, axis_key=None):
+        """
+        Writes group to a HDF file.
+
+        A HDF file can contain multiple groups.
+        The 'key' parameter is a unique identifier for the group.
+        The 'axis_key' parameter is the unique identifier for the associated axis.
+        The associated axis will be saved if not already present in the HDF file.
+
+        Parameters
+        ----------
+        filepath : str
+            Path where the hdf file has to be written.
+        key : str or Group, optional
+            Key (path) of the group within the HDF file (see Notes below).
+            If None, the name of the group is used.
+            Defaults to None.
+        axis_key : str, optional
+            Key (path) of the associated axis in the HDF file (see Notes below).
+            If None, the name of the axis associated with the group is used.
+            Defaults to None.
+
+        Notes
+        -----
+        Objects stored in a HDF file can be grouped together in `HDF groups`.
+        If an object 'my_obj' is stored in a HDF group 'my_group',
+        the key associated with this object is then 'my_group/my_obj'.
+        Be aware that a HDF group can have subgroups.
+
+        Examples
+        --------
+        >>> from larray import Axis
+        >>> a = Axis("a=a0..a2")
+        >>> a.to_hdf('test.h5')
+        >>> a01 = a['a0,a1'] >> 'a01'
+
+        Save group
+
+        >>> # by default, the key is the name of the group
+        >>> # and axis_key the name of the associated axis
+        >>> a01.to_hdf('test.h5')                  # doctest: +SKIP
+
+        Save group with a specific key
+
+        >>> a01.to_hdf('test.h5', 'a_01')           # doctest: +SKIP
+
+        Save group in a specific HDF group
+
+        >>> a.to_hdf('test.h5', 'groups/a01')       # doctest: +SKIP
+
+        The associated axis is saved with the group if not already present in the HDF file
+
+        >>> b = Axis("b=b0..b2")
+        >>> b01 = b['b0,b1'] >> 'b01'
+        >>> # save both the group 'b01' and the associated axis 'b'
+        >>> b01.to_hdf('test.h5')                   # doctest: +SKIP
+        """
+        if key is None:
+            if self.name is None:
+                raise ValueError("Argument key must be provided explicitly in case of anonymous group")
+            key = self.name
+        key = _translate_group_key_hdf(key)
+        if axis_key is None:
+            if self.axis.name is None:
+                raise ValueError("Argument axis_key must be provided explicitly if the associated axis is anonymous")
+            axis_key = self.axis.name
+        s = pd.Series(data=self.eval(), name=self.name)
+        with LHDFStore(filepath) as store:
+            store.put(key, s)
+            store.get_storer(key).attrs.type = 'Group'
+            if axis_key not in store:
+                self.axis.to_hdf(store, key=axis_key)
+            store.get_storer(key).attrs.axis_key = axis_key
+
     # this makes range(LGroup(int)) possible
     def __index__(self):
         return self.eval().__index__()