Skip to content

Fix loading hdf5 files with new versions of pandas #514

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Apr 11, 2019
Merged

Fix loading hdf5 files with new versions of pandas #514

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
86d42bd
Stop passing format kwarg to pandas.read_hdf()
Apr 10, 2019
3df0ed0
Stops passing format kwarg to pandas.read_hdf()
nickponvert Apr 11, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 11 additions & 11 deletions visual_behavior/ophys/dataset/visual_behavior_ophys_dataset.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,13 +96,13 @@ def get_analysis_dir(self):
analysis_dir = LazyLoadable('_analysis_dir', get_analysis_dir)

def get_metadata(self):
self._metadata = pd.read_hdf(os.path.join(self.analysis_dir, 'metadata.h5'), key='df', format='fixed')
self._metadata = pd.read_hdf(os.path.join(self.analysis_dir, 'metadata.h5'), key='df')
return self._metadata

metadata = LazyLoadable('_metadata', get_metadata)

def get_timestamps(self):
self._timestamps = pd.read_hdf(os.path.join(self.analysis_dir, 'timestamps.h5'), key='df', format='fixed')
self._timestamps = pd.read_hdf(os.path.join(self.analysis_dir, 'timestamps.h5'), key='df')
return self._timestamps

timestamps = LazyLoadable('_timestamps', get_timestamps)
Expand All @@ -122,7 +122,7 @@ def get_timestamps_ophys(self):
def get_stimulus_table(self):
self._stimulus_table = pd.read_hdf(
os.path.join(self.analysis_dir, 'stimulus_table.h5'),
key='df', format='fixed'
key='df'
)
self._stimulus_table = self._stimulus_table.reset_index()
# self._stimulus_table = self._stimulus_table.drop(
Expand All @@ -146,42 +146,42 @@ def get_stimulus_template(self):
def get_stimulus_metadata(self):
self._stimulus_metadata = pd.read_hdf(
os.path.join(self.analysis_dir, 'stimulus_metadata.h5'),
key='df', format='fixed'
key='df'
)
self._stimulus_metadata = self._stimulus_metadata.drop(columns='image_category')
return self._stimulus_metadata

stimulus_metadata = LazyLoadable('_stimulus_metadata', get_stimulus_metadata)

def get_running_speed(self):
self._running_speed = pd.read_hdf(os.path.join(self.analysis_dir, 'running_speed.h5'), key='df', format='fixed')
self._running_speed = pd.read_hdf(os.path.join(self.analysis_dir, 'running_speed.h5'), key='df')
return self._running_speed

running_speed = LazyLoadable('_running_speed', get_running_speed)

def get_licks(self):
self._licks = pd.read_hdf(os.path.join(self.analysis_dir, 'licks.h5'), key='df', format='fixed')
self._licks = pd.read_hdf(os.path.join(self.analysis_dir, 'licks.h5'), key='df')
return self._licks

licks = LazyLoadable('_licks', get_licks)

def get_rewards(self):
self._rewards = pd.read_hdf(os.path.join(self.analysis_dir, 'rewards.h5'), key='df', format='fixed')
self._rewards = pd.read_hdf(os.path.join(self.analysis_dir, 'rewards.h5'), key='df')
return self._rewards

rewards = LazyLoadable('_rewards', get_rewards)

def get_task_parameters(self):
self._task_parameters = pd.read_hdf(
os.path.join(self.analysis_dir, 'task_parameters.h5'),
key='df', format='fixed'
key='df'
)
return self._task_parameters

task_parameters = LazyLoadable('_task_parameters', get_task_parameters)

def get_all_trials(self):
self._all_trials = pd.read_hdf(os.path.join(self.analysis_dir, 'trials.h5'), key='df', format='fixed')
self._all_trials = pd.read_hdf(os.path.join(self.analysis_dir, 'trials.h5'), key='df')
return self._all_trials

all_trials = LazyLoadable('_all_trials', get_all_trials)
Expand Down Expand Up @@ -255,7 +255,7 @@ def get_events(self):
events = LazyLoadable('_events', get_events)

def get_roi_metrics(self):
self._roi_metrics = pd.read_hdf(os.path.join(self.analysis_dir, 'roi_metrics.h5'), key='df', format='fixed')
self._roi_metrics = pd.read_hdf(os.path.join(self.analysis_dir, 'roi_metrics.h5'), key='df')
return self._roi_metrics

roi_metrics = LazyLoadable('_roi_metrics', get_roi_metrics)
Expand Down Expand Up @@ -300,7 +300,7 @@ def get_average_image(self):
def get_motion_correction(self):
self._motion_correction = pd.read_hdf(
os.path.join(self.analysis_dir, 'motion_correction.h5'),
key='df', format='fixed'
key='df'
)
return self._motion_correction

Expand Down
6 changes: 3 additions & 3 deletions visual_behavior/ophys/response_analysis/response_analysis.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ def get_trial_response_df(self):
else:
if os.path.exists(self.get_trial_response_df_path()):
print('loading trial response dataframe')
self.trial_response_df = pd.read_hdf(self.get_trial_response_df_path(), key='df', format='fixed')
self.trial_response_df = pd.read_hdf(self.get_trial_response_df_path(), key='df')
tdf = self.trial_response_df
tdf.cell = [int(cell) for cell in tdf.cell.values]
tdf.cell_specimen_id = [int(cell_specimen_id) for cell_specimen_id in tdf.cell_specimen_id.values]
Expand Down Expand Up @@ -247,7 +247,7 @@ def get_flash_response_df(self):
else:
if os.path.exists(self.get_flash_response_df_path()):
print('loading flash response dataframe')
self.flash_response_df = pd.read_hdf(self.get_flash_response_df_path(), key='df', format='fixed')
self.flash_response_df = pd.read_hdf(self.get_flash_response_df_path(), key='df')
fdf = self.flash_response_df
fdf.cell = [int(cell) for cell in fdf.cell.values]
fdf.cell_specimen_id = [int(cell_specimen_id) for cell_specimen_id in fdf.cell_specimen_id.values]
Expand Down Expand Up @@ -334,7 +334,7 @@ def get_omitted_flash_response_df(self):
else:
if os.path.exists(self.get_omitted_flash_response_df_path()):
print('loading omitted flash response dataframe')
self.omitted_flash_response_df = pd.read_hdf(self.get_omitted_flash_response_df_path(), key='df', format='fixed')
self.omitted_flash_response_df = pd.read_hdf(self.get_omitted_flash_response_df_path(), key='df')
fdf = self.omitted_flash_response_df
fdf.cell = [int(cell) for cell in fdf.cell.values]
fdf.cell_specimen_id = [int(cell_specimen_id) for cell_specimen_id in fdf.cell_specimen_id.values]
Expand Down