From 86d42bd65de8bc5b1e9207bb386c1f8c0cf88de8 Mon Sep 17 00:00:00 2001 From: Nick Ponvert Date: Wed, 10 Apr 2019 13:32:50 -0700 Subject: [PATCH 1/2] Stop passing format kwarg to pandas.read_hdf() --- .../dataset/visual_behavior_ophys_dataset.py | 45 ++++++++++++++----- .../response_analysis/response_analysis.py | 9 ++-- 2 files changed, 40 insertions(+), 14 deletions(-) diff --git a/visual_behavior/ophys/dataset/visual_behavior_ophys_dataset.py b/visual_behavior/ophys/dataset/visual_behavior_ophys_dataset.py index 2e4a134a2..f9872380a 100644 --- a/visual_behavior/ophys/dataset/visual_behavior_ophys_dataset.py +++ b/visual_behavior/ophys/dataset/visual_behavior_ophys_dataset.py @@ -96,13 +96,15 @@ def get_analysis_dir(self): analysis_dir = LazyLoadable('_analysis_dir', get_analysis_dir) def get_metadata(self): - self._metadata = pd.read_hdf(os.path.join(self.analysis_dir, 'metadata.h5'), key='df', format='fixed') + # self._metadata = pd.read_hdf(os.path.join(self.analysis_dir, 'metadata.h5'), key='df', format='fixed') + self._metadata = pd.read_hdf(os.path.join(self.analysis_dir, 'metadata.h5'), key='df') return self._metadata metadata = LazyLoadable('_metadata', get_metadata) def get_timestamps(self): - self._timestamps = pd.read_hdf(os.path.join(self.analysis_dir, 'timestamps.h5'), key='df', format='fixed') + # self._timestamps = pd.read_hdf(os.path.join(self.analysis_dir, 'timestamps.h5'), key='df', format='fixed') + self._timestamps = pd.read_hdf(os.path.join(self.analysis_dir, 'timestamps.h5'), key='df') return self._timestamps timestamps = LazyLoadable('_timestamps', get_timestamps) @@ -120,9 +122,13 @@ def get_timestamps_ophys(self): timestamps_ophys = LazyLoadable('_timestamps_ophys', get_timestamps_ophys) def get_stimulus_table(self): + # self._stimulus_table = pd.read_hdf( + # os.path.join(self.analysis_dir, 'stimulus_table.h5'), + # key='df', format='fixed' + # ) self._stimulus_table = pd.read_hdf( os.path.join(self.analysis_dir, 'stimulus_table.h5'), - key='df', format='fixed' + key='df' ) self._stimulus_table = self._stimulus_table.reset_index() # self._stimulus_table = self._stimulus_table.drop( @@ -144,9 +150,13 @@ def get_stimulus_template(self): stimulus_template = LazyLoadable('_stimulus_template', get_stimulus_template) def get_stimulus_metadata(self): + # self._stimulus_metadata = pd.read_hdf( + # os.path.join(self.analysis_dir, 'stimulus_metadata.h5'), + # key='df', format='fixed' + # ) self._stimulus_metadata = pd.read_hdf( os.path.join(self.analysis_dir, 'stimulus_metadata.h5'), - key='df', format='fixed' + key='df' ) self._stimulus_metadata = self._stimulus_metadata.drop(columns='image_category') return self._stimulus_metadata @@ -154,34 +164,42 @@ def get_stimulus_metadata(self): stimulus_metadata = LazyLoadable('_stimulus_metadata', get_stimulus_metadata) def get_running_speed(self): - self._running_speed = pd.read_hdf(os.path.join(self.analysis_dir, 'running_speed.h5'), key='df', format='fixed') + # self._running_speed = pd.read_hdf(os.path.join(self.analysis_dir, 'running_speed.h5'), key='df', format='fixed') + self._running_speed = pd.read_hdf(os.path.join(self.analysis_dir, 'running_speed.h5'), key='df') return self._running_speed running_speed = LazyLoadable('_running_speed', get_running_speed) def get_licks(self): - self._licks = pd.read_hdf(os.path.join(self.analysis_dir, 'licks.h5'), key='df', format='fixed') + # self._licks = pd.read_hdf(os.path.join(self.analysis_dir, 'licks.h5'), key='df', format='fixed') + self._licks = pd.read_hdf(os.path.join(self.analysis_dir, 'licks.h5'), key='df') return self._licks licks = LazyLoadable('_licks', get_licks) def get_rewards(self): - self._rewards = pd.read_hdf(os.path.join(self.analysis_dir, 'rewards.h5'), key='df', format='fixed') + # self._rewards = pd.read_hdf(os.path.join(self.analysis_dir, 'rewards.h5'), key='df', format='fixed') + self._rewards = pd.read_hdf(os.path.join(self.analysis_dir, 'rewards.h5'), key='df') return self._rewards rewards = LazyLoadable('_rewards', get_rewards) def get_task_parameters(self): + # self._task_parameters = pd.read_hdf( + # os.path.join(self.analysis_dir, 'task_parameters.h5'), + # key='df', format='fixed' + # ) self._task_parameters = pd.read_hdf( os.path.join(self.analysis_dir, 'task_parameters.h5'), - key='df', format='fixed' + key='df' ) return self._task_parameters task_parameters = LazyLoadable('_task_parameters', get_task_parameters) def get_all_trials(self): - self._all_trials = pd.read_hdf(os.path.join(self.analysis_dir, 'trials.h5'), key='df', format='fixed') + # self._all_trials = pd.read_hdf(os.path.join(self.analysis_dir, 'trials.h5'), key='df', format='fixed') + self._all_trials = pd.read_hdf(os.path.join(self.analysis_dir, 'trials.h5'), key='df') return self._all_trials all_trials = LazyLoadable('_all_trials', get_all_trials) @@ -255,7 +273,8 @@ def get_events(self): events = LazyLoadable('_events', get_events) def get_roi_metrics(self): - self._roi_metrics = pd.read_hdf(os.path.join(self.analysis_dir, 'roi_metrics.h5'), key='df', format='fixed') + # self._roi_metrics = pd.read_hdf(os.path.join(self.analysis_dir, 'roi_metrics.h5'), key='df', format='fixed') + self._roi_metrics = pd.read_hdf(os.path.join(self.analysis_dir, 'roi_metrics.h5'), key='df') #No support for format=fixed return self._roi_metrics roi_metrics = LazyLoadable('_roi_metrics', get_roi_metrics) @@ -298,9 +317,13 @@ def get_average_image(self): average_image = LazyLoadable('_average_image', get_average_image) def get_motion_correction(self): + # self._motion_correction = pd.read_hdf( + # os.path.join(self.analysis_dir, 'motion_correction.h5'), + # key='df', format='fixed' + # ) self._motion_correction = pd.read_hdf( os.path.join(self.analysis_dir, 'motion_correction.h5'), - key='df', format='fixed' + key='df' ) return self._motion_correction diff --git a/visual_behavior/ophys/response_analysis/response_analysis.py b/visual_behavior/ophys/response_analysis/response_analysis.py index f8d765484..e0940e802 100644 --- a/visual_behavior/ophys/response_analysis/response_analysis.py +++ b/visual_behavior/ophys/response_analysis/response_analysis.py @@ -132,7 +132,8 @@ def get_trial_response_df(self): else: if os.path.exists(self.get_trial_response_df_path()): print('loading trial response dataframe') - self.trial_response_df = pd.read_hdf(self.get_trial_response_df_path(), key='df', format='fixed') + # self.trial_response_df = pd.read_hdf(self.get_trial_response_df_path(), key='df', format='fixed') + self.trial_response_df = pd.read_hdf(self.get_trial_response_df_path(), key='df') tdf = self.trial_response_df tdf.cell = [int(cell) for cell in tdf.cell.values] tdf.cell_specimen_id = [int(cell_specimen_id) for cell_specimen_id in tdf.cell_specimen_id.values] @@ -247,7 +248,8 @@ def get_flash_response_df(self): else: if os.path.exists(self.get_flash_response_df_path()): print('loading flash response dataframe') - self.flash_response_df = pd.read_hdf(self.get_flash_response_df_path(), key='df', format='fixed') + # self.flash_response_df = pd.read_hdf(self.get_flash_response_df_path(), key='df', format='fixed') + self.flash_response_df = pd.read_hdf(self.get_flash_response_df_path(), key='df') fdf = self.flash_response_df fdf.cell = [int(cell) for cell in fdf.cell.values] fdf.cell_specimen_id = [int(cell_specimen_id) for cell_specimen_id in fdf.cell_specimen_id.values] @@ -334,7 +336,8 @@ def get_omitted_flash_response_df(self): else: if os.path.exists(self.get_omitted_flash_response_df_path()): print('loading omitted flash response dataframe') - self.omitted_flash_response_df = pd.read_hdf(self.get_omitted_flash_response_df_path(), key='df', format='fixed') + # self.omitted_flash_response_df = pd.read_hdf(self.get_omitted_flash_response_df_path(), key='df', format='fixed') + self.omitted_flash_response_df = pd.read_hdf(self.get_omitted_flash_response_df_path(), key='df') fdf = self.omitted_flash_response_df fdf.cell = [int(cell) for cell in fdf.cell.values] fdf.cell_specimen_id = [int(cell_specimen_id) for cell_specimen_id in fdf.cell_specimen_id.values] From 3df0ed084647b4e04a8be9c60c5210c904e95470 Mon Sep 17 00:00:00 2001 From: Nick Ponvert Date: Wed, 10 Apr 2019 17:08:42 -0700 Subject: [PATCH 2/2] Stops passing format kwarg to pandas.read_hdf() I think I have found all of the places where we pass the format kwarg to pandas.read_hdf(), which is not allowed anymore by new versions of pandas. This also removes commented-out lines that I had left in an earlier commit. --- .../dataset/visual_behavior_ophys_dataset.py | 25 +------------------ .../response_analysis/response_analysis.py | 3 --- 2 files changed, 1 insertion(+), 27 deletions(-) diff --git a/visual_behavior/ophys/dataset/visual_behavior_ophys_dataset.py b/visual_behavior/ophys/dataset/visual_behavior_ophys_dataset.py index f9872380a..c1fdaeda0 100644 --- a/visual_behavior/ophys/dataset/visual_behavior_ophys_dataset.py +++ b/visual_behavior/ophys/dataset/visual_behavior_ophys_dataset.py @@ -96,14 +96,12 @@ def get_analysis_dir(self): analysis_dir = LazyLoadable('_analysis_dir', get_analysis_dir) def get_metadata(self): - # self._metadata = pd.read_hdf(os.path.join(self.analysis_dir, 'metadata.h5'), key='df', format='fixed') self._metadata = pd.read_hdf(os.path.join(self.analysis_dir, 'metadata.h5'), key='df') return self._metadata metadata = LazyLoadable('_metadata', get_metadata) def get_timestamps(self): - # self._timestamps = pd.read_hdf(os.path.join(self.analysis_dir, 'timestamps.h5'), key='df', format='fixed') self._timestamps = pd.read_hdf(os.path.join(self.analysis_dir, 'timestamps.h5'), key='df') return self._timestamps @@ -122,10 +120,6 @@ def get_timestamps_ophys(self): timestamps_ophys = LazyLoadable('_timestamps_ophys', get_timestamps_ophys) def get_stimulus_table(self): - # self._stimulus_table = pd.read_hdf( - # os.path.join(self.analysis_dir, 'stimulus_table.h5'), - # key='df', format='fixed' - # ) self._stimulus_table = pd.read_hdf( os.path.join(self.analysis_dir, 'stimulus_table.h5'), key='df' @@ -150,10 +144,6 @@ def get_stimulus_template(self): stimulus_template = LazyLoadable('_stimulus_template', get_stimulus_template) def get_stimulus_metadata(self): - # self._stimulus_metadata = pd.read_hdf( - # os.path.join(self.analysis_dir, 'stimulus_metadata.h5'), - # key='df', format='fixed' - # ) self._stimulus_metadata = pd.read_hdf( os.path.join(self.analysis_dir, 'stimulus_metadata.h5'), key='df' @@ -164,31 +154,24 @@ def get_stimulus_metadata(self): stimulus_metadata = LazyLoadable('_stimulus_metadata', get_stimulus_metadata) def get_running_speed(self): - # self._running_speed = pd.read_hdf(os.path.join(self.analysis_dir, 'running_speed.h5'), key='df', format='fixed') self._running_speed = pd.read_hdf(os.path.join(self.analysis_dir, 'running_speed.h5'), key='df') return self._running_speed running_speed = LazyLoadable('_running_speed', get_running_speed) def get_licks(self): - # self._licks = pd.read_hdf(os.path.join(self.analysis_dir, 'licks.h5'), key='df', format='fixed') self._licks = pd.read_hdf(os.path.join(self.analysis_dir, 'licks.h5'), key='df') return self._licks licks = LazyLoadable('_licks', get_licks) def get_rewards(self): - # self._rewards = pd.read_hdf(os.path.join(self.analysis_dir, 'rewards.h5'), key='df', format='fixed') self._rewards = pd.read_hdf(os.path.join(self.analysis_dir, 'rewards.h5'), key='df') return self._rewards rewards = LazyLoadable('_rewards', get_rewards) def get_task_parameters(self): - # self._task_parameters = pd.read_hdf( - # os.path.join(self.analysis_dir, 'task_parameters.h5'), - # key='df', format='fixed' - # ) self._task_parameters = pd.read_hdf( os.path.join(self.analysis_dir, 'task_parameters.h5'), key='df' @@ -198,7 +181,6 @@ def get_task_parameters(self): task_parameters = LazyLoadable('_task_parameters', get_task_parameters) def get_all_trials(self): - # self._all_trials = pd.read_hdf(os.path.join(self.analysis_dir, 'trials.h5'), key='df', format='fixed') self._all_trials = pd.read_hdf(os.path.join(self.analysis_dir, 'trials.h5'), key='df') return self._all_trials @@ -273,8 +255,7 @@ def get_events(self): events = LazyLoadable('_events', get_events) def get_roi_metrics(self): - # self._roi_metrics = pd.read_hdf(os.path.join(self.analysis_dir, 'roi_metrics.h5'), key='df', format='fixed') - self._roi_metrics = pd.read_hdf(os.path.join(self.analysis_dir, 'roi_metrics.h5'), key='df') #No support for format=fixed + self._roi_metrics = pd.read_hdf(os.path.join(self.analysis_dir, 'roi_metrics.h5'), key='df') return self._roi_metrics roi_metrics = LazyLoadable('_roi_metrics', get_roi_metrics) @@ -317,10 +298,6 @@ def get_average_image(self): average_image = LazyLoadable('_average_image', get_average_image) def get_motion_correction(self): - # self._motion_correction = pd.read_hdf( - # os.path.join(self.analysis_dir, 'motion_correction.h5'), - # key='df', format='fixed' - # ) self._motion_correction = pd.read_hdf( os.path.join(self.analysis_dir, 'motion_correction.h5'), key='df' diff --git a/visual_behavior/ophys/response_analysis/response_analysis.py b/visual_behavior/ophys/response_analysis/response_analysis.py index e0940e802..25491c6f1 100644 --- a/visual_behavior/ophys/response_analysis/response_analysis.py +++ b/visual_behavior/ophys/response_analysis/response_analysis.py @@ -132,7 +132,6 @@ def get_trial_response_df(self): else: if os.path.exists(self.get_trial_response_df_path()): print('loading trial response dataframe') - # self.trial_response_df = pd.read_hdf(self.get_trial_response_df_path(), key='df', format='fixed') self.trial_response_df = pd.read_hdf(self.get_trial_response_df_path(), key='df') tdf = self.trial_response_df tdf.cell = [int(cell) for cell in tdf.cell.values] @@ -248,7 +247,6 @@ def get_flash_response_df(self): else: if os.path.exists(self.get_flash_response_df_path()): print('loading flash response dataframe') - # self.flash_response_df = pd.read_hdf(self.get_flash_response_df_path(), key='df', format='fixed') self.flash_response_df = pd.read_hdf(self.get_flash_response_df_path(), key='df') fdf = self.flash_response_df fdf.cell = [int(cell) for cell in fdf.cell.values] @@ -336,7 +334,6 @@ def get_omitted_flash_response_df(self): else: if os.path.exists(self.get_omitted_flash_response_df_path()): print('loading omitted flash response dataframe') - # self.omitted_flash_response_df = pd.read_hdf(self.get_omitted_flash_response_df_path(), key='df', format='fixed') self.omitted_flash_response_df = pd.read_hdf(self.get_omitted_flash_response_df_path(), key='df') fdf = self.omitted_flash_response_df fdf.cell = [int(cell) for cell in fdf.cell.values]