Skip to content

lots of linting #518

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Apr 11, 2019
Merged

lots of linting #518

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
86d42bd
Stop passing format kwarg to pandas.read_hdf()
Apr 10, 2019
3df0ed0
Stops passing format kwarg to pandas.read_hdf()
nickponvert Apr 11, 2019
c5d7574
linting
matchings Apr 11, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions visual_behavior/ophys/dataset/cell_matching_dataset.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,10 +85,10 @@ def get_analysis_folder_name(self):
lims_data = self.lims_data
date = str(lims_data.experiment_date)[:10].split('-')
analysis_folder_name = str(lims_data.lims_id) + '_' + \
str(lims_data.mouse_id) + '_' + date[0][2:] + date[1] + date[2] + '_' + \
lims_data.structure + '_' + str(lims_data.depth) + '_' + \
lims_data.specimen_driver_line.split('-')[0] + '_' + lims_data.rig[3:5] + \
lims_data.rig[6] + '_' + lims_data.session_type
str(lims_data.mouse_id) + '_' + date[0][2:] + date[1] + date[2] + '_' + \
lims_data.structure + '_' + str(lims_data.depth) + '_' + \
lims_data.specimen_driver_line.split('-')[0] + '_' + lims_data.rig[3:5] + \
lims_data.rig[6] + '_' + lims_data.session_type
self.analysis_folder_name = analysis_folder_name
return self.analysis_folder_name

Expand Down
29 changes: 14 additions & 15 deletions visual_behavior/ophys/dataset/visual_behavior_ophys_dataset.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,13 +96,13 @@ def get_analysis_dir(self):
analysis_dir = LazyLoadable('_analysis_dir', get_analysis_dir)

def get_metadata(self):
self._metadata = pd.read_hdf(os.path.join(self.analysis_dir, 'metadata.h5'), key='df', format='fixed')
self._metadata = pd.read_hdf(os.path.join(self.analysis_dir, 'metadata.h5'), key='df')
return self._metadata

metadata = LazyLoadable('_metadata', get_metadata)

def get_timestamps(self):
self._timestamps = pd.read_hdf(os.path.join(self.analysis_dir, 'timestamps.h5'), key='df', format='fixed')
self._timestamps = pd.read_hdf(os.path.join(self.analysis_dir, 'timestamps.h5'), key='df')
return self._timestamps

timestamps = LazyLoadable('_timestamps', get_timestamps)
Expand All @@ -122,7 +122,7 @@ def get_timestamps_ophys(self):
def get_stimulus_table(self):
self._stimulus_table = pd.read_hdf(
os.path.join(self.analysis_dir, 'stimulus_table.h5'),
key='df', format='fixed'
key='df'
)
self._stimulus_table = self._stimulus_table.reset_index()
# self._stimulus_table = self._stimulus_table.drop(
Expand All @@ -146,42 +146,42 @@ def get_stimulus_template(self):
def get_stimulus_metadata(self):
self._stimulus_metadata = pd.read_hdf(
os.path.join(self.analysis_dir, 'stimulus_metadata.h5'),
key='df', format='fixed'
key='df'
)
self._stimulus_metadata = self._stimulus_metadata.drop(columns='image_category')
return self._stimulus_metadata

stimulus_metadata = LazyLoadable('_stimulus_metadata', get_stimulus_metadata)

def get_running_speed(self):
self._running_speed = pd.read_hdf(os.path.join(self.analysis_dir, 'running_speed.h5'), key='df', format='fixed')
self._running_speed = pd.read_hdf(os.path.join(self.analysis_dir, 'running_speed.h5'), key='df')
return self._running_speed

running_speed = LazyLoadable('_running_speed', get_running_speed)

def get_licks(self):
self._licks = pd.read_hdf(os.path.join(self.analysis_dir, 'licks.h5'), key='df', format='fixed')
self._licks = pd.read_hdf(os.path.join(self.analysis_dir, 'licks.h5'), key='df')
return self._licks

licks = LazyLoadable('_licks', get_licks)

def get_rewards(self):
self._rewards = pd.read_hdf(os.path.join(self.analysis_dir, 'rewards.h5'), key='df', format='fixed')
self._rewards = pd.read_hdf(os.path.join(self.analysis_dir, 'rewards.h5'), key='df')
return self._rewards

rewards = LazyLoadable('_rewards', get_rewards)

def get_task_parameters(self):
self._task_parameters = pd.read_hdf(
os.path.join(self.analysis_dir, 'task_parameters.h5'),
key='df', format='fixed'
key='df'
)
return self._task_parameters

task_parameters = LazyLoadable('_task_parameters', get_task_parameters)

def get_all_trials(self):
self._all_trials = pd.read_hdf(os.path.join(self.analysis_dir, 'trials.h5'), key='df', format='fixed')
self._all_trials = pd.read_hdf(os.path.join(self.analysis_dir, 'trials.h5'), key='df')
return self._all_trials

all_trials = LazyLoadable('_all_trials', get_all_trials)
Expand Down Expand Up @@ -255,13 +255,12 @@ def get_events(self):
events = LazyLoadable('_events', get_events)

def get_roi_metrics(self):
self._roi_metrics = pd.read_hdf(os.path.join(self.analysis_dir, 'roi_metrics.h5'), key='df', format='fixed')
self._roi_metrics = pd.read_hdf(os.path.join(self.analysis_dir, 'roi_metrics.h5'), key='df')
return self._roi_metrics

roi_metrics = LazyLoadable('_roi_metrics', get_roi_metrics)

def get_roi_mask_dict(self):
cell_specimen_ids = self.get_cell_specimen_ids()
f = h5py.File(os.path.join(self.analysis_dir, 'roi_masks.h5'), 'r')
roi_mask_dict = {}
for key in f.keys():
Expand Down Expand Up @@ -300,15 +299,15 @@ def get_average_image(self):
def get_motion_correction(self):
self._motion_correction = pd.read_hdf(
os.path.join(self.analysis_dir, 'motion_correction.h5'),
key='df', format='fixed'
key='df'
)
return self._motion_correction

motion_correction = LazyLoadable('_motion_correction', get_motion_correction)

def get_cell_specimen_ids(self):
roi_metrics = self.roi_metrics
self._cell_specimen_ids = np.asarray([roi_metrics[roi_metrics.roi_id==roi_id].id.values[0]
self._cell_specimen_ids = np.asarray([roi_metrics[roi_metrics.roi_id == roi_id].id.values[0]
for roi_id in np.sort(self.roi_metrics.roi_id.values)])
return self._cell_specimen_ids

Expand All @@ -322,12 +321,12 @@ def get_cell_indices(self):

def get_cell_specimen_id_for_cell_index(self, cell_index):
roi_metrics = self.roi_metrics
cell_specimen_id = roi_metrics[roi_metrics.cell_index==cell_index].id.values[0]
cell_specimen_id = roi_metrics[roi_metrics.cell_index == cell_index].id.values[0]
return cell_specimen_id

def get_cell_index_for_cell_specimen_id(self, cell_specimen_id):
roi_metrics = self.roi_metrics
cell_index = roi_metrics[roi_metrics.id==cell_specimen_id].cell_index.values[0]
cell_index = roi_metrics[roi_metrics.id == cell_specimen_id].cell_index.values[0]
return cell_index

@classmethod
Expand Down
56 changes: 30 additions & 26 deletions visual_behavior/ophys/response_analysis/response_analysis.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,10 +42,10 @@ def __init__(self, dataset, overwrite_analysis_files=False, use_events=False):
self.trial_window = [-4, 8] # time, in seconds, around change time to extract portion of cell trace
self.flash_window = [-0.5,
0.75] # time, in seconds, around stimulus flash onset time to extract portion of cell trace
self.omitted_flash_window = [-3,3]
self.omitted_flash_window = [-3, 3]
self.response_window_duration = 0.5 # window, in seconds, over which to take the mean for a given trial or flash
self.response_window = [np.abs(self.trial_window[0]), np.abs(self.trial_window[
0]) + self.response_window_duration] # time, in seconds, around change time to take the mean response
0]) + self.response_window_duration] # time, in seconds, around change time to take the mean response
self.baseline_window = np.asarray(
self.response_window) - self.response_window_duration # time, in seconds, relative to change time to take baseline mean response
self.stimulus_duration = 0.25 # self.dataset.task_parameters['stimulus_duration'].values[0]
Expand Down Expand Up @@ -98,14 +98,16 @@ def generate_trial_response_df(self):
mean_running_speed = ut.get_mean_in_window(running_speed_trace, self.response_window,
self.stimulus_frame_rate)
df_list.append(
[trial, int(cell_index), int(cell_specimen_id), trace, timestamps, mean_response, baseline_response, n_events,
p_value, sd_over_baseline, mean_running_speed, cell_matching_has_been_run, self.dataset.experiment_id])
#running_speed_trace, running_speed_timestamps,

[trial, int(cell_index), int(cell_specimen_id), trace, timestamps, mean_response, baseline_response,
n_events,
p_value, sd_over_baseline, mean_running_speed, cell_matching_has_been_run,
self.dataset.experiment_id])
# running_speed_trace, running_speed_timestamps,

columns = ['trial', 'cell', 'cell_specimen_id', 'trace', 'timestamps', 'mean_response', 'baseline_response',
'n_events', 'p_value', 'sd_over_baseline', 'mean_running_speed', 'cell_matching_has_been_run', 'experiment_id']
#'running_speed_trace', 'running_speed_timestamps',
'n_events', 'p_value', 'sd_over_baseline', 'mean_running_speed', 'cell_matching_has_been_run',
'experiment_id']
# 'running_speed_trace', 'running_speed_timestamps',
trial_response_df = pd.DataFrame(df_list, columns=columns)
trial_metadata = self.dataset.trials
trial_metadata = trial_metadata.rename(columns={'response': 'behavioral_response'})
Expand All @@ -123,7 +125,6 @@ def save_trial_response_df(self, trial_response_df):
def get_trial_response_df(self):
if self.overwrite_analysis_files:
print('overwriting analysis files')
import h5py
file_path = self.get_trial_response_df_path()
if os.path.exists(file_path):
os.remove(file_path)
Expand All @@ -132,7 +133,7 @@ def get_trial_response_df(self):
else:
if os.path.exists(self.get_trial_response_df_path()):
print('loading trial response dataframe')
self.trial_response_df = pd.read_hdf(self.get_trial_response_df_path(), key='df', format='fixed')
self.trial_response_df = pd.read_hdf(self.get_trial_response_df_path(), key='df')
tdf = self.trial_response_df
tdf.cell = [int(cell) for cell in tdf.cell.values]
tdf.cell_specimen_id = [int(cell_specimen_id) for cell_specimen_id in tdf.cell_specimen_id.values]
Expand Down Expand Up @@ -195,15 +196,19 @@ def generate_flash_response_df(self):
self.stimulus_frame_rate)
reward_rate = flash_data.reward_rate.values[0]

row.append([int(cell), int(cell_specimen_id), int(flash), omitted, flash_time, image_name, image_category,
trace, timestamps, mean_response, baseline_response, n_events, p_value_baseline, sd_over_baseline,
reward_rate, mean_running_speed, cell_matching_has_been_run, int(self.dataset.experiment_id)])
row.append(
[int(cell), int(cell_specimen_id), int(flash), omitted, flash_time, image_name, image_category,
trace, timestamps, mean_response, baseline_response, n_events, p_value_baseline, sd_over_baseline,
reward_rate, mean_running_speed, cell_matching_has_been_run, int(self.dataset.experiment_id)])

flash_response_df = pd.DataFrame(data=row,
columns=['cell', 'cell_specimen_id', 'flash_number', 'omitted', 'start_time',
'image_name', 'image_category', 'trace', 'timestamps', 'mean_response',
'baseline_response', 'n_events', 'p_value_baseline', 'sd_over_baseline',
'reward_rate', 'mean_running_speed', 'cell_matching_has_been_run', 'experiment_id'])
'image_name', 'image_category', 'trace', 'timestamps',
'mean_response',
'baseline_response', 'n_events', 'p_value_baseline',
'sd_over_baseline',
'reward_rate', 'mean_running_speed', 'cell_matching_has_been_run',
'experiment_id'])
flash_response_df = ut.annotate_flash_response_df_with_pref_stim(flash_response_df)
flash_response_df = ut.add_repeat_number_to_flash_response_df(flash_response_df, stimulus_table)
flash_response_df = ut.add_image_block_to_flash_response_df(flash_response_df, stimulus_table)
Expand Down Expand Up @@ -238,7 +243,6 @@ def save_flash_response_df(self, flash_response_df):
def get_flash_response_df(self):
if self.overwrite_analysis_files:
# delete old file or else it will keep growing in size
import h5py
file_path = self.get_flash_response_df_path()
if os.path.exists(file_path):
os.remove(file_path)
Expand All @@ -247,7 +251,7 @@ def get_flash_response_df(self):
else:
if os.path.exists(self.get_flash_response_df_path()):
print('loading flash response dataframe')
self.flash_response_df = pd.read_hdf(self.get_flash_response_df_path(), key='df', format='fixed')
self.flash_response_df = pd.read_hdf(self.get_flash_response_df_path(), key='df')
fdf = self.flash_response_df
fdf.cell = [int(cell) for cell in fdf.cell.values]
fdf.cell_specimen_id = [int(cell_specimen_id) for cell_specimen_id in fdf.cell_specimen_id.values]
Expand All @@ -268,7 +272,7 @@ def get_omitted_flash_response_df_path(self):
def generate_omitted_flash_response_df(self):
print('generating omitted flash response df')
stimulus_table = ut.annotate_flashes_with_reward_rate(self.dataset)
stimulus_table = stimulus_table[stimulus_table.omitted==True]
stimulus_table = stimulus_table[stimulus_table.omitted == True]
row = []
for cell in self.dataset.cell_indices:
cell = int(cell)
Expand Down Expand Up @@ -304,13 +308,15 @@ def generate_omitted_flash_response_df(self):
n_events = ut.get_n_nonzero_in_window(trace, response_window, self.ophys_frame_rate)
reward_rate = flash_data.reward_rate.values[0]

row.append([int(cell), int(cell_specimen_id), int(flash), omitted, flash_time, image_name, image_category,
trace, timestamps, mean_response, baseline_response, n_events, p_value, sd_over_baseline,
reward_rate, cell_matching_has_been_run, int(self.dataset.experiment_id)])
row.append(
[int(cell), int(cell_specimen_id), int(flash), omitted, flash_time, image_name, image_category,
trace, timestamps, mean_response, baseline_response, n_events, p_value, sd_over_baseline,
reward_rate, cell_matching_has_been_run, int(self.dataset.experiment_id)])

flash_response_df = pd.DataFrame(data=row,
columns=['cell', 'cell_specimen_id', 'flash_number', 'omitted', 'start_time',
'image_name', 'image_category', 'trace', 'timestamps', 'mean_response',
'image_name', 'image_category', 'trace', 'timestamps',
'mean_response',
'baseline_response', 'n_events', 'p_value', 'sd_over_baseline',
'reward_rate', 'cell_matching_has_been_run', 'experiment_id'])
# flash_response_df = ut.annotate_flash_response_df_with_pref_stim(flash_response_df)
Expand All @@ -325,7 +331,6 @@ def save_omitted_flash_response_df(self, omitted_flash_response_df):
def get_omitted_flash_response_df(self):
if self.overwrite_analysis_files:
# delete old file or else it will keep growing in size
import h5py
file_path = self.get_omitted_flash_response_df_path()
if os.path.exists(file_path):
os.remove(file_path)
Expand All @@ -334,7 +339,7 @@ def get_omitted_flash_response_df(self):
else:
if os.path.exists(self.get_omitted_flash_response_df_path()):
print('loading omitted flash response dataframe')
self.omitted_flash_response_df = pd.read_hdf(self.get_omitted_flash_response_df_path(), key='df', format='fixed')
self.omitted_flash_response_df = pd.read_hdf(self.get_omitted_flash_response_df_path(), key='df')
fdf = self.omitted_flash_response_df
fdf.cell = [int(cell) for cell in fdf.cell.values]
fdf.cell_specimen_id = [int(cell_specimen_id) for cell_specimen_id in fdf.cell_specimen_id.values]
Expand All @@ -344,7 +349,6 @@ def get_omitted_flash_response_df(self):
self.save_omitted_flash_response_df(self.omitted_flash_response_df)
return self.omitted_flash_response_df


def compute_pairwise_correlations(self):
fdf = self.flash_response_df.copy()
if 'omitted' in fdf.keys():
Expand Down
Loading