Spaces:
Runtime error
Runtime error
Download utils/data_processing.py from neuralcomputation/batik: direct link, hf CLI and curl.
- Browser
- Download file 15 kB
-
https://huggingface.co/spaces/neuralcomputation/batik/resolve/main/utils/data_processing.py
- Command line
-
hf download hf://spaces/neuralcomputation/batik/utils/data_processing.py
-
curl -L -o data_processing.py https://huggingface.co/spaces/neuralcomputation/batik/resolve/main/utils/data_processing.py
15 kB
| """Provides the standard data processing functions performed on CNMFe and annotation data""" | |
| import numpy as np | |
| from scipy.signal import correlate | |
| from scipy.stats import zscore | |
| def smooth(data: np.ndarray, window_size=5): | |
| """ | |
| Returns a smoothed version of response data using a moving average filter. | |
| Parameters: | |
| ---------- | |
| data : np.ndarray | |
| A numpy 1-D array containing data to be smoothed. | |
| window_size : int | |
| Number of data points for calculating the smoothed value. If an even number is | |
| passed in, window_size is autmoatically reduced by 1. | |
| Returns: | |
| -------- | |
| smooth_data : np.ndarray | |
| Smoothed data, returned as a 1-D array of the same size as ``data``. | |
| Notes: | |
| ------ | |
| Implements MATLAB's smooth function. | |
| """ | |
| if window_size == 0: | |
| raise ValueError('window_size can not be 0.') | |
| if window_size == 1: | |
| return data | |
| if window_size > data.size: | |
| window_size = data.size | |
| if window_size%2 == 0: | |
| window_size = window_size - 1 | |
| outside_valid_window_size = int((window_size-1)/2) | |
| start = np.array([np.sum(data[0:(2*k+1)]/(2*k+1)) for k in range(outside_valid_window_size)]) | |
| end = np.array([np.sum(data[-(2*k+1):]/(2*k+1)) for k in range(outside_valid_window_size)])[::-1] | |
| smoothed_data = np.convolve(data,np.ones(window_size,dtype=int),'valid')/window_size | |
| return np.hstack((start,smoothed_data,end)) | |
| def corr(x: np.ndarray, y: np.ndarray): | |
| """ | |
| Returns a matrix of the pairwise correlation coefficient between each pair of columns | |
| in the input matrices x and y. | |
| Parameters: | |
| ----------- | |
| x : np.ndarray | |
| Input matrix, specified as an n x k_1 matrix. Its rows correspond to | |
| observations, and the columns correspond to variables. | |
| y : np.ndarray | |
| Input matrix, specified as an n x k_2 matrix. Its rows correspond to | |
| observations, and the columns correspond to variables. | |
| Returns: | |
| -------- | |
| rho - Pairwise linear correlation coefficient, returned as a matrix. | |
| Notes: | |
| ------ | |
| Implements MATLAB's corr function. | |
| """ | |
| return np.corrcoef(x,y)[0][1] | |
| def autocorr(x:np.ndarray, | |
| max_lags=10): | |
| """ | |
| Returns the correlations and associated lags of the univariate time series x. | |
| Parameters: | |
| ----------- | |
| x : np.ndarray | |
| Observed univariate time series. | |
| max_lags : int | |
| Number of lags, specified as a positive integer. | |
| Returns: | |
| acf : np.ndarray | |
| Correlations, returned as a numeric vector of length ``max_lags`` + 1. | |
| lags : np.ndarray | |
| Autocorrelation lags. | |
| Notes: | |
| ------ | |
| Modified version of matplotlib's acorr function. | |
| """ | |
| Nx = len(x) | |
| correls = correlate(x, x, mode="full") | |
| correls = correls / np.dot(x, x) | |
| if max_lags is None: | |
| max_lags = Nx - 1 | |
| if max_lags >= Nx or max_lags < 1: | |
| raise ValueError('maxlags must be None or strictly ' | |
| 'positive < %d' % Nx) | |
| lags = np.arange(-max_lags, max_lags + 1) | |
| acf = correls[Nx - 1 - max_lags:Nx + max_lags] | |
| return acf, lags | |
| def convert_to_rast(behavior_ts, time_max): | |
| """ | |
| Converts a list of behavior time stamps to a one-hot vector where 0 indicates no | |
| presence of the given behavior, and 1 indicates presence of it. | |
| Args: | |
| behavior_ts - a list of time stamps (start and end) for a particular behavior\n | |
| time_max - the length in frames of the vector | |
| Returns: | |
| behavior_rast - a one-hot vector | |
| """ | |
| behavior_rast = np.zeros(time_max) | |
| for time_stamps in behavior_ts: | |
| start = int(round(time_stamps[0])) | |
| end = int(round(time_stamps[1] + 1)) | |
| if start > time_max: | |
| break | |
| if end > time_max: | |
| end = time_max | |
| np.put(behavior_rast,range(start,end),np.ones(end-start)) | |
| return behavior_rast | |
| def convert_to_raster(bouts: list, | |
| neural_activity_sr: float, | |
| observation_sr: float, | |
| max_frame: int): | |
| """ | |
| Converts bouts into a behavior raster, a one hot encoding of a behavior describing | |
| when it is active. | |
| It is often the case that the start and stop timestamps found in ``bouts`` are | |
| collected at a different sample rate than ``neural_activity``, which are often what | |
| behavior rasters align to. In order to align the two, a ratio between the sample | |
| rates of ``neural_activity`` and the bouts of behavior, which are observations, | |
| is calculated and then multiplied to the timestamps. | |
| Parameters: | |
| ----------- | |
| bouts : np.ndarray | |
| An array where each element is a pair of integers where the first integer denotes | |
| the beginning of a bout of behavior, and the second integer denotes the end of | |
| the bout. | |
| neural_activity_sr : float | |
| Sample rate of ``neural_activity``. | |
| observation_sr : float | |
| Sample rate for the ``bouts`` used. | |
| max_frame : int | |
| The length of the behavior raster, often set to the number of frames of | |
| ``neural_activity``. | |
| Returns: | |
| -------- | |
| behavior_raster : np.ndarray | |
| A raster (a one hot encoding) of a behavior, describing when it is active. | |
| """ | |
| sr_ratio = neural_activity_sr/observation_sr | |
| behavior_ts_adjusted = bouts*sr_ratio | |
| behavior_raster = np.zeros(max_frame) | |
| for time_stamps in behavior_ts_adjusted: | |
| start = int(round(time_stamps[0])) | |
| end = int(round(time_stamps[1] + 1)) | |
| if start > max_frame: | |
| break | |
| if end > max_frame: | |
| end = max_frame | |
| np.put(behavior_raster,range(start,end),np.ones(end-start)) | |
| return behavior_raster | |
| def convert_to_bouts(behavior_raster: np.ndarray): | |
| """ | |
| Converts a behavior raster into behavior bouts, an array where each element is a | |
| pair of timestamps (int) where the first timestamp denotes the beginning of a bout of | |
| behavior, and the second timestamp denotes the end of the bout. | |
| Parameters: | |
| ----------- | |
| behavior_raster : np.ndarray | |
| A raster (a one hot encoding) of a behavior, describing when it is active. | |
| Returns: | |
| -------- | |
| bouts : np.ndarray | |
| An array where each element is a pair of timestamps (int) where the first | |
| timestamp denotes the beginning of a bout of behavior, and the second timestamp | |
| denotes the end of the bout. | |
| """ | |
| dt = behavior_raster[1:] - behavior_raster[:-1] | |
| start = np.where(dt==1)[0] + 1 | |
| stop = np.where(dt==-1)[0] | |
| if behavior_raster[0]: | |
| start = np.concatenate((np.array([0]),start)) | |
| if behavior_raster[-1]: | |
| stop = np.concatenate((stop,[behavior_raster.size])) | |
| bouts = np.hstack((np.reshape(start,(len(start),1)), | |
| np.reshape(stop,(len(stop),1)))) | |
| return bouts | |
| def merge_rasters_down(behavior_raster_array: np.ndarray)-> np.ndarray: | |
| """ | |
| For a behavior raster, merges down all rasters to one array in such a way that no | |
| two behaviors are occuring at the same time. | |
| It determines which behavior should remain 'on top' by determening which behavior | |
| has the least amount of active frames. | |
| This method should only be used on behavior rasters where all behaviors come from a | |
| single channel. | |
| Parameters: | |
| ----------- | |
| behavior_raster_array : np.ndarray | |
| An array where each row is a behavior raster, a one hot encoding of behaviors, | |
| describing when that behavior is active. Each row of this array must use a | |
| different value to indicate that a behavior is active (for example, if one | |
| row uses 1s, another row must not use 1 as well). | |
| Returns: | |
| -------- | |
| single_track : np.ndarray | |
| An array which is the length of a behavior raster in ``behavior_raster_array``, | |
| where each entry is either 0 indicating that no behavior is active, or a value | |
| indicating that a specific behavior is active. | |
| """ | |
| # single track | |
| single_track = np.zeros((1,behavior_raster_array.shape[1])) | |
| # determine order to insert row values | |
| num_active_frames = [np.sum(np.where(row > 0, 1, 0)) for row in behavior_raster_array] | |
| for i in range(behavior_raster_array.shape[0]): | |
| max_i = np.argmax(num_active_frames) | |
| num_active_frames[max_i] = -1 | |
| unique_values = np.unique(behavior_raster_array[max_i]) | |
| if len(unique_values) > 1: value = unique_values[1] | |
| else: value = 0 | |
| active_inds = np.where(behavior_raster_array[max_i] == value)[0] | |
| single_track[:,active_inds] = value | |
| return single_track | |
| def separate_tracks(single_track: np.ndarray, | |
| behavior_values: list): | |
| """ | |
| For a single track, separates each unique value (except for 0) into its own raster | |
| within a 2-D array. | |
| Parameters: | |
| ----------- | |
| single_track : np.ndarray | |
| An array which is the length of a behavior raster in ``behavior_raster_array``, | |
| where each entry is either 0 indicating that no behavior is active, or a value | |
| indicating that a specific behavior is active. | |
| behavior_values : list | |
| A list of values corresponding to the specific behaviors within ``single_track``. | |
| Returns: | |
| -------- | |
| behavior_raster_array : np.ndarray | |
| An array where each row is a behavior raster, a one hot encoding of behaviors, | |
| describing when that behavior is active. | |
| """ | |
| if len(behavior_values) < np.unique(single_track).size - 1: | |
| raise KeyError("There are not sufficient values within ``behavior_values`` to " | |
| "accomodate those present in ``single_track``.") | |
| tracks = [] | |
| for value in behavior_values: | |
| tracks.append(np.where(single_track == value, value, 0)) | |
| return np.vstack(tracks) | |
| def config_neural_activity(config: dict, neural_activity: np.ndarray): | |
| """ | |
| Configures `neural_activity` according to parameters set in config. | |
| Parameters: | |
| ----------- | |
| config : dict | |
| A dictionary which specifies the following parameters: 'smooth_window', | |
| 'baseline_frame', and 'zscore_method'. 'zscore_method' is one of "All Data", | |
| "Baseline", or "No Z-Score". | |
| neural_activity : np.ndarray | |
| Neural activity being used. | |
| Returns: | |
| -------- | |
| mod_neural_activity : np.ndarray | |
| Modified `neural_activity`, accodring to `config`. | |
| """ | |
| smooth_window = config['smooth_window'] | |
| zscore_method = config['zscore_method'] | |
| baseline_frame = config['baseline_frame'] | |
| # smooth | |
| if len(neural_activity.shape) > 1: | |
| neural_data_smooth = np.zeros(neural_activity.shape) | |
| for i in range(neural_activity.shape[0]): | |
| neural_data_smooth[i] = smooth(neural_activity[i], int(smooth_window)) | |
| mod_neural_activity = neural_data_smooth | |
| else: | |
| mod_neural_activity = smooth(neural_activity, int(smooth_window)) | |
| # z-score | |
| if zscore_method == 'Baseline' and (not baseline_frame is None or baseline_frame == 0): | |
| if len(neural_activity.shape)> 1: | |
| mean = mod_neural_activity[:,:baseline_frame].mean(axis=1,keepdims=True) | |
| std = mod_neural_activity[:,:baseline_frame].std(axis=1,keepdims=True) | |
| else: | |
| mean = mod_neural_activity[:baseline_frame].mean() | |
| std = mod_neural_activity[:baseline_frame].std() | |
| mod_neural_activity = (mod_neural_activity - mean) / std | |
| elif zscore_method == 'No Z-Score': | |
| mod_neural_activity = mod_neural_activity | |
| else: | |
| if len(neural_activity.shape) > 1: | |
| mod_neural_activity = zscore(mod_neural_activity,axis=1) | |
| else: | |
| mod_neural_activity = zscore(mod_neural_activity) | |
| return mod_neural_activity | |
| def compress_annotations(annot: dict, downsample_rate: int, max_frame: int)-> dict: | |
| """ | |
| Takes in an annotation dictionary and creates a single raster per channel, where the | |
| raster contains the behaviors from their respective channel. | |
| annot : dict | |
| Dictionary of beginning and end frames for behaviors. | |
| downsample_rate : int | |
| The rate at which samples should be taken. Divides bout timing (in frames) by | |
| value. | |
| max_frame : int | |
| The last frame for annotations from `annot`. | |
| """ | |
| annot_single_track = {} | |
| channel_behavior_map = {} | |
| for channel in annot: | |
| channel_rasters = [] | |
| behavior_map = {} | |
| behavior_map.update({0: 'None'}) | |
| for i, behavior in enumerate(annot[channel]): | |
| bouts = annot[channel][behavior] | |
| raster = convert_to_raster(bouts, 1, downsample_rate, max_frame) | |
| channel_rasters.append(raster*(i+1)) | |
| behavior_map.update({(i+1) : behavior}) | |
| channel_raster = merge_rasters_down(np.array(channel_rasters))[0] | |
| annot_single_track.update({channel : channel_raster}) | |
| channel_behavior_map.update({channel : behavior_map}) | |
| return annot_single_track, channel_behavior_map | |
| def compress_compressed_annotations(annot_single_track: dict, | |
| channel_behavior_map: dict, | |
| max_frame: int): | |
| """ | |
| Further compresses the results from `compress_annotations` to get a single array | |
| where each entry is a list of the behaviors present at that frame across all channels. | |
| """ | |
| labels = [] | |
| for frame in range(max_frame): | |
| labels_at_frame = [] | |
| for channel in annot_single_track: | |
| channel_raster = annot_single_track[channel] | |
| behavior_map = channel_behavior_map[channel] | |
| behavior_value = int(channel_raster[frame]) | |
| behavior_label = behavior_map.get(behavior_value) | |
| labels_at_frame.append(behavior_label) | |
| labels.append('||'.join(labels_at_frame)) | |
| return labels | |
| def generate_label_array(annot: dict, | |
| downsample_rate: int, | |
| max_frame: int)-> list[str]: | |
| """ | |
| Generates an array of lists of labels, where each entry is a video frame, and the | |
| labels come from each channel in `annot`. | |
| """ | |
| annot_single_track,\ | |
| channel_behavior_map = compress_annotations(annot, downsample_rate, max_frame) | |
| labels = compress_compressed_annotations(annot_single_track, | |
| channel_behavior_map, | |
| max_frame) | |
| return labels | |