Data Processing Example

from fibermagic.IO.NeurophotometricsIO import read_project_rawdata, read_project_logs
from pathlib import Path
import plotly.express as px
from fibermagic.core.demodulate import zdFF_airPLS, add_zdFF
from fibermagic.core.perievents import perievents
from fibermagic.utils.download_dataset import download

download('arena-data')

Downloading started
Downloading Completed, Extracting...

The data was taken in two batches from different mice. Batch 1 consists of one recording per mouse (n=2). Batch 2 includes two recordings per mouse (n=7). Every mouse was recorded individually (no multiple mice per recording).

The experimental environment was a 1m x 1m sized arena. The mouse was able to move freely. Food pellets, a running wheel, and a novel object was placed in different locations of the arena. Sessions were 15 minutes long.

For behavioral evaluation, a video was recorded using a IR-camera. The video was used to determing the region in which the mouse was.

PROJECT_PATH = Path(r'data')
df_idx = read_project_rawdata(PROJECT_PATH,
                     ['Batch', 'Day', 'FP', 'Mouse_'], 'arena.csv', ignore_dirs=['meta', 'processed', 'processed.zip'])

data\Batch 1\Day1\FP\B9295\arena.csv
data\Batch 1\Day1\FP\B9296\arena.csv
data\Batch 2\Day1\FP\B9443\arena.csv
data\Batch 2\Day1\FP\B9670\arena.csv
data\Batch 2\Day1\FP\B9671\arena.csv
data\Batch 2\Day1\FP\B9888\arena.csv
data\Batch 2\Day1\FP\B9889\arena.csv
data\Batch 2\Day1\FP\B9890\arena.csv
data\Batch 2\Day1\FP\B9891\arena.csv
data\Batch 2\Day2\FP\B9443\arena.csv
data\Batch 2\Day2\FP\B9670\arena.csv
data\Batch 2\Day2\FP\B9671\arena.csv
data\Batch 2\Day2\FP\B9888\arena.csv
data\Batch 2\Day2\FP\B9889\arena.csv
data\Batch 2\Day2\FP\B9890\arena.csv
data\Batch 2\Day2\FP\B9891\arena.csv

A first spike into to investigate the raw data.

By changing values to different mice and days, you can see that mice B9671 and B9889 had bad signals.

df = df_idx.reset_index()
df.Mouse.unique()  # list recorded mice

array(['B9295', 'B9296', 'B9443', 'B9670', 'B9671', 'B9888', 'B9889',
       'B9890', 'B9891'], dtype=object)

px.line(df[(df.Mouse=='B9890') & (df.Day=='Day2') & (df.Channel == 560)].Signal).show()

px.line(df[(df.Mouse=='B9890') & (df.Day=='Day2') & (df.Channel == 560)].Reference)

df_idx = add_zdFF(df_idx, smooth_win=10, remove=200, lambd=5e9).set_index('FrameCounter', append=True)
df_idx

							Signal	Reference	zdFF (airPLS)
Batch	Day	FP	Mouse_	Mouse	Channel	FrameCounter
Batch 1	Day1	FP	B9295	B9295	470	201	0.006334	0.004026	-0.138428
						202	0.006327	0.004017	-0.138428
						203	0.006363	0.004003	-0.258778
						204	0.006339	0.004004	-0.359795
						205	0.006374	0.004003	-0.461999
...	...	...	...	...	...	...	...	...	...
Batch 2	Day2	FP	B9891	B9891	560	23482	0.005995	0.004768	-0.297719
						23483	0.005960	0.004758	-0.276701
						23484	0.006041	0.004695	-0.097237
						23485	0.005952	0.004757	-0.187640
						23486	0.006012	0.004762	-0.384082

735342 rows × 3 columns

Look at eating behavior in the full session

df = df_idx.reset_index()
subset = df[(df.Mouse=='B9890') & (df.Day=='Day2') ]
food_intake_9890 = [2277, 2516, 2692, 3234, 3548, 3712, 3936, 4315, 4483, 4671, 6323, 6418, 6589, 6659, 6853, 7590, 8890]
fig = px.line(x=(subset.FrameCounter - 297)/2.5, y=subset['zdFF (airPLS)'], color=subset.Channel)
for food in food_intake_9890:
    fig = fig.add_vline(x=food)
fig.show()

food_intake_9670 = [195, 289, 324, 362, 739, 781, 1262, 1330, 1454, 2130, 2190, 2310, 2366, 3075, 4022, 4080, 4180, 4240, 4350, 4480, 5140, 7410, 8094, 8150]
subset = df[(df.Mouse=='B9670') & (df.Day=='Day2') & (df.Channel==560) ]
fig = px.line(x=(subset.FrameCounter - 307)/2.5, y=subset['zdFF (airPLS)'])
for food in food_intake_9890:
    fig = fig.add_vline(x=food)
fig.show()

Synchronize photometry recording with behavioral events from video recording

import pandas as pd

def arena_sync(logs, path):
    ttl = pd.read_csv(path/'input1.csv')
    frame_times = pd.read_csv(path/'time.csv')
    first_ttl = ttl.loc[0, 'Item2']

    first_frame = ((first_ttl - frame_times.Item2).abs().idxmin())
    framerate = 30
    logs['Timestamp'] = logs.Frame / 30
    logs['FrameCounter'] = logs.Timestamp * 25 // 1
    logs['FrameCounter'] += first_frame
    logs['FrameCounter'] = logs.FrameCounter.astype(int)
    logs = logs.rename(columns={'ROI': 'Event'})

    # ignore events that are too close to start or end of recording
    logs = logs[(logs.Timestamp > 20) & (logs.Timestamp < max(logs.Timestamp) - 20)]
    return logs.set_index('FrameCounter')

import os
logs = read_project_logs(PROJECT_PATH,
                          ['Batch', 'Day', 'FP', 'Mouse_'], sync_fun=arena_sync, ignore_dirs=['meta', 'processed', 'processed.zip'])
logs

						Frame	Event	Timestamp
Batch	Day	FP	Mouse_	Mouse	FrameCounter
Batch 1	Day1	FP	B9295	B9295	2522	2717	non_roi_out	90.566667
					2523	2718	food_in	90.600000
					2572	2777	food_out	92.566667
					2573	2778	non_roi_in	92.600000
					2578	2784	non_roi_out	92.800000
...	...	...	...	...	...	...	...	...
Batch 2	Day2	FP	B9891	B9891	20104	23044	non_roi_in	768.133333
					20118	23061	non_roi_out	768.700000
					20119	23062	wheel_in	768.733333
					20183	23139	wheel_out	771.300000
					20184	23140	non_roi_in	771.333333

3008 rows × 3 columns

Select appropriate events and calculate perievents

# use only entries if mouse was more than x seconds in region
min_sec_in_region = 10

idxin = None
onset = 0
to_drop = list()
for index, row in logs.iterrows():
    if '_in' in row.Event:
        idxin = index
        onset = row.Timestamp
    elif '_out' in row.Event:
        if row.Timestamp - onset < min_sec_in_region:
            to_drop.append(idxin)
            to_drop.append(index)
logs = logs.drop(index=to_drop)

peri = perievents(df_idx, logs, 15, 25)
peri

c:\users\georg\git\fibermagic\fibermagic\core\perievents.py:41: SettingWithCopyWarning:


A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy

C:\Users\Georg\anaconda3\envs\fibermagic\lib\site-packages\pandas\core\frame.py:3678: SettingWithCopyWarning:


A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy

c:\users\georg\git\fibermagic\fibermagic\core\perievents.py:58: SettingWithCopyWarning:


A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy

Further Analysis

# save datastream, logs and perievents

output_path = PROJECT_PATH / 'processed'

# uncomment if you like to save the output files
# df.to_csv(output_path / 'datastream.csv')
# logs.to_csv(output_path / 'logs_time_criteria_synced.csv')
# peri.to_csv(output_path / 'perievents_time_criteria_synced.csv')

Now, you can use data analysis tools like Tableau or PowerBI or any data analysis technology like R, Matlab, Python (e.g. Seaborn) to visualize your data