Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
658 commits
Select commit Hold shift + click to select a range
031bea8
import the whole math_utils
a-zmz Jun 2, 2025
28ff1aa
add pre_dark_len event
a-zmz Jun 2, 2025
d7a2507
add note & clarify format
a-zmz Jun 2, 2025
9528224
map end of pre dark visible tunnel
a-zmz Jun 2, 2025
da167d6
add todo
a-zmz Jun 2, 2025
8b08373
takes event label to check what is the actual starting position
a-zmz Jun 2, 2025
ee21cd9
remove redundant imports; use InfFlag for automatic bit shift
a-zmz Jun 3, 2025
ec38b23
use intflag to automatically shift bit instead of doing it manually
a-zmz Jun 3, 2025
8175a22
use unsigned int32 dtype to increase upper bound
a-zmz Jun 3, 2025
f9a843a
update refactor version
a-zmz Jun 3, 2025
305f42e
change name of the class to what it actually represents
a-zmz Jun 3, 2025
a56f0c7
reorganise positional events
a-zmz Jun 3, 2025
a0092fd
add notes and change name of the array
a-zmz Jun 3, 2025
d53833c
create view to refer to array
a-zmz Jun 3, 2025
45c00ef
use bitwise_or to assign values to labels array
a-zmz Jun 3, 2025
6cdffd1
map more positional events
a-zmz Jun 3, 2025
2455aa5
change name to represent what it actually is
a-zmz Jun 3, 2025
7947a68
use last frame in zone as reward zone off
a-zmz Jun 3, 2025
ab64025
use NamedTuple
a-zmz Jun 4, 2025
903ec4c
add none type
a-zmz Jun 4, 2025
b8a5686
use enum & auto to map labels
a-zmz Jun 4, 2025
9bf0dd9
use unsigned int32 to ensure no overflowing
a-zmz Jun 4, 2025
4805270
use DARKS
a-zmz Jun 4, 2025
5ec792b
add todo
a-zmz Jun 4, 2025
1fc4307
return more structured
a-zmz Jun 4, 2025
227d12e
remove redundant
a-zmz Jun 4, 2025
fed0066
define world mask so it can be used globally
a-zmz Jun 4, 2025
98a749a
input vr behaviour obj
a-zmz Jun 4, 2025
245abd1
use uint32 not 64
a-zmz Jun 4, 2025
b90dfb1
fix main logic
a-zmz Jun 4, 2025
17d45ad
add doc
a-zmz Jun 4, 2025
1a5c3ae
add stamping methods
a-zmz Jun 4, 2025
6f4b4c9
use helper func to get world index mask
a-zmz Jun 4, 2025
cc4c808
get indices of world-index based events
a-zmz Jun 4, 2025
13345c9
add positional events
a-zmz Jun 4, 2025
4394964
add index helper func
a-zmz Jun 4, 2025
7152cc9
map outcome
a-zmz Jun 4, 2025
7d213d1
map outcome flags
a-zmz Jun 4, 2025
79fe84c
convert action labels to dictionary before saving
a-zmz Jun 4, 2025
af58f60
use arg.name in the cache file name if available
a-zmz Jun 5, 2025
29a8192
add trial condition mask
a-zmz Jun 5, 2025
819511a
get world index based events to fetch dark on events for pre_dark_end
a-zmz Jun 5, 2025
bb4ae71
print name of label
a-zmz Jun 5, 2025
e92cf23
adjust starting locations if trial not aligned to starts
a-zmz Jun 5, 2025
c9a29b9
drop nans if in all columns
a-zmz Jun 5, 2025
fbaf377
make sure name is before doc
a-zmz Jun 5, 2025
f67fe5a
make sure for dark trials, pre dark end is the same frame as dark onset
a-zmz Jun 5, 2025
daedfc1
print name of intflag, not value
a-zmz Jun 5, 2025
6f34e62
pass vr session object, not vr object to synchronisation
a-zmz Jun 6, 2025
6b39c1b
print name
a-zmz Jun 6, 2025
848d71d
map actual starting positions if trials not aligned to starts
a-zmz Jun 6, 2025
fbbd95a
add start as top level on positions
a-zmz Jun 6, 2025
a794f16
no need to calculate starts cuz it's already a level in position
a-zmz Jun 6, 2025
05b9ba4
loop over trial ids and use xs to get trial positions
a-zmz Jun 6, 2025
e8cf96b
formatting
a-zmz Jun 6, 2025
e76f623
use double quote
a-zmz Jun 6, 2025
92cc3d7
no need to index into dict cuz vr_session is a session object
a-zmz Jun 6, 2025
92cd575
print label name rather than value
a-zmz Jun 13, 2025
fc3ff9c
take in binning args to allow psd for binned data
a-zmz Jun 13, 2025
9611ab4
make sure the name is not the same
a-zmz Jun 13, 2025
795d724
potentially checking for multiple crossing of threshold
a-zmz Jun 13, 2025
9126263
take last 1/4 of data check for discontinunity
a-zmz Jun 13, 2025
e25d3b9
move behaviour dict up so that vr_synched & action label can be appen…
a-zmz Jun 13, 2025
d77dcca
remove number, keep unit
a-zmz Jun 13, 2025
52756ae
remove redundant arg
a-zmz Jun 13, 2025
8803aa0
add notes; get occupancy with spike count
a-zmz Jun 13, 2025
b39f45a
allows data to be binned too
a-zmz Jun 13, 2025
18874f5
initiate occupancy as df, not np array
a-zmz Jun 13, 2025
167bb77
only floor if position values are float
a-zmz Jun 13, 2025
659a8f3
interpolate if binned data has missing values
a-zmz Jun 13, 2025
b707ef2
drop nan in occupancy too to save space
a-zmz Jun 13, 2025
bee840a
add note
a-zmz Jun 13, 2025
3677cda
get trial ids then loop cuz now cols r multilevel index
a-zmz Jun 13, 2025
2afc690
add start level to binned data too
a-zmz Jun 13, 2025
1a4328f
add notes and remove redundant arg
a-zmz Jun 13, 2025
8ab87ec
add data binning args to allow psd for binned data
a-zmz Jun 13, 2025
ee394a2
make sure unit is the first arg
a-zmz Jun 19, 2025
a35ef68
implement chance at stream level
a-zmz Jun 19, 2025
d946964
use welch method by default for psd estimation
a-zmz Jun 19, 2025
074bbc9
no need to get bin args since we will separate them
a-zmz Jun 19, 2025
884017d
use minimum as start of index not 0
a-zmz Jun 19, 2025
fad145e
incorporate chance data indexing
a-zmz Jun 19, 2025
e9bc286
remove negative positions
a-zmz Jun 19, 2025
57cfc60
add notes
a-zmz Jun 19, 2025
a9361b5
get chance data
a-zmz Jun 19, 2025
e98a758
get chance psd
a-zmz Jun 19, 2025
f6f8363
add defautl repeats for chance
a-zmz Jun 21, 2025
b44cb7a
use a separate func to get chance paths and positions to speed up mul…
a-zmz Jun 21, 2025
a82ccf5
use sptial sampling frequency to get index step
a-zmz Jun 21, 2025
13717ae
use multiprocessing to get chance psd
a-zmz Jun 21, 2025
6b5ab82
add start level
a-zmz Jun 21, 2025
8e894f7
change key names
a-zmz Jun 21, 2025
be46e2c
add todo
a-zmz Jun 21, 2025
17b477a
check if df is dict before checking if it is nested dict
a-zmz Jun 21, 2025
9866f7c
remove todo
a-zmz Jun 23, 2025
48c9f95
if using all units, columns (unit ids) are the same across conditions
a-zmz Jun 23, 2025
773b2e6
allow to normalise before getting positional data
a-zmz Jul 28, 2025
b735278
do not cache get_positional_data
a-zmz Jul 28, 2025
e600ac3
no need to import Events again, get attr from `event`
a-zmz Jul 28, 2025
b26cb46
make sure to include last frame of the event during alignment
a-zmz Jul 28, 2025
f5eb4b7
delete used vars to save memory
a-zmz Jul 28, 2025
c2ec1b1
no need to do log transformation
a-zmz Jul 28, 2025
5e1c54a
set index to times index too
a-zmz Jul 29, 2025
b2c4332
no need to pass data
a-zmz Jul 29, 2025
7f4875b
implement event alignment
a-zmz Jul 29, 2025
020b813
use more informative names
a-zmz Jul 29, 2025
41af6f6
no need to get positions
a-zmz Jul 29, 2025
068c550
use align_trials implementation in stream
a-zmz Jul 29, 2025
2897bb6
do not copy raw to interim
a-zmz Jul 31, 2025
2746643
print name not event attr
a-zmz Jul 31, 2025
018bb08
reduce duration of event to 1s each side
a-zmz Jul 31, 2025
5fa432e
add starting position to spiked and fr
a-zmz Jul 31, 2025
6ae31f1
put spike in name just to differentiate with lfp
a-zmz Jul 31, 2025
bde43b5
moved implementation to stream.py
a-zmz Jul 31, 2025
8876efc
use cacheable decor in stream.py
a-zmz Jul 31, 2025
557d07d
add option to allow band extraction without preprocessing
a-zmz Aug 8, 2025
2e14c9b
default to spike_event; remove old doc
a-zmz Aug 8, 2025
1c59585
count cpu cores to set workers and multiprocessing context
a-zmz Aug 11, 2025
3ff45f5
change name so that it is more data type specific
a-zmz Aug 11, 2025
f565814
allows to extract unpreprocessed bands
a-zmz Aug 11, 2025
812a214
add todo
a-zmz Aug 11, 2025
5683886
set workers to 4 cuz more than this it would not work
a-zmz Aug 11, 2025
bf74cc6
use spawn to avoid brokenprocesspool error
a-zmz Aug 12, 2025
31b5c79
use CAR not CMR cuz all bad channels are removed, and CAR preserves L…
a-zmz Aug 13, 2025
bd27a36
remove pre-identified bad channels
a-zmz Aug 13, 2025
7e9857a
add typical oscillation bands
a-zmz Aug 15, 2025
9d4f666
add bnadwise psd
a-zmz Aug 15, 2025
b784030
make name more consistent
a-zmz Aug 15, 2025
2ea59f3
do NOT common average referencing in preprocessing, do it on demand
a-zmz Aug 15, 2025
1881be6
separate car and cmr
a-zmz Aug 15, 2025
58046e2
notch a narrow band freq with si
a-zmz Aug 15, 2025
5797993
implement notch freq from scratch
a-zmz Aug 15, 2025
a394c58
notch 50hz noise in lfp before car
a-zmz Aug 15, 2025
ba24caf
bandpass before car
a-zmz Aug 15, 2025
bd61799
do not notch 50hz by default cuz gamma is meaningful...
a-zmz Aug 19, 2025
9865b22
do preprocessing by default
a-zmz Aug 19, 2025
36e3f9d
use logging
a-zmz Aug 19, 2025
21e1672
use ks mc by default
a-zmz Aug 19, 2025
ff3ddec
copy processed data to backup if backup exists
a-zmz Sep 2, 2025
95d1fab
update spikeinterface in ks4 image
a-zmz Sep 2, 2025
2de9877
add multiprocessing options
a-zmz Sep 2, 2025
c7a8ab9
add doc
a-zmz Sep 2, 2025
d59ba1f
copy to local by default otherwise too slow
a-zmz Sep 2, 2025
7bb0a4a
use copytree to copy directory
a-zmz Sep 3, 2025
064188e
use `dredge_ap` as default ap motion correction method
a-zmz Sep 5, 2025
14ac5f8
and set default motion correction method for lfp
a-zmz Sep 5, 2025
1c95081
if backup exists find file in backup first
a-zmz Sep 5, 2025
53f7168
make sure backup dir exists
a-zmz Sep 10, 2025
49a7f4d
typo in x-lim of shank 2
a-zmz Sep 10, 2025
2ef0ed5
use Worlds property to get dark worlds bool
a-zmz Sep 15, 2025
7d335c2
remove old version code
a-zmz Sep 17, 2025
c8fbf24
first try find_file
a-zmz Sep 17, 2025
d5682c4
add label at pure luminance evoked activity post dark onset
a-zmz Sep 18, 2025
b6ce5f6
add logging
a-zmz Sep 18, 2025
7aa3a31
use double quote
a-zmz Sep 18, 2025
805ddc7
only write if it does not exist
a-zmz Sep 18, 2025
b948db5
add todo
a-zmz Sep 18, 2025
12d2eec
allows to add stream_id as key
a-zmz Sep 19, 2025
4d308b7
use spikeinterface to get unit ids by default, and allows units from …
a-zmz Sep 19, 2025
d8a24cd
use label and event to get spike chance
a-zmz Sep 19, 2025
e9be668
write chance into zarr
a-zmz Sep 19, 2025
2dee670
import stuff up front
a-zmz Sep 19, 2025
bb4dee2
write and read dataframe to/from zarr
a-zmz Sep 19, 2025
620c86b
get chance for experiment
a-zmz Sep 19, 2025
5140fbc
add zarr backend for cache
a-zmz Sep 19, 2025
309c931
add default compressor for generic zarr
a-zmz Sep 19, 2025
6d683ed
break down align trials parts to reuse code
a-zmz Sep 19, 2025
250391e
now need stream_id to access units
a-zmz Sep 19, 2025
b647523
cache spike chance into zarr
a-zmz Sep 19, 2025
9baa549
use bool to reduce size
a-zmz Sep 24, 2025
3982cf1
put spiked on shared memory to reduce memory load
a-zmz Sep 24, 2025
252a356
delete redundant vars
a-zmz Sep 24, 2025
2285652
define chunks upfront
a-zmz Sep 24, 2025
9a77ffb
fix indentation
a-zmz Sep 24, 2025
95a5095
make sure group_name is generic string
a-zmz Sep 24, 2025
f87d993
add row and column prefix
a-zmz Sep 24, 2025
05b8039
incorporate row and index multiindex
a-zmz Sep 24, 2025
93a8398
reduce size
a-zmz Sep 24, 2025
9aac4c6
use predefined number of repeats
a-zmz Sep 24, 2025
7c8bc6d
add chunks
a-zmz Sep 24, 2025
65b89fa
call it group_name
a-zmz Sep 24, 2025
cb4e766
incorporate row and index multiindex
a-zmz Sep 24, 2025
8331c26
move implementation to decorator and use it from there
a-zmz Sep 24, 2025
5462bfe
start implementing multiprocessing in firing rate convolution
a-zmz Sep 24, 2025
bc4e34c
make sure the name is correct
a-zmz Sep 24, 2025
d83aa0d
use PixelsError to track errors
a-zmz Sep 29, 2025
e8f5deb
make sure to only include trials with both start and end events
a-zmz Sep 29, 2025
6b197f8
make sure to only get the starting positions of the included trials
a-zmz Sep 29, 2025
3c97854
add alpha
a-zmz Oct 1, 2025
4f419a2
make sure to use the full base name
a-zmz Oct 1, 2025
561510c
use GLM to find landmark responsive units
a-zmz Oct 1, 2025
eb3a996
only inject reserved kwarg if the method accepts
a-zmz Oct 2, 2025
bbc7b75
zarr suffix is zarr
a-zmz Oct 2, 2025
d5283e2
add missing method
a-zmz Oct 2, 2025
57c3d35
explicitly name stat
a-zmz Oct 2, 2025
5801615
cast to str if dtype is object
a-zmz Oct 2, 2025
485bbd8
import constants
a-zmz Oct 2, 2025
a6c19d2
if row or index name is None, set it first
a-zmz Oct 2, 2025
c2a6685
encode both positive and negative response
a-zmz Oct 2, 2025
c75dfba
save both contrasts and responsiveness map
a-zmz Oct 2, 2025
8877042
cache as zarr
a-zmz Oct 2, 2025
5ae4e47
no need to pass event since we will loop through them in stream.py
a-zmz Oct 9, 2025
ad877fc
only copy .bin when necessary
a-zmz Oct 9, 2025
78f5fee
simplify
a-zmz Oct 9, 2025
42270e7
when aligning to a landmark event in dark, make sure it is in dark
a-zmz Oct 9, 2025
eb9dd5d
always check the common trial ids of starts and ends and correct both
a-zmz Oct 9, 2025
43f2835
move landmark loop to stream.py
a-zmz Oct 9, 2025
2ce6c9a
loop through start events too
a-zmz Oct 9, 2025
bd2a018
drop all nan columns
a-zmz Oct 9, 2025
95a9c16
no need to pass pos_bin cuz we now pre-define wall events
a-zmz Oct 11, 2025
1940f84
formatting
a-zmz Oct 11, 2025
81b4335
in case aligning to wall
a-zmz Oct 11, 2025
3c65089
show events of alignment too
a-zmz Oct 11, 2025
8690d9b
get spike data only when it's necessary
a-zmz Oct 11, 2025
9cf79ed
no need to dropna cuz there will not be any
a-zmz Oct 11, 2025
fe420d5
exclude equal cuz we used np.ceil to get position
a-zmz Oct 11, 2025
8d49251
get default position bin size
a-zmz Oct 11, 2025
cd49402
increase dtype cuz there are more events now
a-zmz Oct 11, 2025
53ae11a
make sure to convert event flag into uint64 to avoid ambiguity
a-zmz Oct 11, 2025
a72eed1
make sure to get the exact position for the positional event
a-zmz Oct 11, 2025
54eee61
align to wall on/off events
a-zmz Oct 11, 2025
eb167ce
use 1 index for landmarks to be consistent with virtual_reality.py cu…
a-zmz Oct 11, 2025
f6db632
stuff
a-zmz Oct 11, 2025
be3c024
remove redundant
a-zmz Oct 14, 2025
77a4613
skip if number of starts less than 2
a-zmz Oct 14, 2025
b29ae23
fix typo
a-zmz Oct 14, 2025
f5e5bbe
load merged sorting analyser if there is one
a-zmz Oct 14, 2025
9b5c387
add file with merge units group
a-zmz Oct 14, 2025
225a195
set default to thread for fedora cuz process does not work for now...
a-zmz Oct 14, 2025
7e1f26e
add merged sa and use it if exists
a-zmz Oct 15, 2025
fc717e0
get chance of we do align units from trial start to end
a-zmz Oct 17, 2025
3a62cce
use path not dir
a-zmz Oct 17, 2025
b7204b2
get max channel index
a-zmz Oct 17, 2025
48285e4
rename to differentiate from template metrics rule
a-zmz Oct 17, 2025
238c14d
show removed bad unit ids
a-zmz Oct 17, 2025
5d18df6
use template metrics to filter units
a-zmz Oct 17, 2025
788e08e
implement template metrics filter non somatic units
a-zmz Oct 17, 2025
f73f766
get chance with aligning in base.py
a-zmz Oct 17, 2025
84b3cd9
add todo
a-zmz Oct 17, 2025
01c4946
allow of_dates from mice in nested list
a-zmz Oct 19, 2025
32970cc
change name of arg to em_method (estimate_motion_method) cuz the the …
a-zmz Oct 22, 2025
499ca4d
input motion correction method, then define motion estimation method
a-zmz Oct 22, 2025
be20291
show surface depth
a-zmz Oct 22, 2025
9effe46
add double quotes
a-zmz Oct 22, 2025
ebbda01
get binned chance data
a-zmz Oct 22, 2025
e3e738d
save chance data in zarr not memmap
a-zmz Oct 22, 2025
1dc379c
bin chance given chance is zarr not memmap
a-zmz Oct 22, 2025
833d2de
do not use welch by default it smooth too much
a-zmz Oct 22, 2025
3e4143b
use implementation of df zarr in decorators
a-zmz Oct 22, 2025
900fe4a
add sampling rate
a-zmz Oct 22, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1,875 changes: 1,414 additions & 461 deletions pixels/behaviours/base.py

Large diffs are not rendered by default.

534 changes: 534 additions & 0 deletions pixels/behaviours/passive_viewing.py

Large diffs are not rendered by default.

661 changes: 661 additions & 0 deletions pixels/behaviours/virtual_reality.py

Large diffs are not rendered by default.

49 changes: 49 additions & 0 deletions pixels/configs.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
import logging

from wavpack_numcodecs import WavPack
from numcodecs import Blosc
import spikeinterface as si

# Configure logging to include a timestamp with seconds
logging.basicConfig(
level=logging.INFO,
format='''\n%(asctime)s %(levelname)s: %(message)s\
\n[in %(filename)s:%(lineno)d]''',
datefmt='%Y%m%d %H:%M:%S',
)

#logging.info('This is an info message.')
#logging.warning('This is a warning message.')
#logging.error('This is an error message.')

# set si job_kwargs
job_kwargs = dict(
pool_engine="thread", # instead of default "process"
#pool_engine="process",# does not work on 2025 oct 14
mp_context="fork", # linux
#mp_context="spawn", # mac & win
progress_bar=True,
n_jobs=0.8,
chunk_duration='1s',
max_threads_per_worker=8,
)
si.set_global_job_kwargs(**job_kwargs)

# instantiate WavPack compressor
wv_compressor = WavPack(
level=3, # high compression
bps=None, # lossless
)

# use blosc compressor for generic zarr
compressor = Blosc(
cname="zstd",
clevel=5,
shuffle=Blosc.BITSHUFFLE,
)

# kilosort 4 singularity image names
ks4_0_30_image_name = "si103.0_ks4-0-30_with_wavpack.sif"
#ks4_0_30_image_name = "si102.3_ks4-0-30_with_wavpack.sif"
ks4_0_18_image_name = "ks4-0-18_with_wavpack.sif"
ks4_image_name = ks4_0_30_image_name
36 changes: 36 additions & 0 deletions pixels/constants.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
"""
This file contains some constants parameters for the pixels pipeline.
"""
import numpy as np

SAMPLE_RATE = 2000 # Hz

freq_bands = {
"ap":[300, 9000],
"lfp":[0.5, 500],
"theta":[4, 11], # from Tom
"gamma":[30, 80], # from Tom
"ripple":[110, 220], # from Tom
}

BEHAVIOUR_HZ = 25000

np.random.seed(BEHAVIOUR_HZ)

REPEATS = 100

# latency of luminance change evoked spike
# NOTE: usually it should be between 40-60ms. now we use 500ms just to test
V1_SPIKE_LATENCY = 500 # ms #60
V1_LFP_LATENCY = 40 # ms
LGN_SPIKE_LATENCY = 30 # ms

# chunking for zarr
SMALL_CHUNKS = 64
BIG_CHUNKS = 1024

ALPHA = 0.05

# position bin sizes
POSITION_BIN = 1 # cm
BIG_POSITION_BIN = 10 # cm
Loading