Skip to content

Debugs for reading a .pickle file with signals #15

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
CodyCBakerPhD wants to merge 6 commits into
base: master
Choose a base branch
from
Open

Debugs for reading a .pickle file with signals #15

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
cc3ff6d
remove broken legacy c++ stuff from setup
CodyCBakerPhD Apr 29, 2024
824fc50
debug numpy int
CodyCBakerPhD May 6, 2024
b4fa784
moved several uninstallable packages to localized imports
CodyCBakerPhD May 6, 2024
c4d96f3
Update wormdatamodel/data/recording.py
CodyCBakerPhD May 6, 2024
7c9bfd9
fixed dependency name
CodyCBakerPhD May 13, 2024
ea60428
Merge branch 'fix_setup' of https://github.com/catalystneuro/wormdata…
CodyCBakerPhD May 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
build/
*__pycache__*
line.select
*.egg-info/
.idea
25 changes: 2 additions & 23 deletions setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,27 +1,6 @@
#!/usr/bin/env python

from distutils.core import setup, Extension

@CodyCBakerPhD CodyCBakerPhD May 6, 2024

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Of course, this PR solves the problem by completely removing the C++ stuff

If you'd rather keep it, there are some alternative approaches I can investigate (like making it optional - or splintering it to a different repo) but they will take more time

from distutils.command.build_ext import build_ext
import numpy
import git

class CustomBuildExtCommand(build_ext):
"""build_ext command for use when numpy headers are needed."""

def run(self):

# Import numpy here, only when headers are needed
import numpy
# Add numpy headers to include_dirs
self.include_dirs.append(numpy.get_include())

# Call original build_ext command
build_ext.run(self)

_legacy_c = Extension('wormdatamodel.data._legacy_c',
sources = ['wormdatamodel/data/_legacy_c.cpp'],
include_dirs = [],
extra_compile_args=['-O3','-D_FILE_OFFSET_BITS=64'])#
from distutils.core import setup

# Get git commit info to build version number/tag
#repo = git.Repo('.git')
Expand All @@ -37,5 +16,5 @@ def run(self):
author='Francesco Randi',
author_email='francesco.randi@gmail.com',
packages=['wormdatamodel','wormdatamodel.data','wormdatamodel.signal'],
ext_modules=[_legacy_c]
install_requires=['scikit-learn', 'matplotlib'],
)
1 change: 0 additions & 1 deletion wormdatamodel/data/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
__all__ = ['recording','volume','load_frames_legacy','redToGreen','genRedToGreen']

from .recording import recording
from ._legacy_c import load_frames_legacy
from .volume import volume
from .redtogreen import redToGreen, genRedToGreen
28 changes: 15 additions & 13 deletions wormdatamodel/data/recording.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
import numpy as np
import os
import json
import pickle

import numpy as np

@CodyCBakerPhD CodyCBakerPhD May 6, 2024

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Moving this line was a minor style change - styling patterns such as isort recommend separating core Python libraries from third-party modules from the 'local' module

But if you prefer the first way just let me know; I didn't make this change holistically, but if you'd like that as well the easiest way would just be to setup pre-commit on the repo


import wormdatamodel as wormdm

class recording:
Expand Down Expand Up @@ -128,10 +130,10 @@ class recording:

# Optogenetics
optogeneticsN = 0
optogeneticsFrameCount = np.zeros(optogeneticsN, dtype=np.int)
optogeneticsNPulses = np.zeros(optogeneticsN, dtype=np.int)
optogeneticsRepRateDivider = np.zeros(optogeneticsN, dtype=np.int)
optogeneticsNTrains = np.zeros(optogeneticsN, dtype=np.int)
optogeneticsFrameCount = np.zeros(optogeneticsN, dtype=np.int64)
optogeneticsNPulses = np.zeros(optogeneticsN, dtype=np.int64)
optogeneticsRepRateDivider = np.zeros(optogeneticsN, dtype=np.int64)
optogeneticsNTrains = np.zeros(optogeneticsN, dtype=np.int64)
Comment on lines -131 to +136

@CodyCBakerPhD CodyCBakerPhD May 6, 2024

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Numpy deprecated this behavior a while back (first a soft warning for several versions; then finally a hard removal that caused some of my errors)

There's actually 2 parts to it as well

i) np.int has become np.integer and is the 'base' data type of all specific np.int<number of bits> types
ii) they no longer allow dtype=np.integer which always defaulted a system specific np._int value - the modern recommendation is to set it to an explicit size

optogeneticsTimeBtwTrains = np.zeros(optogeneticsN)
optogeneticsTargetX = np.zeros(optogeneticsN)
optogeneticsTargetY = np.zeros(optogeneticsN)
Expand Down Expand Up @@ -674,9 +676,9 @@ def load_optogenetics(self):
Line.pop(0)
if Line[-1]=="": Line.pop(-1)
self.optogeneticsN = len(Line)
self.optogeneticsFrameCount = np.zeros(self.optogeneticsN, dtype=np.int)
self.optogeneticsNPulses = np.zeros(self.optogeneticsN, dtype=np.int)
self.optogeneticsRepRateDivider = np.zeros(self.optogeneticsN, dtype=np.int)
self.optogeneticsFrameCount = np.zeros(self.optogeneticsN, dtype=np.int64)
self.optogeneticsNPulses = np.zeros(self.optogeneticsN, dtype=np.int64)
self.optogeneticsRepRateDivider = np.zeros(self.optogeneticsN, dtype=np.int64)
self.optogeneticsTargetX = np.zeros(self.optogeneticsN)
self.optogeneticsTargetY = np.zeros(self.optogeneticsN)
self.optogeneticsTargetZ = np.zeros(self.optogeneticsN)
Expand All @@ -686,7 +688,7 @@ def load_optogenetics(self):
self.optogeneticsTime = ["None"]*self.optogeneticsN

#These won't be populated in this case
self.optogeneticsNTrains = np.zeros(self.optogeneticsN, dtype=np.int)
self.optogeneticsNTrains = np.zeros(self.optogeneticsN, dtype=np.int64)
self.optogeneticsTimeBtwTrains = np.zeros(self.optogeneticsN)

for i in np.arange(self.optogeneticsN):
Expand All @@ -707,10 +709,10 @@ def load_optogenetics(self):
Line.pop(0)
if Line[-1]=="": Line.pop(-1)
self.optogeneticsN = len(Line)
self.optogeneticsFrameCount = np.zeros(self.optogeneticsN, dtype=np.int)
self.optogeneticsNPulses = np.zeros(self.optogeneticsN, dtype=np.int)
self.optogeneticsRepRateDivider = np.zeros(self.optogeneticsN, dtype=np.int)
self.optogeneticsNTrains = np.zeros(self.optogeneticsN, dtype=np.int)
self.optogeneticsFrameCount = np.zeros(self.optogeneticsN, dtype=np.int64)
self.optogeneticsNPulses = np.zeros(self.optogeneticsN, dtype=np.int64)
self.optogeneticsRepRateDivider = np.zeros(self.optogeneticsN, dtype=np.int64)
self.optogeneticsNTrains = np.zeros(self.optogeneticsN, dtype=np.int64)
self.optogeneticsTimeBtwTrains = np.zeros(self.optogeneticsN)
self.optogeneticsTargetX = np.zeros(self.optogeneticsN)
self.optogeneticsTargetY = np.zeros(self.optogeneticsN)
Expand Down
3 changes: 2 additions & 1 deletion wormdatamodel/data/volume.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import numpy as np
import mistofrutta as mf

class volume:

Expand All @@ -10,4 +9,6 @@ def __init__(self, frames, z, dx=0.4, dy=0.4):
self.z = z

def plot(self, wait=False):
import mistofrutta as mf

mf.plt.hyperstack(self.frames, order='zc', cmap='viridis', wait=wait)
14 changes: 9 additions & 5 deletions wormdatamodel/signal/signal.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
# Authors: Milena Chakraverti-Wuerthwein and Francesco Randi

import numpy as np
import matplotlib.pyplot as plt
import warnings
import os
import pickle
Expand All @@ -12,9 +11,7 @@
from scipy.io import loadmat as sioloadmat
from copy import deepcopy as deepcopy
from datetime import datetime
import mistofrutta.struct.irrarray as irrarray
import wormdatamodel as wormdm
import savitzkygolay as sg
Comment on lines -15 to -17

@CodyCBakerPhD CodyCBakerPhD May 6, 2024

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I moved each of these into the scopes of the functions that use them so someone can import the wormdatamodel package without needing to find each GitHub repo and do local installs

Using each function would still error if the package is missing, but at least that only occurs when (and 'if', in my case) those functions would be called

@CodyCBakerPhD CodyCBakerPhD May 6, 2024

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

An alternative to this, if you prefer to keep things as they are, might be to include git-based installations in the setup requirements for this package

That could however cascade a series of PRs like this one if there's any trouble installing those packages however

from sklearn.decomposition import FastICA

class Signal:
Expand Down Expand Up @@ -95,6 +92,7 @@ def __init__(self, data, info, description = None,
Detect and correct instantaneous photobleaching. Use only for the
red signal. Default: False.
'''
import mistofrutta.struct.irrarray as irrarray

self.data = data;
self.info = info;
Expand Down Expand Up @@ -630,6 +628,8 @@ def median_filter(self, i=None):
self.log("Median filtering on "+affected_neurons,False)

def get_derivative(self,data,n,poly):
import savitzkygolay as sg

deriv = np.zeros_like(data)
derker = sg.get_1D_filter(n,poly,1)
#derker = savgol_coeffs(n,poly,deriv=1)
Expand Down Expand Up @@ -823,6 +823,7 @@ def appl_photobl(self, j=None, verbose=True):
#def get_photobl_fit(self, X, k=None):

def corr_inst_photobl(self, j=None, poly_width=111, photobl_duration=3, min_distance=30):
import savitzkygolay as sg

if j is None:
iterate_over = np.arange(self.data.shape[1])
Expand Down Expand Up @@ -923,7 +924,8 @@ def trim(self,stride_name, adjust = None):
trimmed.data can now be copied and reshaped into a multidimensional
numpy array.
'''

import mistofrutta.struct.irrarray as irrarray

try:
start = self.data.firstIndex[stride_name];
strideLength = np.diff(start);
Expand Down Expand Up @@ -1064,6 +1066,8 @@ def rolling_window(a, window):

@staticmethod
def get_causal_sg(n,poly):
import savitzkygolay as sg

order = np.arange(poly+1)
out = np.zeros(n)
for i in order:
Expand Down Expand Up @@ -1098,7 +1102,7 @@ def remove_outliers(y,std_th=2.):
y2[outl] = y2[outl-1]

return y2

@classmethod
def get_matchless_nan_th_from_file(cls,folder):
fname = folder+cls.matchless_nan_th_fname
Expand Down