diff --git a/PyMca5/PyMcaCore/SpecFileDataSource.py b/PyMca5/PyMcaCore/SpecFileDataSource.py index c65a003b1..abd0f7814 100644 --- a/PyMca5/PyMcaCore/SpecFileDataSource.py +++ b/PyMca5/PyMcaCore/SpecFileDataSource.py @@ -239,6 +239,9 @@ def __getScanInfo(self, scankey): sourceObject = self._sourceObjectList[index] scandata= sourceObject.select(scankey) + _logger.info("__getScanInfo %s", scandata) + + info={} info["SourceType"] = SOURCE_TYPE #doubts about if refer to the list or to the individual file diff --git a/PyMca5/PyMcaCore/TiledTools.py b/PyMca5/PyMcaCore/TiledTools.py new file mode 100644 index 000000000..8dcf85d02 --- /dev/null +++ b/PyMca5/PyMcaCore/TiledTools.py @@ -0,0 +1,107 @@ +from tiled.client import from_uri,from_profile + +from databroker.queries import TimeRange +from tiled.queries import In + +##fixme ... needs arg to connect to corret service +#def get_tiled_connection(): +# return from_uri("https://tiled-demo.blueskyproject.io") + +class TiledAdaptor(object): + @classmethod + def get_nested(cls,d, keys): + #FIXME to be done more efficient + for k in keys: + d = d[k] + return d + + def __init__(self,host,prefix=None): + print("init TiledAdaptor",host) + if "http" in host: + self._client = from_uri(host) + else: + self._client = from_profile(host) + + #specific hack for shen beamtime + if host=="opls": + #specific hack for shen beamtime, just to reduce the scope + from tiled.queries import FullText + #self._client = self._client.search(TimeRange(since='2024-03-02 01:00', until='2024-03-02 02:00')) + self._client = self._client.search(In("plan_name",["reflection_scan","rel_scan"])).search(TimeRange(since='2024-03-01', until='2024-03-02 07:00')) + + elif prefix: + self._client = TiledAdaptor.get_nested(self._client,prefix.split("/")) + + self.__prefix=prefix + print(self._client) + + def get_node(self,path=None,scan_id=None): + + if not scan_id is None: + if path and "/" in path: + return self.get_nested(self._client[scan_id],path.split("/")) + elif not path: + return self._client[scan_id] + else: + return self._client[scan_id][path] + else: + if "/" in path: + return self.get_nested(self._client,path.split("/")) + elif not path: + return self._client + else: + return self._client[path] + + + @property + def client(self): + return self._client + + # @property + # def _sourceName(self): + # return self.__sourceName + + # @_sourceName.setter + # def _sourceName(self,sn): + # self.__sourceName = sn + + # def close(self): + # #not sure if there is anything that needs to happen here + # return + + # def getSourceInfo(self): + # """ + # Returns a dictionary with the key "KeyList" (list of all available keys + # in this source). Each element in "KeyList" has the form 'n1.n2' where + # n1 is the source number and n2 entry number in file both starting at 1. + # """ + # print("TiledAdaptor getSourceInfo") + # return {"KeyList":["1.1"]} + + # @property + # def name(self): + # return "/" + + # @property + # def filename(self): + # return "toto" + +#very simplistic and "bad" coding ... just going global tiled connection for now + +#just to be a little quicker ... only work against fxi first +_TILED_CLIENT_opls=TiledAdaptor("opls") +_TILED_CLIENT_fxi=TiledAdaptor("https://tiled-demo.blueskyproject.io","fxi/raw") + + +def get_sessions_list(): + return ["opls","demo:fxi"] + +def get_node(path): + #path ... to be resolved from tiled... + if path=="opls": + return _TILED_CLIENT_opls + if path=="demo:fxi": + return _TILED_CLIENT_fxi + else: + print(">>> trying to access: ",path) + raise \ No newline at end of file diff --git a/PyMca5/PyMcaGui/io/QSourceSelector.py b/PyMca5/PyMcaGui/io/QSourceSelector.py index 0f834b06b..e393f2de5 100644 --- a/PyMca5/PyMcaGui/io/QSourceSelector.py +++ b/PyMca5/PyMcaGui/io/QSourceSelector.py @@ -39,14 +39,17 @@ from PyMca5 import PyMcaDirs from PyMca5.PyMcaGui.io import PyMcaFileDialogs -BLISS = False -if sys.version_info > (3, 5): - try: - from PyMca5.PyMcaCore import RedisTools - BLISS = True - except Exception: - _logger.info("Bliss data file direct support not available") - +#Fixme just a quick fix ... dropin replacement of bliss with tiled/bluesky +BLISS = True +# if sys.version_info > (3, 5): +# try: +# from PyMca5.PyMcaCore import RedisTools +# BLISS = True +# except Exception: +# _logger.info("Bliss data file direct support not available") + +from PyMca5.PyMcaCore import TiledTools +#from PyMca5.PyMcaCore import RedisTools class QSourceSelector(qt.QWidget): sigSourceSelectorSignal = qt.pyqtSignal(object) @@ -156,7 +159,7 @@ def openSource(self, sourcename, specsession=None): if specsession is None: if sourcename in sps.getspeclist(): specsession=True - elif BLISS and sourcename in RedisTools.get_sessions_list(): + elif BLISS and sourcename in TiledTools.get_sessions_list(): specsession = "bliss" else: specsession=False @@ -168,25 +171,26 @@ def openFile(self, filename=None, justloaded=None, specsession=False): if specsession == "bliss": specsession = False session = filename - node = RedisTools.get_node(session) + node = TiledTools.get_node(session) if not node: txt = "No REDIS information retrieved from session %s" % \ session raise IOError(txt) - filename = RedisTools.get_session_filename(node) - if not len(filename): - txt = "Cannot retrieve last output filename from session %s" % \ - session - raise IOError(txt) - if not os.path.exists(filename): - txt = "Last output file <%s> does not exist" % filename - raise IOError(txt) - filename = [filename] - key = os.path.basename(filename[0]) - try: - self._emitSourceSelectedOrReloaded(filename, key) - except Exception: - _logger.error("Problem opening %s" % filename[0]) + #NOT GOING HERE AS WE DON'T WANT TO BE HYBRID FILE+DB FOR TILED + # + # if not len(filename): + # txt = "Cannot retrieve last output filename from session %s" % \ + # session + # raise IOError(txt) + # if not os.path.exists(filename): + # txt = "Last output file <%s> does not exist" % filename + # raise IOError(txt) + # filename = [filename] + # key = os.path.basename(filename[0]) + # try: + # self._emitSourceSelectedOrReloaded(filename, key) + # except Exception: + # _logger.error("Problem opening %s" % filename[0]) key = "%s" % session self._emitSourceSelectedOrReloaded([session], key) return @@ -289,12 +293,12 @@ def closeFile(self): def openBlissOrSpec(self): if not BLISS: return self.openSpec() - sessionList = RedisTools.get_sessions_list() + sessionList = TiledTools.get_sessions_list() if not len(sessionList): return self.openSpec() activeList = [] for session in sessionList: - node = RedisTools.get_node(session) + node = TiledTools.get_node(session) if node: activeList.append(session) if not len(activeList): diff --git a/PyMca5/PyMcaGui/io/QSpecFileWidget.py b/PyMca5/PyMcaGui/io/QSpecFileWidget.py index 0b6900c74..95673dbf6 100644 --- a/PyMca5/PyMcaGui/io/QSpecFileWidget.py +++ b/PyMca5/PyMcaGui/io/QSpecFileWidget.py @@ -390,6 +390,10 @@ def __selectionChanged(self): if not len(sel): return info = self.data.getKeyInfo(sel[0]) + _logger.info("__selectionChanged info data %s",self.data) + _logger.info("__selectionChanged info %s",info) + _logger.info("__selectionChanged sel %s",sel) + _logger.info("__selectionChanged sel[0] %s",sel[0]) self.mcaTable.build(info) if False: # This does not work properly yet diff --git a/PyMca5/PyMcaGui/plotting/PyMca_Icons.py b/PyMca5/PyMcaGui/plotting/PyMca_Icons.py index b13251826..3ec387c6e 100644 --- a/PyMca5/PyMcaGui/plotting/PyMca_Icons.py +++ b/PyMca5/PyMcaGui/plotting/PyMca_Icons.py @@ -3585,7 +3585,43 @@ "..##################..", "......................"] - +bluesky = [ +"32 32 2 1 ", +" c None", +"B c #87CEEB", +" ", +" ", +" ", +" ", +" ", +" ", +" ", +" BBBBBBB ", +" BBBBBBB ", +" BBBBBBBBBBBBBB ", +" BBBBBBBBBBBBBB ", +" BBBBBBBBBBBBBBBBBB ", +" BBBBBBBBBBBBBBBBBBB BBB ", +" BBBBBBBBBBBBBBBBBBB BBB ", +" BBBBBBBBBBBBBBBBBBBB BBBBBBBB ", +" BBBBBBBBBBBBBBBBBBBB BBBBBBBB ", +" BBBBBBBBBBBBBBBBBBBB BBBB BB", +" BBBBBBBBBBBBBBBBBBBB BBBB BB", +" BBBBBBBBBBBBBBBBBBBB BBBBBB BB", +" BBBBBBBBBBBBBBBBBBBB BBBBBB BB", +" BBBBBBBBBBBBBBBBBBBB BBBB BB ", +" BBBBBBBBBBBBBBBBBBBB BBBB BB ", +" BBBBBBBBBBBBBBBBBBBBBBBBB ", +" BBBBBBBBBBBBBBBBBBBBBBBBB ", +" ", +" ", +" ", +" ", +" ", +" ", +" ", +" ", +] IconDict0 = { "derive": derive, @@ -3596,7 +3632,7 @@ "filesave": file_save, "fileprint": image_print_data, "spec": spec, - "bliss": bliss, + "bliss": bluesky, "normal": normal, "normalize16": normalize16, "reload": reload_, diff --git a/PyMca5/PyMcaGui/pymca/QDataSource.py b/PyMca5/PyMcaGui/pymca/QDataSource.py index 55e3c6570..7aa0bf935 100644 --- a/PyMca5/PyMcaGui/pymca/QDataSource.py +++ b/PyMca5/PyMcaGui/pymca/QDataSource.py @@ -42,6 +42,9 @@ from PyMca5.PyMcaGui.io import QEdfFileWidget from PyMca5.PyMcaGui.io import QSpecFileWidget +import logging +_logger = logging.getLogger(__name__) + if sys.platform == "win32": source_types = { SpecFileDataSource.SOURCE_TYPE: SpecFileDataSource.SpecFileDataSource, EdfFileDataSource.SOURCE_TYPE: EdfFileDataSource.EdfFileDataSource} @@ -81,6 +84,10 @@ def getSourceType(sourceName0): else: sourceName = sourceName0 + #FIXME: just a quick and dirty fix to get going with tiled + if sourceName == "opls" or sourceName == "demo:fxi": + return SpecFileDataSource.SOURCE_TYPE + if BlissSpecFile.isBlissSpecFile(sourceName): # wrapped as SpecFile return SpecFileDataSource.SOURCE_TYPE @@ -167,6 +174,8 @@ def QDataSource(name=None, source_type=None): except KeyError: #ERROR invalid source type raise TypeError("Invalid Source Type, source type should be one of %s" % source_types.keys()) + _logger.debug("source_types %s", source_types, ) + _logger.debug("sourceClass %s, name %s", sourceClass, name) return sourceClass(name) diff --git a/PyMca5/PyMcaIO/BlissSpecFile.py b/PyMca5/PyMcaIO/BlissSpecFile.py index 03609a10a..803566574 100644 --- a/PyMca5/PyMcaIO/BlissSpecFile.py +++ b/PyMca5/PyMcaIO/BlissSpecFile.py @@ -41,16 +41,20 @@ import logging _logger = logging.getLogger(__name__) -try: - import RedisTools as redis - HAS_REDIS = True -except Exception: - try: - import PyMca5.PyMcaCore.RedisTools as redis - HAS_REDIS = True - except Exception: - _logger.info("Cannot import PyMca5.PyMcaCore.RedisTools") - HAS_REDIS = False +from PyMca5.PyMcaCore import TiledTools +HAS_REDIS = True + + +# try: +# import RedisTools as redis +# HAS_REDIS = True +# except Exception: +# try: +# import PyMca5.PyMcaCore.RedisTools as redis +# HAS_REDIS = True +# except Exception: +# _logger.info("Cannot import PyMca5.PyMcaCore.RedisTools") +# HAS_REDIS = False if HAS_REDIS: from collections import OrderedDict @@ -59,28 +63,28 @@ class BlissSpecFile(object): def __init__(self, filename, nscans=10): """ filename is the name of the bliss session + + nscans to be ignored for tiled """ - if not HAS_REDIS: - raise ImportError("Could not import RedisTools") - if filename not in redis.get_sessions_list(): - raise IOError("Session <%s> not available" % filename) self._scan_nodes = [] self._session = filename - self._filename = redis.get_session_filename(self._session) - self._scan_nodes = redis.get_session_scan_list(self._session, - self._filename) - if len(self._scan_nodes) > nscans: - self._scan_nodes = self._scan_nodes[-10:] + # self._filename = redis.get_session_filename(self._session) + self._tiledAdaptor=TiledTools.get_node(filename) + + #prefer to refer directly to tiled root, therefor no scan_nodes + #self._scan_nodes = TiledTools.get_node(filename) + self.list() self.__lastTime = 0 self.__lastKey = "0.0" + self._updatedOnce=False def list(self): """ Return a string with all the scan keys separated by , """ - _logger.debug("list method called") - scanlist = ["%s" % scan.name.split("_")[0] for scan in self._scan_nodes] + _logger.info("list method called") + scanlist = [f"{x.metadata['summary']['scan_id']}" for x in self._tiledAdaptor.client.values()] self._list = ["%s.1" % idx for idx in scanlist] return ",".join(scanlist) @@ -90,7 +94,9 @@ def __getitem__(self, item): """ _logger.info("__getitem__ called %s" % item) t0 = time.time() + _logger.info("trying to access %s",item) key = self._list[item] + _logger.info("got item %s", key) if key == self.__lastKey and (t0 - self.__lastTime) < 1: # less than one second since last call, return cached value _logger.info("Returning cached value for key %s" % key) @@ -98,7 +104,7 @@ def __getitem__(self, item): if key == self.__lastKey: _logger.info("Re-reading value for key %s" % key) self.__lastKey = key - self.__lastItem = BlissSpecScan(self._scan_nodes[item]) + self.__lastItem = TiledScan(key,self._tiledAdaptor) self.__lastTime = time.time() return self.__lastItem @@ -115,54 +121,181 @@ def scanno(self): """ Gives back the number of scans in the file """ - _logger.debug("scanno called") - return len(self._scan_nodes) + _logger.info("scanno called") + return len(self._tiledAdaptor._client) def allmotors(self): - _logger.debug("allmotors called") + _logger.info("allmotors called") return [] def isUpdated(self): - _logger.debug("BlissSpecFile is updated called") - # get last scan - scan_nodes = redis.get_session_scan_list(self._session, - self._filename) - if not len(scan_nodes): - # if we get no scans, information was emptied/lost and we'll get errors in any case - # just say the file was updated. Perhaps the application asks for an update - return True - scanlist = ["%s" % scan.name.split("_")[0] for scan in scan_nodes] - keylist = ["%s.1" % idx for idx in scanlist] - scankey = keylist[-1] - - # if the last node is different, there are new data - if scankey != self._list[-1]: + _logger.info("BlissSpecFile is updated called") + + if not self._updatedOnce: + self._updatedOnce=True return True - # if the number of points or of mcas in the last node are different there are new data - # the problem is how to obtain the previous number of points and mcas but in any case - # we are going to read again the last scan - if self.__lastKey == scankey: - # we have old data available - previous_npoints = self.__lastItem.lines() - previous_nmca = self.__lastItem.nbmca() - - # read back (I do not force to read for the time being) - scan = self.select(scankey) - npoints = scan.lines() - nmca = scan.nbmca() - if self.__lastKey == scankey: - if npoints > previous_npoints or nmca > previous_nmca: - _logger.info("BlissSpecFile <%s> updated. New last scan data" % self._session) - return True - # there might be new points or mcas in the last scan, but that is easy and fast - # to check by the main application because data are in cache - _logger.debug("BlissSpecFile <%s> NOT updated." % self._session) + #FIXME: UPDATE not implemented for tiled + + # # get last scan + # scan_nodes = redis.get_session_scan_list(self._session, + # self._filename) + # if not len(scan_nodes): + # # if we get no scans, information was emptied/lost and we'll get errors in any case + # # just say the file was updated. Perhaps the application asks for an update + # return True + # scanlist = ["%s" % scan.name.split("_")[0] for scan in scan_nodes] + # keylist = ["%s.1" % idx for idx in scanlist] + # scankey = keylist[-1] + + # # if the last node is different, there are new data + # if scankey != self._list[-1]: + # return True + + # # if the number of points or of mcas in the last node are different there are new data + # # the problem is how to obtain the previous number of points and mcas but in any case + # # we are going to read again the last scan + # if self.__lastKey == scankey: + # # we have old data available + # previous_npoints = self.__lastItem.lines() + # previous_nmca = self.__lastItem.nbmca() + + # # read back (I do not force to read for the time being) + # scan = self.select(scankey) + # npoints = scan.lines() + # nmca = scan.nbmca() + # if self.__lastKey == scankey: + # if npoints > previous_npoints or nmca > previous_nmca: + # _logger.info("BlissSpecFile <%s> updated. New last scan data" % self._session) + # return True + # # there might be new points or mcas in the last scan, but that is easy and fast + # # to check by the main application because data are in cache + # _logger.info("BlissSpecFile <%s> NOT updated." % self._session) return False +class TiledScan(object): + def __init__(self, scan_ref, tiled_adaptor): + self._scan_id=int(scan_ref.split(".")[0]) + self._scoped_tiled=tiled_adaptor.get_node(scan_id=self._scan_id) + + #following the nsls2 specific structure in tiled + self._counters=None + + try: + self._primary_data=self._scoped_tiled["primary"]["data"] + self._tiledAdaptor=tiled_adaptor + self._read_counters() + except Exception: + print("Scan ", scan_ref, "not valid") + self._counters={} + + + + + def _read_counters(self, force=False): + _logger.info("_red_counters") + self._counters={x:(y.shape) for x,y in self._primary_data.items() if len(y.shape)==1} + + def _sort_counters(self, counters): + _logger.info("_sort_counters") + + def alllabels(self): + """ + These are the labels associated to the counters + """ + _logger.info("alllabels called") + if self._counters is None: + self.read_counters() + return [key for key in self._counters] + + def allmotors(self): + _logger.info("allmotors called") + + def allmotorpos(self): + _logger.info("allmotorpos called") + + def cols(self): + _logger.info("cols called") + + def command(self): + _logger.info("command called") + return "still to be added" + #return self._scan_info.get("title", "No COMMAND") + + def data(self): + # somehow I have to manage to get the same number of points in all counters + _logger.info("TiledScan data called") + keys = list(self._counters.keys()) + n_expected = self.lines() + _logger.info("TiledScan data before data") + data = numpy.empty((len(keys), n_expected), dtype=numpy.float32) + _logger.info("TiledScan data after data") + + _logger.info("data shape %s ",data.shape) + i = 0 + try: + for key in keys: + cdata = numpy.array(self._primary_data[key]) + n = cdata.size + if n >= n_expected: + data[i] = cdata[:n_expected] + else: + data[i, :n] = cdata + data[i, n:n_expected] = numpy.nan + i += 1 + except Exception as e: + print(e) + _logger.info("data shape %s, data %s",data.shape,data) + return data + + def datacol(self, col): + _logger.info("datacol called") + return self.data()[col, :] + + def dataline(self,line): + _logger.info("dataline called") + return self.data()[:, line] + + def date(self): + _logger.info("date called") + return "2024-04-23" #FIXME + + def fileheader(self, key=''): + _logger.info("fileheader called") + # this implementations returns the scan header instead of the correct + # keys #E (file), #D (date) #O0 (motor names) + + + def header(self,key): + _logger.info("header called") + + def order(self): + _logger.info("order called") + return 1 + + def number(self): + _logger.info("number called") + + def lines(self): + _logger.info("lines called") + if self._counters is None: + self._read_counters() + ##return number of counters + + return max(self._counters.values())[0] + + def nbmca(self): + _logger.info("nbmca called") + return 0 + + def mca(self,number): + _logger.info("mca called") + return 0 + +""" class BlissSpecScan(object): def __init__(self, scanNode): - _logger.debug("__init__ called %s" % scanNode.name) + _logger.info("__init__ called %s" % scanNode.name) self._node = scanNode self._identification = scanNode.name.split("_")[0] + ".1" self._scan_info = redis.scan_info(self._node) @@ -234,20 +367,18 @@ def _sort_counters(self, counters): return ordered def alllabels(self): - """ - These are the labels associated to the counters - """ - _logger.debug("alllabels called") + #These are the labels associated to the counters + _logger.info("alllabels called") self._read_counters() return [key for key in self._counters] def allmotors(self): - _logger.debug("allmotors called") + _logger.info("allmotors called") positioners = self._motors.get("positioners_start", {}) return [key for key in positioners if not hasattr(positioners[key], "endswith")] def allmotorpos(self): - _logger.debug("allmotorpos called") + _logger.info("allmotorpos called") positioners = self._motors.get("positioners_start", {}) return [positioners[key] for key in positioners if not hasattr(positioners[key], "endswith")] @@ -262,7 +393,7 @@ def command(self): def data(self): # somehow I have to manage to get the same number of points in all counters - _logger.info("data called") + _logger.info("BlissSpecScan data called") self._read_counters(force=True) counters = self._counters keys = list(counters.keys()) @@ -368,7 +499,7 @@ def mca(self,number): raise IndexError("Mca numbering starts at 1") elif number > self.nbmca(): raise IndexError("Only %d MCAs in file" % self.nbmca()) - return self._spectra[0][number - 1] + return self._spectra[0][number - 1] """ def isBlissSpecFile(filename): if os.path.exists(filename): diff --git a/PyMca5/PyMcaIO/specfilewrapper.py b/PyMca5/PyMcaIO/specfilewrapper.py index 41e4b63ea..54cd48e2f 100644 --- a/PyMca5/PyMcaIO/specfilewrapper.py +++ b/PyMca5/PyMcaIO/specfilewrapper.py @@ -76,6 +76,11 @@ def bytes(*var, **kw): return var[0] def Specfile(filename): + + #FIXME: just a quick and dirty fix to get going with tiled + if filename == "opls" or filename == "demo:fxi": + return BlissSpecFile.BlissSpecFile(filename) + if BlissSpecFile.isBlissSpecFile(filename): return BlissSpecFile.BlissSpecFile(filename) if sys.version_info < (3, 0): diff --git a/PyMca5/__init__.py b/PyMca5/__init__.py index eb68f9530..deb9ad27a 100644 --- a/PyMca5/__init__.py +++ b/PyMca5/__init__.py @@ -47,9 +47,9 @@ import ctypes from ctypes.wintypes import MAX_PATH -if os.path.exists(os.path.join(\ - os.path.dirname(os.path.dirname(__file__)), 'bootstrap.py')): - raise ImportError('PyMca cannot be imported from source directory') +#if os.path.exists(os.path.join(\ +# os.path.dirname(os.path.dirname(__file__)), 'bootstrap.py')): +# raise ImportError('PyMca cannot be imported from source directory') def version(): return __version__