Refactor 24 trend

.gitignore

@@ -41,6 +41,11 @@ venv
 tests.log
 /.pytest_cache
 
+# Unit test / coverage reports
+htmlcov/
+.coverage
+.cache/
+
 # Installer files
 /installers/build
 /installers/dist

.pre-commit-config.yaml

@@ -2,7 +2,7 @@ default_install_hook_types: [pre-commit, pre-push]
 
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.14.1
+    rev: v0.12.9
     hooks:
       # Run the linter, applying any available fixes
       - id: ruff-check

docs/source/user/data/data_formats_help.rst

@@ -55,7 +55,7 @@ separated by whitespaces or commas or semicolons, in the following order:
 
 where *Q* is assumed to have units of 1/Angstrom, *I(Q)* is assumed to have
 units of 1/cm, *dI(Q)* is the uncertainty on the intensity value (also as 1/cm),
-and *dQ(Q)* **is the one-sigma FWHM Gaussian instrumental resolution in** *Q*,
+and *dQ(Q)* **is the one-sigma Gaussian instrumental resolution in** *Q*,
 **assumed to have arisen from pinhole geometry**. If the data are slit-smeared,
 see `Slit-Smeared Data`_.
 

requirements.txt

@@ -16,4 +16,4 @@ html5lib
 
 # Other stuff
 matplotlib
-pre-commit
+pre-commit

sasdata/ascii_reader_metadata.py

@@ -26,7 +26,7 @@
 T = TypeVar('T')
 
 # TODO: There may be a better place for this.
-pairings = {'I': 'dI', 'Q': 'dQ', 'Qx': 'dQx', 'Qy': 'dQy'}
+pairings = {'I': 'dI', 'Q': 'dQ', 'Qx': 'dQx', 'Qy': 'dQy', 'Qz': 'dQz'}
 pairing_error = {value: key for key, value in pairings.items()}
 # Allows this to be bidirectional.
 bidirectional_pairings = pairings | pairing_error

sasdata/checklist.txt

@@ -0,0 +1,3 @@
+Things to check once everything is in place:
+
+1) Do any centigrade fields read in incorrectly?

sasdata/data.py

@@ -69,7 +69,7 @@ def abscissae(self) -> Quantity:
                 # TODO: Won't work when there's errors involved. On reflection, we
                 # probably want to avoid creating a new Quantity but at the moment I
                 # can't see a way around it.
-                return Quantity(data_contents, reference_data_content.units)
+                return Quantity(data_contents, reference_data_content.units, name=self._data_contents["Qx"].name, id_header=self._data_contents["Qx"]._id_header)
             case dataset_types.sesans:
                 return self._data_contents["SpinEchoLength"]
             case _:

sasdata/data_backing.py

@@ -27,7 +27,7 @@ def summary(self, indent_amount: int = 0, indent: str = "  ") -> str:
         s += f"{indent*(indent_amount+1)}{shorten_string(str(self.data))}\n"
         for key in self.attributes:
             value = self.attributes[key]
-            if isinstance(value, (Group | Dataset)):
+            if isinstance(value, (Group, Dataset)):
                 value_string = value.summary(indent_amount+1, indent)
             else:
                 value_string = f"{indent * (indent_amount+1)}{key}: {shorten_string(repr(value))}\n"

sasdata/data_util/err1d.py

@@ -1,6 +1,6 @@
 # This program is public domain
 """
-Error propogation algorithms for simple arithmetic
+Error propagation algorithms for simple arithmetic
 
 Warning: like the underlying numpy library, the inplace operations
 may return values of the wrong type if some of the arguments are

sasdata/data_util/nxsunit.py

@@ -51,8 +51,8 @@
 __all__ = ['Converter', 'standardize_units']
 T = TypeVar('T')
 ConversionType = float | tuple[float, float]
-DIMENSIONS = {}  # type: Dict[str, Dict[str, ConversionType]]
-AMBIGUITIES = {}  # type: Dict[str, str]
+DIMENSIONS: dict[str, dict[str, ConversionType]] = {}
+AMBIGUITIES: dict[str, str] = {}
 PREFIX = dict(peta=1e15, tera=1e12, giga=1e9, mega=1e6, kilo=1e3, deci=1e-1, centi=1e-2, milli=1e-3, mili=1e-3,
               micro=1e-6, nano=1e-9, pico=1e-12, femto=1e-15)
 SHORT_PREFIX = dict(P=1e15, T=1e12, G=1e9, M=1e6, k=1e3, d=1e-1, c=1e-2, m=1e-3, u=1e-6, n=1e-9, p=1e-12, f=1e-15)
@@ -258,7 +258,7 @@ def _build_all_units():
     # APS files may be using 'a.u.' for 'arbitrary units'.  Other
     # facilities are leaving the units blank, using ??? or not even
     # writing the units attributes.
-    unknown = {}  # type: Dict[str, ConversionType]
+    unknown: dict[str, ConversionType] = {}
     unknown.update(
         {'None': 1, '???': 1, '': 1, 'A.U.': 1,  'a.u.': 1, 'arbitrary': 1, 'arbitrary units': 1,
          'Counts': 1, 'counts': 1, 'Cts': 1, 'cts': 1, 'unitless': 1, 'unknown': 1, 'Unknown': 1, 'Unk': 1}
@@ -356,15 +356,15 @@ class Converter:
     value name.
     """
     #: Name of the source units (km, Ang, us, ...)
-    _units = None  # type: List[str]
+    _units: list[str] = None
     #: Type of the source units (distance, time, frequency, ...)
-    dimension = None  # type: List[str]
+    dimension: list[str] = None
     #: Scale converter, mapping unit name to scale factor or (scale, offset)
     #: for temperature units.
-    scalemap = None  # type: List[Dict[str, ConversionType]]
+    scalemap: list[dict[str, ConversionType]] = None
     #: Scale base for the source units
-    scalebase = None  # type: float
-    scaleoffset = None  # type: float
+    scalebase: float = None
+    scaleoffset: float = None
 
     @property
     def units(self) -> str:
@@ -375,7 +375,7 @@ def units(self, unit: str):
         self._units = standardize_units(unit)
 
     def __init__(self, units: str | None = None, dimension: list[str] | None = None):
-        self.units = units if units is not None else 'a.u.'  # type: str
+        self.units: str = units if units is not None else 'a.u.'
 
         # Lookup dimension if not given
         if dimension:

sasdata/dataloader/data_info.py

@@ -768,7 +768,7 @@ def is_slit_smeared(self):
         :return: True is slit smearing info is present, False otherwise
         """
         def _check(v):
-            return (isinstance(v.__class__ == list | np.ndarray)
+            return (isinstance(v.__class__, list | np.ndarray)
                     and len(v) > 0 and min(v) > 0)
         return _check(self.dxl) or _check(self.dxw)
 

sasdata/dataloader/readers/red2d_reader.py

@@ -82,7 +82,6 @@ def get_file_contents(self):
 
         wavelength = None
         distance = None
-        transmission = None
 
         pixel_x = None
         pixel_y = None

sasdata/dataset_types.py

@@ -27,12 +27,21 @@ class DatasetType:
 two_dim = DatasetType(
             name="2D I vs Q",
             required=["Qx", "Qy", "I"],
-            optional=["dQx", "dQy", "dI", "Qz", "ShadowFactor", "mask"],
+            optional=["dQx", "dQy", "dQz", "dI", "Qz", "ShadowFactor", "mask"],
             expected_orders=[
                 ["Qx", "Qy", "I"],
                 ["Qx", "Qy", "I", "dI"],
                 ["Qx", "Qy", "dQx", "dQy", "I", "dI"]])
 
+three_dim = DatasetType(
+            name="3D I vs Q",
+            required=["Qx", "Qy", "Qz", "I"],
+            optional=["dQx", "dQy", "dQz", "dI", "ShadowFactor", "mask"],
+            expected_orders=[
+                ["Qx", "Qy", "Qz", "I"],
+                ["Qx", "Qy", "Qz", "I", "dI"],
+                ["Qx", "Qy", "Qz", "dQx", "dQy", "dQz", "I", "dI"]])
+
 sesans = DatasetType(
     name="SESANS",
     required=["SpinEchoLength", "Depolarisation", "Wavelength"],

sasdata/distributions.py

@@ -0,0 +1,11 @@
+
+
+class DistributionModel:
+
+
+    @property
+    def is_density(self) -> bool:
+        return False
+
+    def standard_deviation(self) -> Quantity:
+        return NotImplementedError("Variance not implemented yet")

sasdata/manual_tests/interpolation.py

@@ -0,0 +1,43 @@
+import matplotlib.pyplot as plt
+import numpy as np
+
+from sasdata.quantities import units
+from sasdata.quantities.plotting import quantity_plot
+from sasdata.quantities.quantity import NamedQuantity
+from sasdata.transforms.rebinning import InterpolationOptions, calculate_interpolation_matrix_1d
+
+
+def linear_interpolation_check():
+
+    for from_bins in [(-10, 10, 10),
+                      (-10, 10, 1000),
+                      (-15, 5, 10),
+                      (15,5, 10)]:
+        for to_bins in [
+            (-15, 0, 10),
+            (-15, 15, 10),
+            (0, 20, 100)]:
+
+            plt.figure()
+
+            x = NamedQuantity("x", np.linspace(*from_bins), units=units.meters)
+            y = x**2
+
+            quantity_plot(x, y)
+
+            new_x = NamedQuantity("x_new", np.linspace(*to_bins), units=units.meters)
+
+            rebin_mat = calculate_interpolation_matrix_1d(x, new_x, order=InterpolationOptions.LINEAR)
+
+            new_y = y @ rebin_mat
+
+            quantity_plot(new_x, new_y)
+
+            print(new_y.history.summary())
+
+    plt.show()
+
+
+
+
+linear_interpolation_check()

sasdata/metadata.py

@@ -458,14 +458,19 @@ def to_string(self, header=""):
             )
         else:
             attributes = ""
-        if self.contents:
-            if type(self.contents) is str:
+        match self.contents:
+            case str():
                 children = f"\n{header}  {self.contents}"
-            else:
+            case list() | tuple():
                 children = "".join([n.to_string(header + "  ") for n in self.contents])
-        else:
-            children = ""
-
+            case Quantity():
+                children = f"\n{header}  {self.contents}"
+            case ndarray():
+                children = f"\n{header}  {self.contents}"
+            case None:
+                children = ""
+            case _:
+                children = f"\n{header}  {self.contents}"
         return f"\n{header}{self.name}:{attributes}{children}"
 
     def filter(self, name: str) -> list[ndarray | Quantity | str]:
@@ -538,7 +543,7 @@ class Metadata:
     process: list[Process]
     sample: Sample | None
     instrument: Instrument | None
-    raw: MetaNode
+    raw: MetaNode | None
 
     def summary(self):
         run_string = str(self.run[0] if len(self.run) == 1 else self.run)
@@ -567,6 +572,14 @@ def from_json(obj):
             raw=MetaNode.from_json(obj["raw"]),
         )
 
+    @property
+    def id_header(self):
+        """Generate a header for used in the unique_id for datasets"""
+        title = ""
+        if self.title is not None:
+            title = self.title
+        return f"{title}:{",".join(self.run)}"
+
     def as_h5(self, f: h5py.Group):
         """Export data onto an HDF5 group"""
         for idx, run in enumerate(self.run):

sasdata/postprocess.py

@@ -0,0 +1,57 @@
+"""
+
+Post processing for loaded files
+
+"""
+
+import numpy as np
+
+from sasdata.data import SasData
+
+
+def fix_mantid_units_error(data: SasData) -> SasData:
+    pass
+
+
+
+def apply_fixes(data: SasData, mantid_unit_error=True):
+    if mantid_unit_error:
+        data = fix_mantid_units_error(data)
+
+    return data
+
+
+def deduce_qz(data: SasData):
+    """Calculates and appends Qz to SasData if Qx, Qy, and wavelength are all present"""
+    # if Qz is not already in the dataset, but Qx and Qy are
+    if 'Qz' not in data._data_contents and 'Qx' in data._data_contents and 'Qy' in data._data_contents:
+        # we start by making the approximation that qz=0
+        data._data_contents['Qz'] = 0*data._data_contents['Qx']
+
+        # now check if metadata has wavelength information
+        wavelength = getattr(
+            getattr(
+                getattr(
+                    getattr(data, "metadata", None),
+                    "instrument",
+                    None
+                ),
+                "source",
+                None
+            ),
+            "wavelength",
+            None
+        )
+
+        if wavelength is not None:
+            # we can deduce the value of qz from qx and qy
+            # if we have the wavelength
+            qx = data._data_contents['Qx']
+            qy = data._data_contents['Qy']
+
+            # this is how you convert qx, qy, and wavelength to qz
+            k0 = 2*np.pi/wavelength
+            qz = k0-(k0**2-qx**2-qy**2)**(0.5)
+
+            data._data_contents['Qz'] = qz
+