diff --git a/.prospector.yml b/.prospector.yml new file mode 100644 index 00000000..c269d432 --- /dev/null +++ b/.prospector.yml @@ -0,0 +1,22 @@ +# Prospector profile for Codacy. CI uses ruff + ty; this file aligns Codacy with those +# conventions. Numpydoc-style docstrings (Parameters/Returns/References sections) conflict +# with strict pydocstyle (D203, D412, D413, D417, …), so pydocstyle is disabled here. + +pydocstyle: + run: false + +pylint: + disable: + - too-many-arguments + - too-many-positional-arguments + - too-many-locals + - too-many-branches + - import-outside-toplevel + - consider-using-from-import + - line-too-long + options: + max-line-length: 120 + +mccabe: + options: + max-complexity: 20 diff --git a/direct/common/subsample.py b/direct/common/subsample.py index 5df40a81..926fe04c 100644 --- a/direct/common/subsample.py +++ b/direct/common/subsample.py @@ -14,7 +14,8 @@ """DIRECT samplers module. This module contains classes for creating sub-sampling masks. The masks are used to sample k-space data in MRI -reconstruction. The masks are created by selecting a subset of samples from the input k-space data.""" +reconstruction. The masks are created by selecting a subset of samples from the input k-space data. +""" # Code and comments can be shared with code of FastMRI under the same MIT license: # https://github.com/facebookresearch/fastMRI/ @@ -37,8 +38,9 @@ from direct.common._gaussian import gaussian_mask_1d, gaussian_mask_2d # pylint: disable=no-name-in-module from direct.common._poisson import poisson as _poisson # pylint: disable=no-name-in-module from direct.environment import DIRECT_CACHE_DIR -from direct.types import DirectEnum, MaskFuncMode, Number, TensorOrNdarray +from direct.types import DirectEnum, MaskFuncMode, Number, RangeMode, TensorOrNdarray from direct.utils import str_to_class +from direct.utils.distributions import triangular_distribution from direct.utils.io import download_url # pylint: disable=arguments-differ @@ -65,6 +67,7 @@ "KtUniformMaskFunc", "MagicMaskFunc", "MaskFuncMode", + "RangeMode", "RadialMaskFunc", "RandomMaskFunc", "SpiralMaskFunc", @@ -89,86 +92,104 @@ def temp_seed(rng, seed): class BaseMaskFunc: - """:class:`BaseMaskFunc` is the base class to create a sub-sampling mask of a given shape. - - Parameters - ---------- - accelerations : Union[list[Number], tuple[Number, ...]] - Amount of under-sampling_mask. An acceleration of 4 retains 25% of the k-space, the method is given by - mask_type. Has to be the same length as center_fractions if uniform_range is not True. - center_fractions : Union[list[float], tuple[float, ...]], optional - Fraction of low-frequency columns (float < 1.0) or number of low-frequence columns (integer) to be retained. - If multiple values are provided, then one of these numbers is chosen uniformly each time. If uniform_range - is True, then two values should be given. Default: None. - uniform_range : bool - If True then an acceleration will be uniformly sampled between the two values. Default: False. - mode : MaskFuncMode - Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE. - If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be - broadcasted to the shape by expanding other dimensions with 1, if applicable. If MaskFuncMode.DYNAMIC, - this expects the shape to have more then 3 dimensions, and the mask will be created for each time frame - along the fourth last dimension. Similarly for MaskFuncMode.MULTISLICE, the mask will be created for each - slice along the fourth last dimension. Default: MaskFuncMode.STATIC. + """Base class to create a sub-sampling mask of a given shape. + + Args: + accelerations: Amount of under-sampling. An acceleration of 4 retains 25% of + k-space. Must match ``center_fractions`` length when ``range_mode`` is + ``RangeMode.DISCRETE``. + center_fractions: Fraction of low-frequency columns (float < 1.0) or number of + low-frequency columns (int) to retain. If ``range_mode`` is UNIFORM or LINEAR, + two values should be given. Defaults to None. + range_mode: How accelerations are sampled. ``DISCRETE`` picks from the configured + values; ``UNIFORM`` samples uniformly between the two endpoints; ``LINEAR`` + samples from a triangular distribution biased toward higher acceleration. + Defaults to ``RangeMode.UNIFORM``. + mode: Mask function mode (``STATIC``, ``DYNAMIC``, or ``MULTISLICE``). + Defaults to ``MaskFuncMode.STATIC``. """ def __init__( self, accelerations: Union[list[Number], tuple[Number, ...]], center_fractions: Optional[Union[list[float], tuple[float, ...]]] = None, - uniform_range: bool = True, + range_mode: RangeMode = RangeMode.UNIFORM, mode: MaskFuncMode = MaskFuncMode.STATIC, ) -> None: - """Inits :class:`BaseMaskFunc`. - - Parameters - ---------- - accelerations : Union[list[Number], tuple[Number, ...]] - Amount of under-sampling_mask. An acceleration of 4 retains 25% of the k-space, the method is given by - mask_type. Has to be the same length as center_fractions if uniform_range is not True. - center_fractions : Union[list[float], tuple[float, ...]], optional - Fraction of low-frequency columns (float < 1.0) or number of low-frequence columns (integer) to be retained. - If multiple values are provided, then one of these numbers is chosen uniformly each time. If uniform_range - is True, then two values should be given. Default: None. - uniform_range : bool - If True then an acceleration will be uniformly sampled between the two values. Default: False. - mode : MaskFuncMode - Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE. - If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be - broadcasted to the shape by expanding other dimensions with 1, if applicable. If MaskFuncMode.DYNAMIC, - this expects the shape to have more then 3 dimensions, and the mask will be created for each time frame - along the fourth last dimension. Similarly for MaskFuncMode.MULTISLICE, the mask will be created for each - slice along the fourth last dimension. Default: MaskFuncMode.STATIC. + """Initialize :class:`BaseMaskFunc`. + + Args: + accelerations: Amount of under-sampling. An acceleration of 4 retains 25% of + k-space. Must match ``center_fractions`` length when ``range_mode`` is + ``RangeMode.DISCRETE``. + center_fractions: Fraction of low-frequency columns (float < 1.0) or number of + low-frequency columns (int) to retain. If ``range_mode`` is UNIFORM or LINEAR, + two values should be given. Defaults to None. + range_mode: How accelerations are sampled. ``DISCRETE`` picks from the configured + values; ``UNIFORM`` samples uniformly between the two endpoints; ``LINEAR`` + samples from a triangular distribution biased toward higher acceleration. + Defaults to ``RangeMode.UNIFORM``. + mode: Mask function mode (``STATIC``, ``DYNAMIC``, or ``MULTISLICE``). + Defaults to ``MaskFuncMode.STATIC``. + + Raises: + ValueError: If ``range_mode`` is UNIFORM or LINEAR and ``center_fractions`` / + ``accelerations`` are not length-2, or if their lengths differ for DISCRETE. """ if center_fractions is not None: - if len([center_fractions]) != len([accelerations]): + if range_mode in (RangeMode.UNIFORM, RangeMode.LINEAR): + if len(center_fractions) != 2 or len(accelerations) != 2: + raise ValueError( + "When range_mode is UNIFORM or LINEAR, both center_fractions and " + f"accelerations must have length two. Got center_fractions={center_fractions} " + f"and accelerations={accelerations}." + ) + elif len(center_fractions) != len(accelerations): raise ValueError( f"Number of center fractions should match number of accelerations. " - f"Got {len([center_fractions])} {len([accelerations])}." + f"Got {len(center_fractions)} and {len(accelerations)}." ) self.center_fractions = center_fractions self.accelerations = accelerations - - self.uniform_range = uniform_range + self.range_mode = range_mode self.mode = mode - self.rng = np.random.RandomState() + self._last_acceleration: Optional[float] = None + self._last_center_fraction: Optional[float] = None - def _choose_index(self) -> int: - """Picks a random index into ``self.accelerations``. - - Raises - ------ - ValueError - If no accelerations have been configured. - NotImplementedError - If uniform range is requested but not yet implemented. - """ + def _draw_acceleration_value(self) -> Number: if not self.accelerations: raise ValueError("No accelerations configured for this mask function.") - if self.uniform_range: - raise NotImplementedError("Uniform range is not yet implemented.") - return int(self.rng.randint(0, len(self.accelerations))) + if self.range_mode == RangeMode.UNIFORM: + accel = [float(value) for value in self.accelerations] + return self.rng.uniform(min(accel), max(accel)) + if self.range_mode == RangeMode.LINEAR: + accel = [float(value) for value in self.accelerations] + return triangular_distribution(min(accel), max(accel), 1, self.rng)[0] + return self.accelerations[int(self.rng.randint(0, len(self.accelerations)))] + + def _draw_acceleration_pair(self) -> tuple[Number, Number]: + if self.center_fractions is None: + raise ValueError( + "_draw_acceleration_pair requires center_fractions; use choose_acceleration_only " + "for mask functions that do not configure center fractions." + ) + if self.range_mode == RangeMode.UNIFORM: + accel = [float(value) for value in self.accelerations] + acceleration = self.rng.uniform(min(accel), max(accel)) + center_values = [float(value) for value in self.center_fractions] + center_fraction = self.rng.uniform(min(center_values), max(center_values)) + elif self.range_mode == RangeMode.LINEAR: + accel = [float(value) for value in self.accelerations] + acceleration = triangular_distribution(min(accel), max(accel), 1, self.rng)[0] + center_values = [float(value) for value in self.center_fractions] + center_fraction = self.rng.uniform(min(center_values), max(center_values)) + else: + choice = int(self.rng.randint(0, len(self.accelerations))) + acceleration = self.accelerations[choice] + center_fraction = self.center_fractions[choice] # type: ignore[index] + return acceleration, center_fraction def choose_acceleration(self) -> tuple[Number, Number]: """Chooses an acceleration and matching center fraction. @@ -188,12 +209,21 @@ def choose_acceleration(self) -> tuple[Number, Number]: "choose_acceleration requires center_fractions; use choose_acceleration_only " "for mask functions that do not configure center fractions." ) - choice = self._choose_index() - return self.center_fractions[choice], self.accelerations[choice] + + acceleration, center_fraction = self._draw_acceleration_pair() + center_fraction = float(center_fraction) + if center_fraction < 1.0: + center_fraction = min(1.0 / float(acceleration), center_fraction) + self._last_acceleration = float(acceleration) + self._last_center_fraction = float(center_fraction) + return center_fraction, acceleration def choose_acceleration_only(self) -> Number: """Chooses an acceleration without an associated center fraction.""" - return self.accelerations[self._choose_index()] + acceleration = self._draw_acceleration_value() + self._last_acceleration = float(acceleration) + self._last_center_fraction = min(1.0 / float(acceleration), 1.0) + return acceleration @abstractmethod def mask_func(self, *args, **kwargs) -> torch.Tensor: @@ -220,9 +250,9 @@ def _reshape_and_add_coil_axis(self, mask: TensorOrNdarray, shape: Sequence[int] (nt or num_slices, num_rows, num_cols) if mode is MaskFuncMode.DYNAMIC or MaskFuncMode.MULTISLICE to be reshaped. shape : tuple of ints - Shape of the output array after reshaping. Expects shape to be (\*, num_rows, num_cols, channels) for - mode MaskFuncMode.STATIC, and (\*, nt or num_slices, num_rows, num_cols, channels) for mode - MaskFuncMode.DYNAMIC where \* is any number of dimensions. + Shape of the output array after reshaping. Expects shape to be (\\*, num_rows, num_cols, channels) for + mode MaskFuncMode.STATIC, and (\\*, nt or num_slices, num_rows, num_cols, channels) for mode + MaskFuncMode.DYNAMIC where \\* is any number of dimensions. Returns ------- @@ -250,7 +280,9 @@ def _reshape_and_add_coil_axis(self, mask: TensorOrNdarray, shape: Sequence[int] return mask - def __call__(self, shape: tuple[int, ...], *args, **kwargs) -> torch.Tensor: + def __call__( + self, shape: tuple[int, ...], *args, **kwargs + ) -> Union[torch.Tensor, tuple[torch.Tensor, float, float]]: """Calls the mask function. Parameters @@ -265,16 +297,36 @@ def __call__(self, shape: tuple[int, ...], *args, **kwargs) -> torch.Tensor: Returns ------- - torch.Tensor - Sampling mask. + torch.Tensor or tuple + Sampling mask, or ``(mask, acceleration, center_fraction)`` when + ``return_acceleration`` is True. """ if len(shape) < 3: raise ValueError("Shape should have 3 or more dimensions.") if self.mode in [MaskFuncMode.DYNAMIC, MaskFuncMode.MULTISLICE] and len(shape) < 4: raise ValueError("Shape should have 4 or more dimensions for dynamic or multislice mode.") + return_acceleration = kwargs.pop("return_acceleration", False) + self._last_acceleration = None + self._last_center_fraction = None + mask = self.mask_func(shape, *args, **kwargs) - return mask + + if not return_acceleration: + return mask + + if ( + isinstance(mask, tuple) + and len(mask) == 3 + and isinstance(mask[1], (float, int)) + and isinstance(mask[2], (float, int)) + ): + return mask + + if self._last_acceleration is None or self._last_center_fraction is None: + return mask + + return mask, self._last_acceleration, self._last_center_fraction class CartesianVerticalMaskFunc(BaseMaskFunc): @@ -290,8 +342,8 @@ class CartesianVerticalMaskFunc(BaseMaskFunc): Amount of under-sampling. center_fractions : Union[list[float], tuple[float, ...]] Fraction of low-frequency columns (float < 1.0) or number of low-frequence columns (integer) to be retained. - uniform_range : bool, optional - If True then an acceleration will be uniformly sampled between the two values, by default False. + range_mode : RangeMode, optional + How accelerations are sampled. Defaults to RangeMode.DISCRETE. mode : MaskFuncMode Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE. If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be @@ -305,7 +357,7 @@ def __init__( self, accelerations: Union[list[Number], tuple[Number, ...]], center_fractions: Union[list[float], tuple[float, ...]], - uniform_range: bool = False, + range_mode: RangeMode = RangeMode.DISCRETE, mode: MaskFuncMode = MaskFuncMode.STATIC, ) -> None: """Inits :class:`CartesianVerticalMaskFunc`. @@ -316,8 +368,8 @@ def __init__( Amount of under-sampling. center_fractions : Union[list[float], tuple[float, ...]] Fraction of low-frequency columns (float < 1.0) or number of low-frequence columns (integer) to be retained. - uniform_range : bool, optional - If True then an acceleration will be uniformly sampled between the two values, by default False. + range_mode : RangeMode, optional + How accelerations are sampled. Defaults to RangeMode.DISCRETE. mode : MaskFuncMode Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE. If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be @@ -329,7 +381,7 @@ def __init__( super().__init__( accelerations=accelerations, center_fractions=center_fractions, - uniform_range=uniform_range, + range_mode=range_mode, mode=mode, ) @@ -420,8 +472,8 @@ class RandomMaskFunc(CartesianVerticalMaskFunc): center_fractions : Union[list[Number], tuple[Number, ...]] If < 1.0 this corresponds to the fraction of low-frequency columns to be retained. If >= 1 (integer) this corresponds to the exact number of low-frequency columns to be retained. - uniform_range : bool, optional - If True then an acceleration will be uniformly sampled between the two values, by default False. + range_mode : RangeMode, optional + How accelerations are sampled. Defaults to RangeMode.DISCRETE. mode : MaskFuncMode Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE. If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be @@ -435,7 +487,7 @@ def __init__( self, accelerations: Union[list[Number], tuple[Number, ...]], center_fractions: Union[list[Number], tuple[Number, ...]], - uniform_range: bool = False, + range_mode: RangeMode = RangeMode.DISCRETE, mode: MaskFuncMode = MaskFuncMode.STATIC, ) -> None: """Inits :class:`RandomMaskFunc`. @@ -447,8 +499,8 @@ def __init__( center_fractions : Union[list[Number], tuple[Number, ...]] If < 1.0 this corresponds to the fraction of low-frequency columns to be retained. If >= 1 (integer) this corresponds to the exact number of low-frequency columns to be retained. - uniform_range : bool, optional - If True then an acceleration will be uniformly sampled between the two values, by default False. + range_mode : RangeMode, optional + How accelerations are sampled. Defaults to RangeMode.DISCRETE. mode : MaskFuncMode Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE. If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be @@ -460,7 +512,7 @@ def __init__( super().__init__( accelerations=accelerations, center_fractions=center_fractions, - uniform_range=uniform_range, + range_mode=range_mode, mode=mode, ) @@ -483,7 +535,6 @@ def mask_func( Seed for the random number generator. Setting the seed ensures the same mask is generated each time for the same shape. Default: None. - Returns ------- mask : torch.Tensor @@ -544,8 +595,8 @@ class FastMRIRandomMaskFunc(RandomMaskFunc): Amount of under-sampling. center_fractions : Union[list[float], tuple[float, ...]] Fraction of low-frequency columns (float < 1.0) to be retained. - uniform_range : bool, optional - If True then an acceleration will be uniformly sampled between the two values, by default False. + range_mode : RangeMode, optional + How accelerations are sampled. Defaults to RangeMode.DISCRETE. mode : MaskFuncMode Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE. If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be @@ -559,7 +610,7 @@ def __init__( self, accelerations: Union[list[Number], tuple[Number, ...]], center_fractions: Union[list[float], tuple[float, ...]], - uniform_range: bool = False, + range_mode: RangeMode = RangeMode.DISCRETE, mode: MaskFuncMode = MaskFuncMode.STATIC, ) -> None: """Inits :class:`FastMRIRandomMaskFunc`. @@ -570,8 +621,8 @@ def __init__( Amount of under-sampling. center_fractions : Union[list[float], tuple[float, ...]] Fraction of low-frequency columns (float < 1.0) to be retained. - uniform_range : bool, optional - If True then an acceleration will be uniformly sampled between the two values, by default False. + range_mode : RangeMode, optional + How accelerations are sampled. Defaults to RangeMode.DISCRETE. mode : MaskFuncMode Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE. If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be @@ -588,7 +639,7 @@ def __init__( super().__init__( accelerations=accelerations, center_fractions=center_fractions, - uniform_range=uniform_range, + range_mode=range_mode, mode=mode, ) @@ -603,11 +654,11 @@ class CartesianRandomMaskFunc(RandomMaskFunc): ---------- accelerations : Union[list[Number], tuple[Number, ...]] Amount of under-sampling_mask. An acceleration of 4 retains 25% of the k-space, the method is given by - mask_type. Has to be the same length as center_fractions if uniform_range is not True. + mask_type. Has to be the same length as center_fractions if range_mode is RangeMode.DISCRETE. center_fractions : Union[list[int], tuple[int, ...]] Number of low-frequence (center) columns to be retained. - uniform_range : bool - If True then an acceleration will be uniformly sampled between the two values. Default: False. + range_mode : RangeMode + How accelerations are sampled. Defaults to RangeMode.DISCRETE. mode : MaskFuncMode Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE. If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be @@ -621,7 +672,7 @@ def __init__( self, accelerations: Union[list[Number], tuple[Number, ...]], center_fractions: Union[list[int], tuple[int, ...]], - uniform_range: bool = False, + range_mode: RangeMode = RangeMode.DISCRETE, mode: MaskFuncMode = MaskFuncMode.STATIC, ) -> None: """Inits :class:`CartesianRandomMaskFunc`. @@ -630,11 +681,11 @@ def __init__( ---------- accelerations : Union[list[Number], tuple[Number, ...]] Amount of under-sampling_mask. An acceleration of 4 retains 25% of the k-space, the method is given by - mask_type. Has to be the same length as center_fractions if uniform_range is not True. + mask_type. Has to be the same length as center_fractions if range_mode is RangeMode.DISCRETE. center_fractions : Union[list[int], tuple[int, ...]] Number of low-frequence (center) columns to be retained. - uniform_range : bool - If True then an acceleration will be uniformly sampled between the two values. Default: False. + range_mode : RangeMode + How accelerations are sampled. Defaults to RangeMode.DISCRETE. mode : MaskFuncMode Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE. If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be @@ -651,7 +702,7 @@ def __init__( super().__init__( accelerations=accelerations, center_fractions=list(center_fractions), - uniform_range=uniform_range, + range_mode=range_mode, mode=mode, ) @@ -683,8 +734,8 @@ class EquispacedMaskFunc(CartesianVerticalMaskFunc): center_fractions : Union[list[Number], tuple[Number, ...]] If < 1.0 this corresponds to the fraction of low-frequency columns to be retained. If >= 1 (integer) this corresponds to the exact number of low-frequency columns to be retained. - uniform_range : bool, optional - If True then an acceleration will be uniformly sampled between the two values, by default False. + range_mode : RangeMode, optional + How accelerations are sampled. Defaults to RangeMode.DISCRETE. mode : MaskFuncMode Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE. If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be @@ -698,7 +749,7 @@ def __init__( self, accelerations: Union[list[Number], tuple[Number, ...]], center_fractions: Union[list[Number], tuple[Number, ...]], - uniform_range: bool = False, + range_mode: RangeMode = RangeMode.DISCRETE, mode: MaskFuncMode = MaskFuncMode.STATIC, ) -> None: """Inits :class:`EquispacedMaskFunc`. @@ -710,8 +761,8 @@ def __init__( center_fractions : Union[list[Number], tuple[Number, ...]] If < 1.0 this corresponds to the fraction of low-frequency columns to be retained. If >= 1 (integer) this corresponds to the exact number of low-frequency columns to be retained. - uniform_range : bool, optional - If True then an acceleration will be uniformly sampled between the two values, by default False. + range_mode : RangeMode, optional + How accelerations are sampled. Defaults to RangeMode.DISCRETE. mode : MaskFuncMode Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE. If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be @@ -723,7 +774,7 @@ def __init__( super().__init__( accelerations=accelerations, center_fractions=center_fractions, - uniform_range=uniform_range, + range_mode=range_mode, mode=mode, ) @@ -771,12 +822,17 @@ def mask_func( if return_acs: return self._reshape_and_add_coil_axis(self._broadcast_mask(mask, num_rows), shape) + # When the ACS already covers the target acceleration, skip equispaced lines. + if num_low_freqs - num_cols // acceleration >= 0: + return self._reshape_and_add_coil_axis(self._broadcast_mask(mask, num_rows), shape) + # determine acceleration rate by adjusting for the number of low frequencies adjusted_accel = (acceleration * (num_low_freqs - num_cols)) / (num_low_freqs * acceleration - num_cols) + offset_high = max(1, round(adjusted_accel)) mask = mask.reshape(num_slc_or_time, -1) # In case mode != MaskFuncMode.STATIC: for i in range(num_slc_or_time): - offset = self.rng.randint(0, round(adjusted_accel)) + offset = self.rng.randint(0, offset_high) accel_samples = np.arange(offset, num_cols - 1, adjusted_accel) accel_samples = np.around(accel_samples).astype(np.uint) mask[i, accel_samples] = True @@ -809,8 +865,8 @@ class FastMRIEquispacedMaskFunc(EquispacedMaskFunc): Amount of under-sampling. center_fractions : Union[list[float], tuple[float, ...]] Fraction (< 1.0) of low-frequency columns to be retained. - uniform_range : bool, optional - If True then an acceleration will be uniformly sampled between the two values, by default False. + range_mode : RangeMode, optional + How accelerations are sampled. Defaults to RangeMode.DISCRETE. mode : MaskFuncMode Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE. If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be @@ -824,7 +880,7 @@ def __init__( self, accelerations: Union[list[Number], tuple[Number, ...]], center_fractions: Union[list[float], tuple[float, ...]], - uniform_range: bool = False, + range_mode: RangeMode = RangeMode.DISCRETE, mode: MaskFuncMode = MaskFuncMode.STATIC, ) -> None: """Inits :class:`FastMRIEquispacedMaskFunc`. @@ -835,8 +891,8 @@ def __init__( Amount of under-sampling. center_fractions : Union[list[float], tuple[float, ...]] Fraction (< 1.0) of low-frequency columns to be retained. - uniform_range : bool, optional - If True then an acceleration will be uniformly sampled between the two values, by default False. + range_mode : RangeMode, optional + How accelerations are sampled. Defaults to RangeMode.DISCRETE. mode : MaskFuncMode Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE. If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be @@ -853,7 +909,7 @@ def __init__( super().__init__( accelerations=accelerations, center_fractions=center_fractions, - uniform_range=uniform_range, + range_mode=range_mode, mode=mode, ) @@ -868,11 +924,11 @@ class CartesianEquispacedMaskFunc(EquispacedMaskFunc): ---------- accelerations : Union[list[Number], tuple[Number, ...]] Amount of under-sampling_mask. An acceleration of 4 retains 25% of the k-space, the method is given by - mask_type. Has to be the same length as center_fractions if uniform_range is not True. + mask_type. Has to be the same length as center_fractions if range_mode is RangeMode.DISCRETE. center_fractions : Union[list[int], tuple[int, ...]] Number of low-frequence (center) columns to be retained. - uniform_range : bool - If True then an acceleration will be uniformly sampled between the two values. Default: False. + range_mode : RangeMode + How accelerations are sampled. Defaults to RangeMode.DISCRETE. mode : MaskFuncMode Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE. If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be @@ -886,7 +942,7 @@ def __init__( self, accelerations: Union[list[Number], tuple[Number, ...]], center_fractions: Union[list[int], tuple[int, ...]], - uniform_range: bool = False, + range_mode: RangeMode = RangeMode.DISCRETE, mode: MaskFuncMode = MaskFuncMode.STATIC, ) -> None: """Inits :class:`CartesianEquispacedMaskFunc`. @@ -895,11 +951,11 @@ def __init__( ---------- accelerations : Union[list[Number], tuple[Number, ...]] Amount of under-sampling_mask. An acceleration of 4 retains 25% of the k-space, the method is given by - mask_type. Has to be the same length as center_fractions if uniform_range is not True. + mask_type. Has to be the same length as center_fractions if range_mode is RangeMode.DISCRETE. center_fractions : Union[list[int], tuple[int, ...]] Number of low-frequence (center) columns to be retained. - uniform_range : bool - If True then an acceleration will be uniformly sampled between the two values. Default: False. + range_mode : RangeMode + How accelerations are sampled. Defaults to RangeMode.DISCRETE. mode : MaskFuncMode Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE. If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be @@ -916,7 +972,7 @@ def __init__( super().__init__( accelerations=accelerations, center_fractions=list(center_fractions), - uniform_range=uniform_range, + range_mode=range_mode, mode=mode, ) @@ -937,12 +993,12 @@ class MagicMaskFunc(CartesianVerticalMaskFunc): ---------- accelerations : Union[list[Number], tuple[Number, ...]] Amount of under-sampling_mask. An acceleration of 4 retains 25% of the k-space, the method is given by - mask_type. Has to be the same length as center_fractions if uniform_range is not True. + mask_type. Has to be the same length as center_fractions if range_mode is RangeMode.DISCRETE. center_fractions : Union[list[Number], tuple[Number, ...]] If < 1.0 this corresponds to the fraction of low-frequency columns to be retained. If >= 1 (integer) this corresponds to the exact number of low-frequency columns to be retained. - uniform_range : bool - If True then an acceleration will be uniformly sampled between the two values. Default: False. + range_mode : RangeMode + How accelerations are sampled. Defaults to RangeMode.DISCRETE. mode : MaskFuncMode Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE. If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be @@ -956,7 +1012,7 @@ def __init__( self, accelerations: Union[list[Number], tuple[Number, ...]], center_fractions: Union[list[Number], tuple[Number, ...]], - uniform_range: bool = False, + range_mode: RangeMode = RangeMode.DISCRETE, mode: MaskFuncMode = MaskFuncMode.STATIC, ) -> None: """Inits :class:`MagicMaskFunc`. @@ -965,12 +1021,12 @@ def __init__( ---------- accelerations : Union[list[Number], tuple[Number, ...]] Amount of under-sampling_mask. An acceleration of 4 retains 25% of the k-space, the method is given by - mask_type. Has to be the same length as center_fractions if uniform_range is not True. + mask_type. Has to be the same length as center_fractions if range_mode is RangeMode.DISCRETE. center_fractions : Union[list[Number], tuple[Number, ...]] If < 1.0 this corresponds to the fraction of low-frequency columns to be retained. If >= 1.0 this corresponds to the exact number of low-frequency columns to be retained. - uniform_range : bool - If True then an acceleration will be uniformly sampled between the two values. Default: False. + range_mode : RangeMode + How accelerations are sampled. Defaults to RangeMode.DISCRETE. mode : MaskFuncMode Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE. If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be @@ -982,7 +1038,7 @@ def __init__( super().__init__( accelerations=accelerations, center_fractions=center_fractions, - uniform_range=uniform_range, + range_mode=range_mode, mode=mode, ) @@ -1044,6 +1100,11 @@ def mask_func( adjusted_acceleration = int(round(num_cols / adjusted_target_cols_to_sample)) acs_mask = acs_mask.reshape(num_slc_or_time, -1) # In case mode != MaskFuncMode.STATIC: + + if adjusted_target_cols_to_sample <= 0: + mask = acs_mask.squeeze() + return self._reshape_and_add_coil_axis(self._broadcast_mask(mask, num_rows), shape) + mask = [] for i in range(num_slc_or_time): offset = self.rng.randint(0, high=adjusted_acceleration) @@ -1087,11 +1148,11 @@ class FastMRIMagicMaskFunc(MagicMaskFunc): ---------- accelerations : Union[list[Number], tuple[Number, ...]] Amount of under-sampling_mask. An acceleration of 4 retains 25% of the k-space, the method is given by - mask_type. Has to be the same length as center_fractions if uniform_range is not True. + mask_type. Has to be the same length as center_fractions if range_mode is RangeMode.DISCRETE. center_fractions : Union[list[float], tuple[float, ...]] Fraction (< 1.0) of low-frequency columns to be retained. - uniform_range : bool - If True then an acceleration will be uniformly sampled between the two values. Default: False. + range_mode : RangeMode + How accelerations are sampled. Defaults to RangeMode.DISCRETE. mode : MaskFuncMode Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE. If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be @@ -1105,7 +1166,7 @@ def __init__( self, accelerations: Union[list[Number], tuple[Number, ...]], center_fractions: Union[list[float], tuple[float, ...]], - uniform_range: bool = False, + range_mode: RangeMode = RangeMode.DISCRETE, mode: MaskFuncMode = MaskFuncMode.STATIC, ) -> None: """Inits :class:`FastMRIMagicMaskFunc`. @@ -1114,11 +1175,11 @@ def __init__( ---------- accelerations : Union[list[Number], tuple[Number, ...]] Amount of under-sampling_mask. An acceleration of 4 retains 25% of the k-space, the method is given by - mask_type. Has to be the same length as center_fractions if uniform_range is not True. + mask_type. Has to be the same length as center_fractions if range_mode is RangeMode.DISCRETE. center_fractions : Union[list[float], tuple[float, ...]] Fraction (< 1.0) of low-frequency columns to be retained. - uniform_range : bool - If True then an acceleration will be uniformly sampled between the two values. Default: False. + range_mode : RangeMode + How accelerations are sampled. Defaults to RangeMode.DISCRETE. mode : MaskFuncMode Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE. If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be @@ -1136,7 +1197,7 @@ def __init__( super().__init__( accelerations=accelerations, center_fractions=center_fractions, - uniform_range=uniform_range, + range_mode=range_mode, mode=mode, ) @@ -1156,11 +1217,11 @@ class CartesianMagicMaskFunc(MagicMaskFunc): ---------- accelerations : Union[list[Number], tuple[Number, ...]] Amount of under-sampling_mask. An acceleration of 4 retains 25% of the k-space, the method is given by - mask_type. Has to be the same length as center_fractions if uniform_range is not True. + mask_type. Has to be the same length as center_fractions if range_mode is RangeMode.DISCRETE. center_fractions : Union[list[int], tuple[int, ...]] Number of low-frequence (center) columns to be retained. - uniform_range : bool - If True then an acceleration will be uniformly sampled between the two values. Default: False. + range_mode : RangeMode + How accelerations are sampled. Defaults to RangeMode.DISCRETE. mode : MaskFuncMode Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE. If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be @@ -1174,7 +1235,7 @@ def __init__( self, accelerations: Union[list[Number], tuple[Number, ...]], center_fractions: Union[list[int], tuple[int, ...]], - uniform_range: bool = False, + range_mode: RangeMode = RangeMode.DISCRETE, mode: MaskFuncMode = MaskFuncMode.STATIC, ) -> None: """Inits :class:`CartesianMagicMaskFunc`. @@ -1183,11 +1244,11 @@ def __init__( ---------- accelerations : Union[list[Number], tuple[Number, ...]] Amount of under-sampling_mask. An acceleration of 4 retains 25% of the k-space, the method is given by - mask_type. Has to be the same length as center_fractions if uniform_range is not True. + mask_type. Has to be the same length as center_fractions if range_mode is RangeMode.DISCRETE. center_fractions : Union[list[int], tuple[int, ...]] Number of low-frequence (center) columns to be retained. - uniform_range : bool - If True then an acceleration will be uniformly sampled between the two values. Default: False. + range_mode : RangeMode + How accelerations are sampled. Defaults to RangeMode.DISCRETE. mode : MaskFuncMode Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE. If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be @@ -1204,7 +1265,7 @@ def __init__( super().__init__( accelerations=accelerations, center_fractions=list(center_fractions), - uniform_range=uniform_range, + range_mode=range_mode, mode=mode, ) @@ -1225,7 +1286,7 @@ class CalgaryCampinasMaskFunc(BaseMaskFunc): ---------- accelerations : Union[list[int], tuple[int, ...]] Amount of under-sampling_mask. An acceleration of 4 retains 25% of the k-space, the method is given by - mask_type. Has to be the same length as center_fractions if uniform_range is not True. + mask_type. Has to be the same length as center_fractions if range_mode is RangeMode.DISCRETE. Raises ------ @@ -1263,7 +1324,7 @@ def __init__(self, accelerations: Union[list[int], tuple[int, ...]], **kwargs) - If the acceleration is not 5 or 10. """ - super().__init__(accelerations=list(accelerations), uniform_range=False) + super().__init__(accelerations=list(accelerations), range_mode=RangeMode.DISCRETE) if not all(_ in [5, 10] for _ in accelerations): raise ValueError("CalgaryCampinas only provide 5x and 10x acceleration masks.") @@ -1403,8 +1464,8 @@ class CIRCUSMaskFunc(BaseMaskFunc): center_fractions : Union[list[float], tuple[float, ...]], optional Fraction (< 1.0) of low-frequency samples to be retained. If None, it will calculate the acs mask based on the maximum masked disk in the generated mask (with a tolerance).Default: None. - uniform_range : bool - If True then an acceleration will be uniformly sampled between the two values. Default: False. + range_mode : RangeMode + How accelerations are sampled. Defaults to RangeMode.DISCRETE. mode : MaskFuncMode Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE. If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be @@ -1426,7 +1487,7 @@ def __init__( subsampling_scheme: CIRCUSSamplingMode, accelerations: Union[list[Number], tuple[Number, ...]], center_fractions: Optional[Union[list[float], tuple[float, ...]]] = None, - uniform_range: bool = False, + range_mode: RangeMode = RangeMode.DISCRETE, mode: MaskFuncMode = MaskFuncMode.STATIC, ) -> None: """Inits :class:`CIRCUSMaskFunc`. @@ -1441,8 +1502,8 @@ def __init__( center_fractions : Union[list[float], tuple[float, ...]], optional Fraction (< 1.0) of low-frequency samples to be retained. If None, it will calculate the acs mask based on the maximum masked disk in the generated mask (with a tolerance).Default: None. - uniform_range : bool - If True then an acceleration will be uniformly sampled between the two values. Default: False. + range_mode : RangeMode + How accelerations are sampled. Defaults to RangeMode.DISCRETE. mode : MaskFuncMode Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE. If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be @@ -1458,11 +1519,14 @@ def __init__( """ super().__init__( accelerations=accelerations, - center_fractions=center_fractions if center_fractions else tuple(0 for _ in range(len(accelerations))), - uniform_range=uniform_range, + center_fractions=(center_fractions if center_fractions else tuple(0 for _ in range(len(accelerations)))), + range_mode=range_mode, mode=mode, ) - if subsampling_scheme not in [CIRCUSSamplingMode.CIRCUS_RADIAL, CIRCUSSamplingMode.CIRCUS_SPIRAL]: + if subsampling_scheme not in [ + CIRCUSSamplingMode.CIRCUS_RADIAL, + CIRCUSSamplingMode.CIRCUS_SPIRAL, + ]: raise NotImplementedError( f"Currently CIRCUSMaskFunc is only implemented for 'circus-radial' or 'circus-spiral' " f"as a subsampling_scheme. Got subsampling_scheme={subsampling_scheme}." @@ -1739,8 +1803,8 @@ class RadialMaskFunc(CIRCUSMaskFunc): center_fractions : Union[list[float], tuple[float, ...]], optional Fraction (< 1.0) of low-frequency samples to be retained. If None, it will calculate the acs mask based on the maximum masked disk in the generated mask (with a tolerance).Default: None. - uniform_range : bool - If True then an acceleration will be uniformly sampled between the two values. Default: False. + range_mode : RangeMode + How accelerations are sampled. Defaults to RangeMode.DISCRETE. mode : MaskFuncMode Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE. If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be @@ -1754,7 +1818,7 @@ def __init__( self, accelerations: Union[list[Number], tuple[Number, ...]], center_fractions: Optional[Union[list[float], tuple[float, ...]]] = None, - uniform_range: bool = False, + range_mode: RangeMode = RangeMode.DISCRETE, mode: MaskFuncMode = MaskFuncMode.STATIC, ) -> None: """Inits :class:`RadialMaskFunc`. @@ -1766,8 +1830,8 @@ def __init__( center_fractions : Union[list[float], tuple[float, ...]], optional Fraction (< 1.0) of low-frequency samples to be retained. If None, it will calculate the acs mask based on the maximum masked disk in the generated mask (with a tolerance).Default: None. - uniform_range : bool - If True then an acceleration will be uniformly sampled between the two values. Default: False. + range_mode : RangeMode + How accelerations are sampled. Defaults to RangeMode.DISCRETE. mode : MaskFuncMode Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE. If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be @@ -1780,7 +1844,7 @@ def __init__( accelerations=accelerations, center_fractions=center_fractions, subsampling_scheme=CIRCUSSamplingMode.CIRCUS_RADIAL, - uniform_range=uniform_range, + range_mode=range_mode, mode=mode, ) @@ -1795,8 +1859,8 @@ class SpiralMaskFunc(CIRCUSMaskFunc): center_fractions : Union[list[float], tuple[float, ...]], optional Fraction (< 1.0) of low-frequency samples to be retained. If None, it will calculate the acs mask based on the maximum masked disk in the generated mask (with a tolerance).Default: None. - uniform_range : bool - If True then an acceleration will be uniformly sampled between the two values. Default: False. + range_mode : RangeMode + How accelerations are sampled. Defaults to RangeMode.DISCRETE. mode : MaskFuncMode Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE. If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be @@ -1810,7 +1874,7 @@ def __init__( self, accelerations: Union[list[Number], tuple[Number, ...]], center_fractions: Optional[Union[list[float], tuple[float, ...]]] = None, - uniform_range: bool = False, + range_mode: RangeMode = RangeMode.DISCRETE, mode: MaskFuncMode = MaskFuncMode.STATIC, ) -> None: """Inits :class:`SpiralMaskFunc`. @@ -1822,8 +1886,8 @@ def __init__( center_fractions : Union[list[float], tuple[float, ...]], optional Fraction (< 1.0) of low-frequency samples to be retained. If None, it will calculate the acs mask based on the maximum masked disk in the generated mask (with a tolerance).Default: None. - uniform_range : bool - If True then an acceleration will be uniformly sampled between the two values. Default: False. + range_mode : RangeMode + How accelerations are sampled. Defaults to RangeMode.DISCRETE. mode : MaskFuncMode Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE. If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be @@ -1836,7 +1900,7 @@ def __init__( accelerations=accelerations, center_fractions=center_fractions, subsampling_scheme=CIRCUSSamplingMode.CIRCUS_SPIRAL, - uniform_range=uniform_range, + range_mode=range_mode, mode=mode, ) @@ -1928,7 +1992,7 @@ def __init__( super().__init__( accelerations=accelerations, center_fractions=center_fractions, - uniform_range=False, + range_mode=RangeMode.DISCRETE, mode=mode, ) self.crop_corner = crop_corner @@ -1980,7 +2044,15 @@ def mask_func( mask = [] for _ in range(num_slc_or_time): - mask.append(self.poisson(num_rows, num_cols, center_fraction, acceleration, self.rng.randint(100_000))) + mask.append( + self.poisson( + num_rows, + num_cols, + center_fraction, + acceleration, + self.rng.randint(100_000), + ) + ) mask = np.stack(mask, axis=0).squeeze() return self._reshape_and_add_coil_axis(mask, shape) @@ -2035,7 +2107,15 @@ def poisson( mask = np.zeros((num_rows, num_cols), dtype=int) - _poisson(num_rows, num_cols, self.max_attempts or 10, mask, radius_x, radius_y, seed) + _poisson( + num_rows, + num_cols, + self.max_attempts or 10, + mask, + radius_x, + radius_y, + seed, + ) mask = mask | centered_disk_mask((num_rows, num_cols), center_fraction) @@ -2070,8 +2150,8 @@ class Gaussian1DMaskFunc(CartesianVerticalMaskFunc): Amount of under-sampling. center_fractions : Union[list[float], tuple[float, ...]] Fraction of low-frequency columns (float < 1.0) to be retained. - uniform_range : bool, optional - If True then an acceleration will be uniformly sampled between the two values, by default False. + range_mode : RangeMode, optional + How accelerations are sampled. Defaults to RangeMode.DISCRETE. mode : MaskFuncMode Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE. If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be @@ -2085,7 +2165,7 @@ def __init__( self, accelerations: Union[list[Number], tuple[Number, ...]], center_fractions: Union[list[float], tuple[float, ...]], - uniform_range: bool = False, + range_mode: RangeMode = RangeMode.DISCRETE, mode: MaskFuncMode = MaskFuncMode.STATIC, ) -> None: """Inits :class:`Gaussian1DMaskFunc`. @@ -2096,8 +2176,8 @@ def __init__( Amount of under-sampling. center_fractions : Union[list[float], tuple[float, ...]] Fraction of low-frequency columns (float < 1.0) to be retained. - uniform_range : bool, optional - If True then an acceleration will be uniformly sampled between the two values, by default False. + range_mode : RangeMode, optional + How accelerations are sampled. Defaults to RangeMode.DISCRETE. mode : MaskFuncMode Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE. If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be @@ -2109,7 +2189,7 @@ def __init__( super().__init__( accelerations=accelerations, center_fractions=center_fractions, - uniform_range=uniform_range, + range_mode=range_mode, mode=mode, ) @@ -2195,8 +2275,8 @@ class Gaussian2DMaskFunc(BaseMaskFunc): Amount of under-sampling. center_fractions : Union[list[float], tuple[float, ...]] Fraction of low-frequency samples (float < 1.0) to be retained. - uniform_range : bool, optional - If True then an acceleration will be uniformly sampled between the two values, by default False. + range_mode : RangeMode, optional + How accelerations are sampled. Defaults to RangeMode.DISCRETE. mode : MaskFuncMode Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE. If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be @@ -2210,7 +2290,7 @@ def __init__( self, accelerations: Union[list[Number], tuple[Number, ...]], center_fractions: Union[list[float], tuple[float, ...]], - uniform_range: bool = False, + range_mode: RangeMode = RangeMode.DISCRETE, mode: MaskFuncMode = MaskFuncMode.STATIC, ) -> None: """Inits :class:`Gaussian2DMaskFunc`. @@ -2221,8 +2301,8 @@ def __init__( Amount of under-sampling. center_fractions : Union[list[float], tuple[float, ...]] Fraction of low-frequency samples (float < 1.0) to be retained. - uniform_range : bool, optional - If True then an acceleration will be uniformly sampled between the two values, by default False. + range_mode : RangeMode, optional + How accelerations are sampled. Defaults to RangeMode.DISCRETE. mode : MaskFuncMode Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE. If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be @@ -2234,7 +2314,7 @@ def __init__( super().__init__( accelerations=accelerations, center_fractions=center_fractions, - uniform_range=uniform_range, + range_mode=range_mode, mode=mode, ) @@ -2319,15 +2399,15 @@ class KtBaseMaskFunc(BaseMaskFunc): Amount of under-sampling. center_fractions : Union[list[float], tuple[float, ...]] Fraction of low-frequency columns (float < 1.0) or number of low-frequence columns (integer) to be retained. - uniform_range : bool, optional - If True then an acceleration will be uniformly sampled between the two values, by default False. + range_mode : RangeMode, optional + How accelerations are sampled. Defaults to RangeMode.DISCRETE. """ def __init__( self, accelerations: Union[list[Number], tuple[Number, ...]], center_fractions: Union[list[float], tuple[float, ...]], - uniform_range: bool = False, + range_mode: RangeMode = RangeMode.DISCRETE, ) -> None: """Inits :class:`KtBaseMaskFunc`. @@ -2337,13 +2417,13 @@ def __init__( Amount of under-sampling. center_fractions : Union[list[float], tuple[float, ...]] Fraction of low-frequency columns (float < 1.0) or number of low-frequence columns (integer) to be retained. - uniform_range : bool, optional - If True then an acceleration will be uniformly sampled between the two values, by default False. + range_mode : RangeMode, optional + How accelerations are sampled. Defaults to RangeMode.DISCRETE. """ super().__init__( accelerations=accelerations, center_fractions=center_fractions, - uniform_range=uniform_range, + range_mode=range_mode, mode=MaskFuncMode.DYNAMIC, ) @@ -2378,7 +2458,10 @@ def zero_pad_to_center(array: np.ndarray, target_shape: tuple[int, ...]) -> np.n # Calculate the slices for inserting the original array into the padded array insert_slices = tuple( - slice((target_dim - current_dim) // 2, (target_dim - current_dim) // 2 + current_dim) + slice( + (target_dim - current_dim) // 2, + (target_dim - current_dim) // 2 + current_dim, + ) for target_dim, current_dim in zip(target_shape, current_shape) ) @@ -2518,7 +2601,10 @@ def crop_center(array, target_height, target_width): # Calculate the slices for cropping the array crop_slices = tuple( - slice((current_dim - target_dim) // 2, (current_dim - target_dim) // 2 + target_dim) + slice( + (current_dim - target_dim) // 2, + (current_dim - target_dim) // 2 + target_dim, + ) for current_dim, target_dim in zip(current_shape, target_shape) ) @@ -2536,8 +2622,8 @@ class KtRadialMaskFunc(KtBaseMaskFunc): Amount of under-sampling. center_fractions : Union[list[float], tuple[float, ...]] Fraction of low-frequency columns (float < 1.0) or number of low-frequence columns (integer) to be retained. - uniform_range : bool, optional - If True then an acceleration will be uniformly sampled between the two values, by default False. + range_mode : RangeMode, optional + How accelerations are sampled. Defaults to RangeMode.DISCRETE. crop_corner : bool, optional If True, the mask is cropped to the corners. Default: False. """ @@ -2546,7 +2632,7 @@ def __init__( self, accelerations: Union[list[Number], tuple[Number, ...]], center_fractions: Union[list[float], tuple[float, ...]], - uniform_range: bool = False, + range_mode: RangeMode = RangeMode.DISCRETE, crop_corner: bool = False, ) -> None: """Inits :class:`KtRadialMaskFunc`. @@ -2557,15 +2643,15 @@ def __init__( Amount of under-sampling. center_fractions : Union[list[float], tuple[float, ...]] Fraction of low-frequency columns (float < 1.0) or number of low-frequence columns (integer) to be retained. - uniform_range : bool, optional - If True then an acceleration will be uniformly sampled between the two values, by default False. + range_mode : RangeMode, optional + How accelerations are sampled. Defaults to RangeMode.DISCRETE. crop_corner : bool, optional If True, the mask is cropped to the corners. Default: False. """ super().__init__( accelerations=accelerations, center_fractions=center_fractions, - uniform_range=uniform_range, + range_mode=range_mode, ) self.crop_corner = crop_corner @@ -2596,7 +2682,7 @@ def mask_func( if len(shape) not in [4, 5]: raise ValueError("Shape should have 4 or 5 dimensions.") - (nt, num_rows, num_cols) = shape[-4:-1] + nt, num_rows, num_cols = shape[-4:-1] with temp_seed(self.rng, seed): center_fraction, acceleration = self.choose_acceleration() @@ -2625,7 +2711,8 @@ def mask_func( aux[int(temp_size / 2), :] = 1 base_mask = np.sum( - [rotate(aux, angle, reshape=False, order=0) for angle in np.linspace(0, 180, num_beams)], axis=0 + [rotate(aux, angle, reshape=False, order=0) for angle in np.linspace(0, 180, num_beams)], + axis=0, ) mask = [self.crop_center(base_mask, num_rows, num_cols)] @@ -2649,15 +2736,15 @@ class KtUniformMaskFunc(KtBaseMaskFunc): Amount of under-sampling. center_fractions : Union[list[float], tuple[float, ...]] Fraction of low-frequency columns (float < 1.0) or number of low-frequence columns (integer) to be retained. - uniform_range : bool, optional - If True then an acceleration will be uniformly sampled between the two values, by default False. + range_mode : RangeMode, optional + How accelerations are sampled. Defaults to RangeMode.DISCRETE. """ def __init__( self, accelerations: Union[list[Number], tuple[Number, ...]], center_fractions: Union[list[float], tuple[float, ...]], - uniform_range: bool = False, + range_mode: RangeMode = RangeMode.DISCRETE, ) -> None: """Inits :class:`KtUniformMaskFunc`. @@ -2667,13 +2754,13 @@ def __init__( Amount of under-sampling. center_fractions : Union[list[float], tuple[float, ...]] Fraction of low-frequency columns (float < 1.0) or number of low-frequence columns (integer) to be retained. - uniform_range : bool, optional - If True then an acceleration will be uniformly sampled between the two values, by default False. + range_mode : RangeMode, optional + How accelerations are sampled. Defaults to RangeMode.DISCRETE. """ super().__init__( accelerations=accelerations, center_fractions=center_fractions, - uniform_range=uniform_range, + range_mode=range_mode, ) def mask_func( @@ -2703,7 +2790,7 @@ def mask_func( if len(shape) not in [4, 5]: raise ValueError("Shape should have 4 or 5 dimensions.") - (nt, num_rows, num_cols) = shape[-4:-1] + nt, num_rows, num_cols = shape[-4:-1] with temp_seed(self.rng, seed): center_fraction, acceleration = self.choose_acceleration() @@ -2711,7 +2798,8 @@ def mask_func( # Fully sampled rectangle region acs_mask = self.zero_pad_to_center( - np.ones((nt, num_rows, num_low_freqs)), (int(nt), int(num_rows), int(num_cols)) + np.ones((nt, num_rows, num_low_freqs)), + (int(nt), int(num_rows), int(num_cols)), ) if return_acs: @@ -2724,8 +2812,14 @@ def mask_func( ptmp = np.zeros(num_cols) ttmp = np.zeros(nt) - ptmp[np.arange(self.rng.randint(0, adjusted_acceleration), num_cols, adjusted_acceleration).astype(int)] = 1 - ttmp[np.arange(self.rng.randint(0, acceleration), nt, acceleration).astype(int)] = 1 + ptmp[ + np.arange( + self.rng.randint(0, max(1, int(adjusted_acceleration))), + num_cols, + adjusted_acceleration, + ).astype(int) + ] = 1 + ttmp[np.arange(self.rng.randint(0, max(1, int(acceleration))), nt, acceleration).astype(int)] = 1 top_mat = toeplitz(ptmp, ttmp) ind = np.where(top_mat.ravel())[0] @@ -2755,8 +2849,8 @@ class KtGaussian1DMaskFunc(KtBaseMaskFunc): Amount of under-sampling. center_fractions : Union[list[float], tuple[float, ...]] Fraction of low-frequency columns (float < 1.0) or number of low-frequence columns (integer) to be retained. - uniform_range : bool, optional - If True then an acceleration will be uniformly sampled between the two values, by default False. + range_mode : RangeMode, optional + How accelerations are sampled. Defaults to RangeMode.DISCRETE. alpha : float, optional 0 < alpha < 1 controls sampling density; 0: uniform density, 1: maximally non-uniform density. Default: 0.28. @@ -2768,7 +2862,7 @@ def __init__( self, accelerations: Union[list[Number], tuple[Number, ...]], center_fractions: Union[list[float], tuple[float, ...]], - uniform_range: bool = False, + range_mode: RangeMode = RangeMode.DISCRETE, alpha: float = 0.28, std_scale: float = 5.0, ) -> None: @@ -2780,8 +2874,8 @@ def __init__( Amount of under-sampling. center_fractions : Union[list[float], tuple[float, ...]] Fraction of low-frequency columns (float < 1.0) or number of low-frequence columns (integer) to be retained. - uniform_range : bool, optional - If True then an acceleration will be uniformly sampled between the two values, by default False. + range_mode : RangeMode, optional + How accelerations are sampled. Defaults to RangeMode.DISCRETE. alpha : float, optional 0 < alpha < 1 controls sampling density; 0: uniform density, 1: maximally non-uniform density. Default: 0.28. @@ -2791,7 +2885,7 @@ def __init__( super().__init__( accelerations=accelerations, center_fractions=center_fractions, - uniform_range=uniform_range, + range_mode=range_mode, ) self.alpha = alpha self.std_scale = std_scale @@ -2823,7 +2917,7 @@ def mask_func( if len(shape) not in [4, 5]: raise ValueError("Shape should have 4 or 5 dimensions.") - (nt, num_rows, num_cols) = shape[-4:-1] + nt, num_rows, num_cols = shape[-4:-1] with temp_seed(self.rng, seed): center_fraction, acceleration = self.choose_acceleration() @@ -2831,7 +2925,8 @@ def mask_func( # Fully sampled rectangle region acs_mask = self.zero_pad_to_center( - np.ones((nt, num_rows, num_low_freqs)), (int(nt), int(num_rows), int(num_cols)) + np.ones((nt, num_rows, num_low_freqs)), + (int(nt), int(num_rows), int(num_cols)), ) if return_acs: @@ -2858,7 +2953,11 @@ def mask_func( n_tmp = tr - len(a) prob_tmp = prob.copy() prob_tmp[a] = 0 - p_tmp = self.rng.choice(np.arange(-num_cols // 2, num_cols // 2), n_tmp, p=prob_tmp / prob_tmp.sum()) + p_tmp = self.rng.choice( + np.arange(-num_cols // 2, num_cols // 2), + n_tmp, + p=prob_tmp / prob_tmp.sum(), + ) ti[ind : ind + n_tmp] = i ph[ind : ind + n_tmp] = p_tmp ind += n_tmp @@ -2876,7 +2975,9 @@ def mask_func( return self._reshape_and_add_coil_axis(mask, shape) -def integerize_seed(seed: Union[None, int, tuple[int, ...], list[int], Iterable[int]]) -> int: +def integerize_seed( + seed: Union[None, int, tuple[int, ...], list[int], Iterable[int]], +) -> int: """Returns an integer seed. If input is integer, will return the input. If input is None, will return a random integer seed. @@ -2933,9 +3034,9 @@ def build_masking_function( name: str, accelerations: Union[list[Number], tuple[Number, ...]], center_fractions: Optional[Union[list[Number], tuple[Number, ...]]] = None, - uniform_range: bool = False, + range_mode: RangeMode = RangeMode.DISCRETE, mode: MaskFuncMode = MaskFuncMode.STATIC, - **kwargs: dict[str, Any], + **kwargs: Any, ) -> BaseMaskFunc: """Builds a mask function. @@ -2947,10 +3048,10 @@ def build_masking_function( Amount of under-sampling. center_fractions : Union[list[Number], tuple[Number, ...]], optional Fraction of low-frequency columns (float < 1.0) or number of low-frequence columns (integer) to be retained. - If multiple values are provided, then one of these numbers is chosen uniformly each time. If uniform_range - is True, then two values should be given, by default None. - uniform_range : bool, optional - If True then an acceleration will be uniformly sampled between the two values, by default False. + If multiple values are provided, then one of these numbers is chosen uniformly each time. If range_mode is UNIFORM or LINEAR, + then two values should be given, by default None. + range_mode : RangeMode, optional + How accelerations are sampled. Defaults to RangeMode.DISCRETE. mode : MaskFuncMode, optional Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE. If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be @@ -2981,7 +3082,7 @@ def build_masking_function( kwargs.update( { "center_fractions": center_fractions, - "uniform_range": uniform_range, + "range_mode": range_mode, "mode": mode, } ) @@ -2995,4 +3096,7 @@ def build_masking_function( # Create the MaskFunc instance with the prepared arguments mask_func = MaskFunc(**init_args) + if isinstance(mask_func, BaseMaskFunc) and "range_mode" not in init_args: + mask_func.range_mode = range_mode + return mask_func diff --git a/direct/common/subsample_config.py b/direct/common/subsample_config.py index 56393f43..a640cfa1 100644 --- a/direct/common/subsample_config.py +++ b/direct/common/subsample_config.py @@ -19,7 +19,7 @@ from omegaconf import MISSING from direct.config.defaults import BaseConfig -from direct.types import MaskFuncMode +from direct.types import MaskFuncMode, RangeMode @dataclass @@ -27,7 +27,7 @@ class MaskingConfig(BaseConfig): name: str = MISSING accelerations: tuple[float, ...] = (5.0,) center_fractions: Optional[tuple[float, ...]] = (0.1,) - uniform_range: bool = False + range_mode: RangeMode = RangeMode.DISCRETE mode: MaskFuncMode = MaskFuncMode.STATIC val_accelerations: tuple[float, ...] = (5.0, 10.0) diff --git a/direct/data/datasets.py b/direct/data/datasets.py index 60566777..88e266ee 100644 --- a/direct/data/datasets.py +++ b/direct/data/datasets.py @@ -35,7 +35,7 @@ from direct.data.sens import simulate_sensitivity_maps from direct.types import PathOrString from direct.utils import remove_keys, str_to_class -from direct.utils.dataset import get_filenames_for_datasets +from direct.utils.dataset import get_filenames_for_datasets, maybe_attach_field_strength logger = logging.getLogger(__name__) @@ -245,6 +245,8 @@ def __getitem__(self, index: int) -> dict[str, Any]: if sample["kspace"].ndim == 2: # Singlecoil data does not always have coils at the first axis. sample["kspace"] = sample["kspace"][np.newaxis, ...] + maybe_attach_field_strength(sample) + if self.transform: sample = self.transform(sample) @@ -612,7 +614,9 @@ def __init__( self.logger.error(e) raise ValueError(e) filenames = get_filenames_for_datasets( - lists=filenames_lists, files_root=filenames_lists_root, data_root=data_root + lists=filenames_lists, + files_root=filenames_lists_root, + data_root=data_root, ) self.logger.info("Attempting to load %s filenames from list(s).", len(filenames)) else: @@ -804,7 +808,11 @@ def __getitem__(self, index: int) -> dict[str, Any]: if kspace.ndim == 2: # Single-coil data. kspace = kspace[np.newaxis, ...] - sample: dict[str, Any] = {"kspace": kspace, "filename": str(filename), "slice_no": slice_no} + sample: dict[str, Any] = { + "kspace": kspace, + "filename": str(filename), + "slice_no": slice_no, + } if self.compute_mask or (any("mask" in key for key in extra_data)): nx, ny = kspace.shape[-2:] @@ -834,7 +842,11 @@ def __getitem__(self, index: int) -> dict[str, Any]: sample.update(extra_data) shape = kspace.shape - sample["reconstruction_size"] = (int(np.round(shape[-2] / 3)), int(np.round(shape[-1] / 2)), 1) + sample["reconstruction_size"] = ( + int(np.round(shape[-2] / 3)), + int(np.round(shape[-1] / 2)), + 1, + ) if self.kspace_context: # Add context dimension in reconstruction size without any crop @@ -843,6 +855,8 @@ def __getitem__(self, index: int) -> dict[str, Any]: # If context put coil dim first sample["kspace"] = np.swapaxes(sample["kspace"], 0, 1) + maybe_attach_field_strength(sample) + if self.transform: sample = self.transform(sample) # ty: ignore[invalid-argument-type] @@ -1028,7 +1042,7 @@ def __init__( """ self.logger = logging.getLogger(type(self).__name__) - (self.nx, self.ny, self.nz) = (shape, shape, shape) if isinstance(shape, int) else tuple(shape) + self.nx, self.ny, self.nz = (shape, shape, shape) if isinstance(shape, int) else tuple(shape) self.num_coils = num_coils assert intensity in self.IMAGE_INTENSITIES, ( @@ -1097,7 +1111,11 @@ def sample_image(self, idx: int) -> np.ndarray: # pylint: disable=too-many-loca # Put ellipses where they need to be for j in range(self.ellipsoids.shape[0]): - center_x, center_y, center_z = self.center_xs[j], self.center_ys[j], self.center_zs[j] + center_x, center_y, center_z = ( + self.center_xs[j], + self.center_ys[j], + self.center_zs[j], + ) a, b, c = self.half_ax_as[j], self.half_ax_bs[j], self.half_ax_cs[j] ct0, st0 = ct[j], st[j] @@ -1141,6 +1159,7 @@ def __getitem__(self, index: int) -> dict[str, Any]: kspace = self.fft(image) sample = {"kspace": kspace, "filename": self.name, "slice_no": index} + maybe_attach_field_strength(sample) if self.transform is not None: sample = self.transform(sample) @@ -1154,15 +1173,15 @@ def default_mr_ellipsoid_parameters() -> np.ndarray: ------- ellipsoids : np.ndarray Array containing the parameters for the ellipsoids used to construct the phantom. - Each row of the form [x, y, z, a, b, c, \theta, m_0, A, C, T1, T2, \chi] represents an ellipsoid, where: + Each row of the form [x, y, z, a, b, c, theta, m_0, A, C, T1, T2, chi] represents an ellipsoid, where: * (x, y, z): denotes the center of the ellipsoid * (a, b, c): denote the lengths of the semi-major axis aligned with the x, y, z-axis, respectively - * \theta: denotes the rotation angle of the ellipsoid in rads + * theta: denotes the rotation angle of the ellipsoid in rads * m_0: denotes the spin density * (A, C): denote the T1 parameters - * T1: denotes the T1 value if explicit, otherwise T1 = A \times B_0^{C} + * T1: denotes the T1 value if explicit, otherwise T1 = A * B_0^{C} * T2: denotes the T2 value - * \chi: denotes the \chi value + * chi: denotes the chi value References ---------- @@ -1171,23 +1190,138 @@ def default_mr_ellipsoid_parameters() -> np.ndarray: """ params = _mr_relaxation_parameters() - ellipsoids = np.zeros((SheppLoganDataset.DEFAULT_NUM_ELLIPSOIDS, SheppLoganDataset.ELLIPSOID_NUM_PARAMS)) + ellipsoids = np.zeros( + ( + SheppLoganDataset.DEFAULT_NUM_ELLIPSOIDS, + SheppLoganDataset.ELLIPSOID_NUM_PARAMS, + ) + ) ellipsoids[0, :] = [0, 0, 0, 0.72, 0.95, 0.93, 0, 0.8, *params["scalp"]] ellipsoids[1, :] = [0, 0, 0, 0.69, 0.92, 0.9, 0, 0.12, *params["marrow"]] ellipsoids[2, :] = [0, -0.0184, 0, 0.6624, 0.874, 0.88, 0, 0.98, *params["csf"]] - ellipsoids[3, :] = [0, -0.0184, 0, 0.6524, 0.864, 0.87, 0, 0.745, *params["gray-matter"]] - ellipsoids[4, :] = [-0.22, 0, -0.25, 0.41, 0.16, 0.21, np.deg2rad(-72), 0.98, *params["csf"]] - ellipsoids[5, :] = [0.22, 0, -0.25, 0.31, 0.11, 0.22, np.deg2rad(72), 0.98, *params["csf"]] - ellipsoids[6, :] = [0, 0.35, -0.25, 0.21, 0.25, 0.35, 0, 0.617, *params["white-matter"]] - ellipsoids[7, :] = [0, 0.1, -0.25, 0.046, 0.046, 0.046, 0, 0.95, *params["tumor"]] - ellipsoids[8, :] = [-0.08, -0.605, -0.25, 0.046, 0.023, 0.02, 0, 0.95, *params["tumor"]] - ellipsoids[9, :] = [0.06, -0.605, -0.25, 0.046, 0.023, 0.02, np.deg2rad(-90), 0.95, *params["tumor"]] - ellipsoids[10, :] = [0, -0.1, -0.25, 0.046, 0.046, 0.046, 0, 0.95, *params["tumor"]] - ellipsoids[11, :] = [0, -0.605, -0.25, 0.023, 0.023, 0.023, 0, 0.95, *params["tumor"]] - ellipsoids[12, :] = [0.06, -0.105, 0.0625, 0.056, 0.04, 0.1, np.deg2rad(-90), 0.93, *params["tumor"]] + ellipsoids[3, :] = [ + 0, + -0.0184, + 0, + 0.6524, + 0.864, + 0.87, + 0, + 0.745, + *params["gray-matter"], + ] + ellipsoids[4, :] = [ + -0.22, + 0, + -0.25, + 0.41, + 0.16, + 0.21, + np.deg2rad(-72), + 0.98, + *params["csf"], + ] + ellipsoids[5, :] = [ + 0.22, + 0, + -0.25, + 0.31, + 0.11, + 0.22, + np.deg2rad(72), + 0.98, + *params["csf"], + ] + ellipsoids[6, :] = [ + 0, + 0.35, + -0.25, + 0.21, + 0.25, + 0.35, + 0, + 0.617, + *params["white-matter"], + ] + ellipsoids[7, :] = [ + 0, + 0.1, + -0.25, + 0.046, + 0.046, + 0.046, + 0, + 0.95, + *params["tumor"], + ] + ellipsoids[8, :] = [ + -0.08, + -0.605, + -0.25, + 0.046, + 0.023, + 0.02, + 0, + 0.95, + *params["tumor"], + ] + ellipsoids[9, :] = [ + 0.06, + -0.605, + -0.25, + 0.046, + 0.023, + 0.02, + np.deg2rad(-90), + 0.95, + *params["tumor"], + ] + ellipsoids[10, :] = [ + 0, + -0.1, + -0.25, + 0.046, + 0.046, + 0.046, + 0, + 0.95, + *params["tumor"], + ] + ellipsoids[11, :] = [ + 0, + -0.605, + -0.25, + 0.023, + 0.023, + 0.023, + 0, + 0.95, + *params["tumor"], + ] + ellipsoids[12, :] = [ + 0.06, + -0.105, + 0.0625, + 0.056, + 0.04, + 0.1, + np.deg2rad(-90), + 0.93, + *params["tumor"], + ] ellipsoids[13, :] = [0, 0.1, 0.625, 0.056, 0.056, 0.1, 0, 0.98, *params["csf"]] - ellipsoids[14, :] = [0.56, -0.4, -0.25, 0.2, 0.03, 0.1, np.deg2rad(70), 0.85, *params["blood-clot"]] + ellipsoids[14, :] = [ + 0.56, + -0.4, + -0.25, + 0.2, + 0.03, + 0.1, + np.deg2rad(70), + 0.85, + *params["blood-clot"], + ] # Need to subtract some ellipses here... ellipsoids_neg = np.zeros(ellipsoids.shape) @@ -1491,10 +1625,20 @@ def build_dataset_from_input( f"Got {kwargs.get('initial_images')} and {kwargs.get('initial_kspaces')}." ) if "initial_images" in kwargs: - pass_h5s = {"initial_image": (dataset_config.input_image_key, kwargs.get("initial_images"))} + pass_h5s = { + "initial_image": ( + dataset_config.input_image_key, + kwargs.get("initial_images"), + ) + } del kwargs["initial_images"] elif "initial_kspaces" in kwargs: - pass_h5s = {"initial_kspace": (dataset_config.input_kspace_key, kwargs.get("initial_kspaces"))} + pass_h5s = { + "initial_kspace": ( + dataset_config.input_kspace_key, + kwargs.get("initial_kspaces"), + ) + } del kwargs["initial_kspaces"] if pass_h5s is not None: kwargs.update({"pass_h5s": pass_h5s}) @@ -1503,7 +1647,8 @@ def build_dataset_from_input( # case the arguments in kwargs. # For example, `data_root` can be passed both from the command line and in the configuration file. config_kwargs = remove_keys( - dict(dataset_config), ["name", "transforms"] + list(kwargs.keys() & dict(dataset_config).keys()) + dict(dataset_config), + ["name", "transforms"] + list(kwargs.keys() & dict(dataset_config).keys()), ) dataset = build_dataset( name=dataset_config.name, # type: ignore diff --git a/direct/data/h5_data.py b/direct/data/h5_data.py index ce55130e..4212af25 100644 --- a/direct/data/h5_data.py +++ b/direct/data/h5_data.py @@ -22,7 +22,7 @@ from direct.types import PathOrString from direct.utils import cast_as_path -from direct.utils.dataset import get_filenames_for_datasets +from direct.utils.dataset import get_filenames_for_datasets, maybe_attach_field_strength logger = logging.getLogger(__name__) @@ -248,6 +248,7 @@ def __getitem__(self, index: int) -> Dict[str, Any]: raise ValueError(f"Trying to add key {key} to sample dict, but this key already exists.") sample[key] = curr_slice + maybe_attach_field_strength(sample) return sample def get_slice_data(self, filename, slice_no, key="kspace", pass_attrs=False, extra_keys=None): diff --git a/direct/data/mri_transforms.py b/direct/data/mri_transforms.py index 653255be..a6570ad8 100644 --- a/direct/data/mri_transforms.py +++ b/direct/data/mri_transforms.py @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. """The `direct.data.mri_transforms` module contains mri transformations utilized to transform or augment k-space data, -used for DIRECT's training pipeline. They can be also used individually by importing them into python scripts.""" +used for DIRECT's training pipeline. They can be also used individually by importing them into python scripts. +""" from __future__ import annotations @@ -210,9 +211,7 @@ def __call__(self, sample: dict[str, Any]) -> dict[str, Any]: else ( (-1,) if self.flip == "vertical" - else (-2, -1) - if self.flip == "both" - else (random.SystemRandom().choice([-2, -1]),) + else ((-2, -1) if self.flip == "both" else (random.SystemRandom().choice([-2, -1]),)) ) ) @@ -285,6 +284,13 @@ def __call__(self, sample: dict[str, Any]) -> dict[str, Any]: return sample +def _mask_func_supports_return_acceleration(mask_func: Callable) -> bool: + """Check whether a mask callable can return sampled acceleration metadata.""" + from direct.common.subsample import BaseMaskFunc + + return isinstance(mask_func, BaseMaskFunc) + + class CreateSamplingMask(DirectTransform): """Data Transformer for training MRI reconstruction models. @@ -341,13 +347,22 @@ def __call__(self, sample: dict[str, Any]) -> dict[str, Any]: seed = None if not self.use_seed else tuple(map(ord, str(sample["filename"]))) - sampling_mask = self.mask_func(shape=shape, seed=seed, return_acs=False) + mask_kwargs = {"shape": shape, "seed": seed, "return_acs": False} + if _mask_func_supports_return_acceleration(self.mask_func): + mask_kwargs["return_acceleration"] = True + sampling_mask, acceleration, center_fraction = self.mask_func(**mask_kwargs) + if "padding" in sample: + sampling_mask = T.apply_padding(sampling_mask, sample["padding"]) - if "padding" in sample: - sampling_mask = T.apply_padding(sampling_mask, sample["padding"]) + sample["sampling_mask"] = sampling_mask + sample["acceleration"] = torch.tensor([acceleration], dtype=sample["kspace"].dtype) + sample["center_fraction"] = torch.tensor([center_fraction], dtype=sample["kspace"].dtype) + else: + sampling_mask = self.mask_func(**mask_kwargs) + if "padding" in sample: + sampling_mask = T.apply_padding(sampling_mask, sample["padding"]) - # Shape 3D: (1, 1, height, width, 1), 2D: (1, height, width, 1) - sample["sampling_mask"] = sampling_mask + sample["sampling_mask"] = sampling_mask if self.return_acs: sample["acs_mask"] = self.mask_func(shape=shape, seed=seed, return_acs=True) @@ -475,7 +490,7 @@ def __init__( else: self.crop_func = functools.partial( T.complex_random_crop, - sampler=random_crop_sampler_type if random_crop_sampler_type is not None else "uniform", + sampler=(random_crop_sampler_type if random_crop_sampler_type is not None else "uniform"), sigma=random_crop_sampler_gaussian_sigma, ) self.random_crop_sampler_use_seed = random_crop_sampler_use_seed @@ -656,7 +671,9 @@ def __call__(self, sample: dict[str, Any]) -> dict[str, Any]: backprojected_kspace = backprojected_kspace.unsqueeze(0) rescaled_backprojected_kspace = T.complex_image_resize( - backprojected_kspace, self.shape, self.rescale_mode if self.rescale_mode is not None else "nearest" + backprojected_kspace, + self.shape, + self.rescale_mode if self.rescale_mode is not None else "nearest", ) if (kspace.ndim == 4) or (kspace.ndim == 5 and not self.rescale_2d_if_3d): @@ -917,14 +934,14 @@ def forward(self, sample: dict[str, Any]) -> dict[str, Any]: Parameters ---------- sample: dict[str, Any] - Contains key kspace_key with value a torch.Tensor of shape (coil,\*spatial_dims, complex=2). + Contains key kspace_key with value a torch.Tensor of shape (coil, \\*spatial_dims, complex=2). Returns ------- sample: dict - Contains key target_key with value a torch.Tensor of shape (\*spatial_dims) if `type_reconstruction` is + Contains key target_key with value a torch.Tensor of shape (\\*spatial_dims) if `type_reconstruction` is ReconstructionType.RSS, ReconstructionType.COMPLEX_MOD, ReconstructionType.SENSE_MOD, - and of shape (\*spatial_dims, complex_dim=2) otherwise. + and of shape (\\*spatial_dims, complex_dim=2) otherwise. """ kspace_data = sample[self.kspace_key] dim = self.spatial_dims.TWO_D if kspace_data.ndim == 5 else self.spatial_dims.THREE_D @@ -1281,7 +1298,10 @@ def forward(self, sample: dict[str, Any]) -> dict[str, Any]: if ndim == 6: compressed_k_space = compressed_k_space.reshape( - shape[0] // num_slice_or_time, num_slice_or_time, self.num_coils, *shape[2:] + shape[0] // num_slice_or_time, + num_slice_or_time, + self.num_coils, + *shape[2:], ).permute(0, 2, 1, 3, 4) compressed_k_space = torch.view_as_real(compressed_k_space) @@ -2240,7 +2260,9 @@ def build_supervised_mri_transforms( ] mri_transforms += [ ComputeScalingFactor( - normalize_key=scaling_key, percentile=scale_percentile, scaling_factor_key=TransformKey.SCALING_FACTOR + normalize_key=scaling_key, + percentile=scale_percentile, + scaling_factor_key=TransformKey.SCALING_FACTOR, ), Normalize( scaling_factor_key=TransformKey.SCALING_FACTOR, @@ -2483,8 +2505,8 @@ def build_mri_transforms( sensitivity_maps_espirit_kernel_size=sensitivity_maps_espirit_kernel_size, sensitivity_maps_espirit_crop=sensitivity_maps_espirit_crop, sensitivity_maps_espirit_max_iters=sensitivity_maps_espirit_max_iters, - delete_acs_mask=delete_acs_mask if transforms_type == TransformsType.SUPERVISED else False, - delete_kspace=delete_kspace if transforms_type == TransformsType.SUPERVISED else False, + delete_acs_mask=(delete_acs_mask if transforms_type == TransformsType.SUPERVISED else False), + delete_kspace=(delete_kspace if transforms_type == TransformsType.SUPERVISED else False), image_recon_type=image_recon_type, compress_coils=compress_coils, pad_coils=pad_coils, diff --git a/direct/data/sens.py b/direct/data/sens.py index 90c74d8f..28994bfe 100644 --- a/direct/data/sens.py +++ b/direct/data/sens.py @@ -18,7 +18,10 @@ def simulate_sensitivity_maps( - shape: Union[List[int], Tuple[int, ...]], num_coils: int, var: float = 1, seed: Optional[int] = None + shape: Union[List[int], Tuple[int, ...]], + num_coils: int, + var: float = 1, + seed: Optional[int] = None, ) -> np.ndarray: r"""Simulates coil sensitivities using bi-variate or tri-variate gaussian distribution. @@ -36,7 +39,7 @@ def simulate_sensitivity_maps( Returns ------- sensitivity_map : nd.array - Simulated coil sensitivity maps of shape (num_coils, \*shape). + Simulated coil sensitivity maps of shape (num_coils, \\*shape). Notes ----- diff --git a/direct/data/transforms.py b/direct/data/transforms.py index 0b0dd9a3..61bd979b 100644 --- a/direct/data/transforms.py +++ b/direct/data/transforms.py @@ -9,7 +9,8 @@ This module contains functions for complex-valued data manipulation in PyTorch. This includes functions for complex multiplication, division, modulus, fft, ifft, fftshift, ifftshift, and more. The functions are designed to work with complex-valued data where the last axis denotes the real and imaginary parts respectively. The functions are designed to -work with complex-valued data where the last axis denotes the real and imaginary parts respectively.""" +work with complex-valued data where the last axis denotes the real and imaginary parts respectively. +""" from __future__ import annotations @@ -74,12 +75,12 @@ def view_as_complex(data): ---------- data: torch.Tensor Input data with torch.dtype torch.float64 and torch.float32 with complex axis (last) of dimension 2 - and of shape (N, \*, 2). + and of shape (N, \\*, 2). Returns ------- complex_valued_data: torch.Tensor - Output complex-valued data of shape (N, \*) with complex torch.dtype. + Output complex-valued data of shape (N, \\*) with complex torch.dtype. """ return torch.view_as_complex(data) @@ -93,12 +94,12 @@ def view_as_real(data): Parameters ---------- data: torch.Tensor - Input data with complex torch.dtype of shape (N, \*). + Input data with complex torch.dtype of shape (N, \\*). Returns ------- real_valued_data: torch.Tensor - Output real-valued data of shape (N, \*, 2). + Output real-valued data of shape (N, \\*, 2). """ return torch.view_as_real(data) @@ -119,7 +120,7 @@ def fft2( Parameters ---------- data: torch.Tensor - Complex-valued input tensor. Should be of shape (\*, 2) and dim is in \*. + Complex-valued input tensor. Should be of shape (\\*, 2) and dim is in \\*. dim: tuple, list or int Dimensions over which to compute. Should be positive. Negative indexing not supported Default is (1, 2), corresponding to ('height', 'width'). @@ -179,7 +180,7 @@ def ifft2( Parameters ---------- data: torch.Tensor - Complex-valued input tensor. Should be of shape (\*, 2) and dim is in \*. + Complex-valued input tensor. Should be of shape (\\*, 2) and dim is in \\*. dim: tuple, list or int Dimensions over which to compute. Should be positive. Negative indexing not supported Default is (1, 2), corresponding to ( 'height', 'width'). @@ -460,10 +461,12 @@ def complex_division(input_tensor: torch.Tensor, other_tensor: torch.Tensor) -> denominator = other_tensor[..., 0] ** 2 + other_tensor[..., 1] ** 2 real_part = safe_divide( - input_tensor[..., 0] * other_tensor[..., 0] + input_tensor[..., 1] * other_tensor[..., 1], denominator + input_tensor[..., 0] * other_tensor[..., 0] + input_tensor[..., 1] * other_tensor[..., 1], + denominator, ) imaginary_part = safe_divide( - input_tensor[..., 1] * other_tensor[..., 0] - input_tensor[..., 0] * other_tensor[..., 1], denominator + input_tensor[..., 1] * other_tensor[..., 0] - input_tensor[..., 0] * other_tensor[..., 1], + denominator, ) division = torch.cat( @@ -569,7 +572,7 @@ def apply_padding( Parameters ---------- data : torch.Tensor - Batched or not input to be padded of shape (`batch`, \*, `height`, `width`, \*). + Batched or not input to be padded of shape (`batch`, \\*, `height`, `width`, \\*). padding : torch.Tensor or None Binary tensor of shape (`batch`, 1, `height`, `width`, 1). Entries in `padding` with non-zero value point to samples in `data` that will be zero-padded. If None, `data` will be returned. @@ -873,12 +876,12 @@ def crop_to_acs(acs_mask: torch.Tensor, kspace: torch.Tensor) -> torch.Tensor: acs_mask : torch.Tensor Autocalibration mask of shape (height, width). kspace : torch.Tensor - K-space of shape (coil, height, width, \*). + K-space of shape (coil, height, width, \\*). Returns ------- torch.Tensor - Cropped k-space of shape (coil, height', width', \*), where height' and width' are the new dimensions derived + Cropped k-space of shape (coil, height', width', \\*), where height' and width' are the new dimensions derived from the acs_mask. """ nonzero_idxs = torch.nonzero(acs_mask) diff --git a/direct/engine.py b/direct/engine.py index ec0d5622..5e1d6fb2 100644 --- a/direct/engine.py +++ b/direct/engine.py @@ -518,7 +518,11 @@ def validation_loop( ) storage.add_image(f"{key_prefix}target", visualize_target) - self.logger.info("Done evaluation of %s at iteration %s.", str(curr_dataset_name), str(iter_idx)) + self.logger.info( + "Done evaluation of %s at iteration %s.", + str(curr_dataset_name), + str(iter_idx), + ) self.model.train() def process_slices_for_visualization(self, visualize_slices, visualize_target): @@ -526,7 +530,7 @@ def process_slices_for_visualization(self, visualize_slices, visualize_target): # Compute the difference as well, and normalize for visualization difference_slices = [a - b for a, b in zip(visualize_slices, visualize_target)] # Normalize slices - difference_slices = [(d / np.abs(d)) * 0.5 + 0.5 for d in difference_slices] + difference_slices = [(d / d.abs().clamp(min=1e-8)) * 0.5 + 0.5 for d in difference_slices] visualize_slices = [normalize_image(image) for image in visualize_slices] # Visualize slices, and crop to the largest volume diff --git a/direct/nn/adain/__init__.py b/direct/nn/adain/__init__.py new file mode 100644 index 00000000..1be13d14 --- /dev/null +++ b/direct/nn/adain/__init__.py @@ -0,0 +1,18 @@ +# Copyright 2026 AI for Oncology Research Group. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""direct.nn.adain module.""" + +from direct.nn.adain.adain import AdaIN2d, AdaIN3d, NormType + +__all__ = ["AdaIN2d", "AdaIN3d", "NormType"] diff --git a/direct/nn/adain/adain.py b/direct/nn/adain/adain.py new file mode 100644 index 00000000..4ce4c134 --- /dev/null +++ b/direct/nn/adain/adain.py @@ -0,0 +1,168 @@ +# Copyright 2026 AI for Oncology Research Group. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Adaptive Instance Normalization (AdaIN) modules for 2D and 3D tensors based on [1]_. + +References +---------- + +.. [1] Moriakov, N., Yiasemis, G., Sonke, J.-J. & Teuwen, J. (2026). Conditional Learned Reconstruction for + Medical Imaging. Proceedings of The 9th International Conference on Medical Imaging with Deep Learning, + PMLR 315:754-780. https://proceedings.mlr.press/v315/moriakov26a.html +""" + +from __future__ import annotations + +from enum import Enum + +import torch +from torch import nn + +__all__ = ["AdaIN2d", "AdaIN3d", "NormType"] + + +class NormType(str, Enum): + INSTANCE = "instance" + ADAIN = "adain" + + +class AdaIN2d(nn.Module): + """Adaptive Instance Normalization for 2D tensors based on [1]_. + + Given input x of shape (B, C, H, W) and auxiliary vector y of shape (B, F), + produces per-sample, per-channel affine parameters from y. + + References + ---------- + + .. [1] Moriakov, N., Yiasemis, G., Sonke, J.-J. & Teuwen, J. (2026). Conditional Learned Reconstruction for + Medical Imaging. Proceedings of The 9th International Conference on Medical Imaging with Deep Learning, + PMLR 315:754-780. https://proceedings.mlr.press/v315/moriakov26a.html + """ + + def __init__( + self, + num_channels: int, + aux_in_features: int, + hidden_features: int | tuple[int, ...] | None = None, + activation: nn.Module | None = None, + eps: float = 1e-5, + use_one_plus_gamma: bool = True, + ): + super().__init__() + self.num_channels = num_channels + self.eps = eps + self.use_one_plus_gamma = use_one_plus_gamma + + if activation is None: + activation = nn.SiLU() + + if hidden_features is None: + hidden = [] + elif isinstance(hidden_features, int): + hidden = [hidden_features] + else: + hidden = list(hidden_features) + + layers: list[nn.Module] = [] + in_f = aux_in_features + for h in hidden: + layers += [nn.Linear(in_f, h), activation] + in_f = h + layers += [nn.Linear(in_f, 2 * num_channels)] + self.mlp = nn.Sequential(*layers) + + if isinstance(self.mlp[-1], nn.Linear): + nn.init.zeros_(self.mlp[-1].weight) + nn.init.zeros_(self.mlp[-1].bias) + + def forward(self, x: torch.Tensor, y: torch.Tensor) -> torch.Tensor: + mean = x.mean(dim=(2, 3), keepdim=True) + var = x.var(dim=(2, 3), keepdim=True, unbiased=False) + x_norm = (x - mean) / torch.sqrt(var + self.eps) + + params = self.mlp(y) + gamma, beta = params.chunk(2, 1) + + gamma = gamma.view(-1, self.num_channels, 1, 1) + beta = beta.view(-1, self.num_channels, 1, 1) + + if self.use_one_plus_gamma: + return x_norm * (1.0 + gamma) + beta + return x_norm * gamma + beta + + +class AdaIN3d(nn.Module): + """Adaptive Instance Normalization for 3D tensors based on [1]_. + + Given input x of shape (B, C, Z, H, W) and auxiliary vector y of shape (B, F), + produces per-sample, per-channel affine parameters from y. + + References + ---------- + + .. [1] Moriakov, N., Yiasemis, G., Sonke, J.-J. & Teuwen, J. (2026). Conditional Learned Reconstruction for + Medical Imaging. Proceedings of The 9th International Conference on Medical Imaging with Deep Learning, + PMLR 315:754-780. https://proceedings.mlr.press/v315/moriakov26a.html + """ + + def __init__( + self, + num_channels: int, + aux_in_features: int, + hidden_features: int | tuple[int, ...] | None = None, + activation: nn.Module | None = None, + eps: float = 1e-5, + use_one_plus_gamma: bool = True, + ): + super().__init__() + self.num_channels = num_channels + self.eps = eps + self.use_one_plus_gamma = use_one_plus_gamma + + if activation is None: + activation = nn.SiLU() + + if hidden_features is None: + hidden = [] + elif isinstance(hidden_features, int): + hidden = [hidden_features] + else: + hidden = list(hidden_features) + + layers: list[nn.Module] = [] + in_f = aux_in_features + for h in hidden: + layers += [nn.Linear(in_f, h), activation] + in_f = h + layers += [nn.Linear(in_f, 2 * num_channels)] + self.mlp = nn.Sequential(*layers) + + if isinstance(self.mlp[-1], nn.Linear): + nn.init.zeros_(self.mlp[-1].weight) + nn.init.zeros_(self.mlp[-1].bias) + + def forward(self, x: torch.Tensor, y: torch.Tensor) -> torch.Tensor: + mean = x.mean(dim=(2, 3, 4), keepdim=True) + var = x.var(dim=(2, 3, 4), keepdim=True, unbiased=False) + x_norm = (x - mean) / torch.sqrt(var + self.eps) + + params = self.mlp(y) + gamma, beta = params.chunk(2, dim=-1) + + gamma = gamma.view(-1, self.num_channels, 1, 1, 1) + beta = beta.view(-1, self.num_channels, 1, 1, 1) + + if self.use_one_plus_gamma: + return x_norm * (1.0 + gamma) + beta + return x_norm * gamma + beta diff --git a/direct/nn/conv/conv.py b/direct/nn/conv/conv.py index 5df2547d..abb56f75 100644 --- a/direct/nn/conv/conv.py +++ b/direct/nn/conv/conv.py @@ -11,16 +11,27 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import List +from __future__ import annotations + +from typing import List, Optional import torch import torch.nn as nn +from direct.nn.conv.modulated import ( + ModConv2dBias, + ModConvActivation, + ModConvType, + ModulationParams, + mod_conv2d, +) + class Conv2d(nn.Module): """Implementation of a simple cascade of 2D convolutions. If `batchnorm` is set to True, batch normalization layer is applied after each convolution. + Supports modulated convolutions when `modulation` is not ModConvType.NONE. """ def __init__( @@ -31,6 +42,13 @@ def __init__( n_convs: int = 3, activation: nn.Module = nn.PReLU(), batchnorm: bool = False, + modulation: ModConvType = ModConvType.NONE, + modulation_params: Optional[ModulationParams] = None, + aux_in_features: Optional[int] = None, + fc_hidden_features: Optional[tuple[int] | int] = None, + fc_groups: int = 1, + fc_activation: ModConvActivation = ModConvActivation.SIGMOID, + num_weights: Optional[int] = None, ): """Inits :class:`Conv2d`. @@ -48,37 +66,90 @@ def __init__( Activation function. batchnorm: bool If True a batch normalization layer is applied after every convolution. + modulation : ModConvType + Modulation type. Default: ModConvType.NONE. + aux_in_features : int, optional + Number of features in the auxiliary input for modulation. + fc_hidden_features : int or tuple of int, optional + Hidden features in the modulation MLP. + fc_groups : int + Groups for modulation MLP output interpolation. Default: 1. + fc_activation : ModConvActivation + Activation after modulation MLP. Default: ModConvActivation.SIGMOID. + num_weights : int, optional + Number of weight bases for ModConvType.SUM. """ super().__init__() - conv: List[nn.Module] = [] + if modulation_params is None: + modulation_params = ModulationParams( + modulation=modulation, + aux_in_features=aux_in_features, + fc_hidden_features=fc_hidden_features, + fc_groups=fc_groups, + fc_activation=fc_activation, + num_weights=num_weights, + ) + self.modulation = modulation_params.modulation + + conv_layers: List[nn.Module] = [] + norm_layers: List[Optional[nn.Module]] = [] + act_layers: List[Optional[nn.Module]] = [] + for idx in range(n_convs): - conv.append( - nn.Conv2d( - in_channels if idx == 0 else hidden_channels, - hidden_channels if idx != n_convs - 1 else out_channels, + ic = in_channels if idx == 0 else hidden_channels + oc = hidden_channels if idx != n_convs - 1 else out_channels + + conv_layers.append( + mod_conv2d( + ic, + oc, kernel_size=3, padding=1, + bias=ModConv2dBias.PARAM, + modulation_params=modulation_params, ) ) if batchnorm: - conv.append(nn.BatchNorm2d(hidden_channels if idx != n_convs - 1 else out_channels, eps=1e-4)) + norm_layers.append(nn.BatchNorm2d(oc, eps=1e-4)) + else: + norm_layers.append(None) if idx != n_convs - 1: - conv.append(activation) - self.conv = nn.Sequential(*conv) + act_layers.append(activation) + else: + act_layers.append(None) + + self.conv_layers = nn.ModuleList(conv_layers) + self.norm_layers = nn.ModuleList([m for m in norm_layers if m is not None]) if batchnorm else None + self.act_layers = act_layers + self.n_convs = n_convs + self.batchnorm = batchnorm - def forward(self, x: torch.Tensor) -> torch.Tensor: + def forward(self, x: torch.Tensor, y: Optional[torch.Tensor] = None) -> torch.Tensor: """Performs the forward pass of :class:`Conv2d`. Parameters ---------- x: torch.Tensor Input tensor. + y: torch.Tensor, optional + Auxiliary signal for modulation of shape (N, aux_in_features). Returns ------- out: torch.Tensor Convoluted output. """ - out = self.conv(x) - return out + norm_idx = 0 + for idx in range(self.n_convs): + if self.modulation != ModConvType.NONE: + x = self.conv_layers[idx](x, y) + else: + x = self.conv_layers[idx](x) + if self.batchnorm and self.norm_layers is not None: + x = self.norm_layers[norm_idx](x) + norm_idx += 1 + act_layer = self.act_layers[idx] + if act_layer is not None: + x = act_layer(x) + return x diff --git a/direct/nn/conv/modulated/__init__.py b/direct/nn/conv/modulated/__init__.py new file mode 100644 index 00000000..4fe70f64 --- /dev/null +++ b/direct/nn/conv/modulated/__init__.py @@ -0,0 +1,62 @@ +# Copyright 2026 AI for Oncology Research Group. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Modulated convolution layers and auxiliary conditioning utilities.""" + +from direct.nn.conv.modulated.auxiliary_data import ( + AUXILIARY_FEATURE_REGISTRY, + DEFAULT_AUXILIARY_FEATURE_NAMES, + AuxiliaryFeature, + ModulationConfig, + prepare_auxiliary_data, + register_auxiliary_feature, + resolve_auxiliary_features, +) +from direct.nn.conv.modulated.factory import ( + ModulationParams, + mod_conv2d, + mod_conv3d, + mod_conv_transpose2d, + mod_conv_transpose3d, +) +from direct.nn.conv.modulated.modulated_conv import ( + ModConv2d, + ModConv2dBias, + ModConv3d, + ModConvActivation, + ModConvTranspose2d, + ModConvTranspose3d, + ModConvType, +) + +__all__ = [ + "ModulationParams", + "mod_conv2d", + "mod_conv3d", + "mod_conv_transpose2d", + "mod_conv_transpose3d", + "AUXILIARY_FEATURE_REGISTRY", + "AuxiliaryFeature", + "DEFAULT_AUXILIARY_FEATURE_NAMES", + "ModConv2d", + "ModConv2dBias", + "ModConv3d", + "ModConvActivation", + "ModConvTranspose2d", + "ModConvTranspose3d", + "ModConvType", + "ModulationConfig", + "prepare_auxiliary_data", + "register_auxiliary_feature", + "resolve_auxiliary_features", +] diff --git a/direct/nn/conv/modulated/auxiliary_data.py b/direct/nn/conv/modulated/auxiliary_data.py new file mode 100644 index 00000000..9c943220 --- /dev/null +++ b/direct/nn/conv/modulated/auxiliary_data.py @@ -0,0 +1,231 @@ +# Copyright 2026 AI for Oncology Research Group. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Utilities for building auxiliary conditioning vectors for modulated convolutions.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Mapping, Optional, Protocol, Sequence + +import torch + +from direct.nn.adain.adain import NormType +from direct.nn.conv.modulated.modulated_conv import ModConvType + +__all__ = [ + "AUXILIARY_FEATURE_REGISTRY", + "AuxiliaryFeature", + "DEFAULT_AUXILIARY_FEATURE_NAMES", + "ModulationConfig", + "prepare_auxiliary_data", + "register_auxiliary_feature", + "resolve_auxiliary_features", +] + + +@dataclass(frozen=True) +class AuxiliaryFeature: + """Metadata for one auxiliary conditioning channel. + + Parameters + ---------- + key : str + Key in the batch dictionary. + log_scale : float + Multiplier applied to the feature before ``log`` when ``log_aux`` is enabled. + """ + + key: str + log_scale: float = 1.0 + + +AUXILIARY_FEATURE_REGISTRY: dict[str, AuxiliaryFeature] = { + "acceleration": AuxiliaryFeature("acceleration"), + "center_fraction": AuxiliaryFeature("center_fraction", log_scale=100.0), + "field_strength": AuxiliaryFeature("field_strength"), +} + +DEFAULT_AUXILIARY_FEATURE_NAMES: tuple[str, ...] = tuple(AUXILIARY_FEATURE_REGISTRY.keys()) + + +class ModulationConfig(Protocol): + """Minimal model configuration required to build auxiliary data.""" + + conv_modulation: ModConvType + aux_in_features: int + log_aux: bool + auxiliary_features: Optional[tuple[str, ...]] + + +def register_auxiliary_feature(feature: AuxiliaryFeature) -> None: + """Register a custom auxiliary feature for use in ``auxiliary_features`` configs.""" + AUXILIARY_FEATURE_REGISTRY[feature.key] = feature + + +def resolve_auxiliary_features( + feature_names: Optional[Sequence[str]], + aux_in_features: int, +) -> tuple[AuxiliaryFeature, ...]: + """Resolve auxiliary feature names from config into feature metadata. + + Parameters + ---------- + feature_names : Sequence[str], optional + Explicit ordered list of feature keys. When ``None``, the first ``aux_in_features`` entries + from :data:`DEFAULT_AUXILIARY_FEATURE_NAMES` are used. + aux_in_features : int + Expected number of auxiliary channels. Must match the resolved list length. + + Returns + ------- + tuple[AuxiliaryFeature, ...] + Resolved feature metadata in request order. + + Raises + ------ + ValueError + If feature names are unknown, or the list length does not match ``aux_in_features``. + """ + if aux_in_features <= 0: + raise ValueError(f"aux_in_features must be positive, got {aux_in_features}.") + + if feature_names is None: + if aux_in_features > len(DEFAULT_AUXILIARY_FEATURE_NAMES): + raise ValueError( + f"aux_in_features={aux_in_features} exceeds the number of default auxiliary " + f"features ({len(DEFAULT_AUXILIARY_FEATURE_NAMES)}): " + f"{list(DEFAULT_AUXILIARY_FEATURE_NAMES)}. " + f"Set auxiliary_features explicitly in the model config." + ) + names = DEFAULT_AUXILIARY_FEATURE_NAMES[:aux_in_features] + else: + names = tuple(feature_names) + if len(names) != aux_in_features: + raise ValueError( + f"auxiliary_features has length {len(names)} ({list(names)}) but aux_in_features={aux_in_features}." + ) + + unknown = sorted(set(names) - AUXILIARY_FEATURE_REGISTRY.keys()) + if unknown: + raise ValueError( + f"Unknown auxiliary feature(s): {unknown}. Known features: {sorted(AUXILIARY_FEATURE_REGISTRY)}." + ) + + return tuple(AUXILIARY_FEATURE_REGISTRY[name] for name in names) + + +def _needs_auxiliary_data(cfg: Any) -> bool: + """Return whether the model config requires auxiliary conditioning vectors.""" + if not hasattr(cfg, "conv_modulation"): + return False + + conv_modulation = cfg.conv_modulation + if isinstance(conv_modulation, str): + conv_modulation = ModConvType.from_str(conv_modulation) or ModConvType.NONE + + if conv_modulation != ModConvType.NONE: + return True + + for attr in ("image_unet_norm_type", "unet_norm_type"): + norm_type = getattr(cfg, attr, None) + if norm_type is None: + continue + if isinstance(norm_type, str) and norm_type.upper() == "ADAIN": + return True + if getattr(norm_type, "name", "").upper() == "ADAIN": + return True + if norm_type == NormType.ADAIN: + return True + + return False + + +def prepare_auxiliary_data( + data: Mapping[str, Any], + cfg: Optional[ModulationConfig], + *, + features: Optional[Sequence[AuxiliaryFeature]] = None, +) -> Optional[torch.Tensor]: + """Build an auxiliary conditioning vector for modulated models. + + Parameters + ---------- + data : Mapping[str, Any] + Batch dictionary containing the auxiliary feature tensors. + cfg : ModulationConfig + Model configuration with modulation settings. Uses ``cfg.auxiliary_features`` when + ``features`` is not provided. + features : Sequence[AuxiliaryFeature], optional + Explicit feature list, mainly for testing. Overrides ``cfg.auxiliary_features``. + + Returns + ------- + torch.Tensor or None + Tensor of shape ``(batch_size, aux_in_features)``, or ``None`` when modulation is disabled. + + Raises + ------ + ValueError + If auxiliary configuration is invalid or a feature tensor has an unexpected shape. + KeyError + If a required auxiliary feature is missing from ``data``. + """ + if cfg is None or not _needs_auxiliary_data(cfg): + return None + + log_aux = getattr(cfg, "log_aux", False) + auxiliary_features = getattr(cfg, "auxiliary_features", None) + aux_in_features = getattr(cfg, "aux_in_features", None) + if aux_in_features is None: + return None + + selected_features = ( + tuple(features) if features is not None else resolve_auxiliary_features(auxiliary_features, aux_in_features) + ) + + components = [_prepare_feature(data, feature, log_aux=log_aux) for feature in selected_features] + auxiliary_data = torch.cat(components, dim=1) + + if log_aux: + auxiliary_data = auxiliary_data.log() + + return auxiliary_data + + +def _prepare_feature(data: Mapping[str, Any], feature: AuxiliaryFeature, *, log_aux: bool) -> torch.Tensor: + if feature.key not in data: + raise KeyError( + f"Missing auxiliary feature '{feature.key}' required for modulation. Available keys: {sorted(data.keys())}." + ) + + tensor = _to_aux_column(data[feature.key]) + if log_aux and feature.log_scale != 1.0: + tensor = tensor * feature.log_scale + return tensor + + +def _to_aux_column(tensor: torch.Tensor) -> torch.Tensor: + """Convert a scalar auxiliary value to shape ``(batch_size, 1)``.""" + tensor = tensor.float() + + if tensor.ndim == 0: + return tensor.reshape(1, 1) + if tensor.ndim == 1: + return tensor.unsqueeze(-1) + if tensor.ndim == 2 and tensor.shape[-1] == 1: + return tensor + + raise ValueError( + f"Expected auxiliary feature with shape (batch_size,) or (batch_size, 1), got {tuple(tensor.shape)}." + ) diff --git a/direct/nn/conv/modulated/factory.py b/direct/nn/conv/modulated/factory.py new file mode 100644 index 00000000..75f4c216 --- /dev/null +++ b/direct/nn/conv/modulated/factory.py @@ -0,0 +1,175 @@ +# Copyright 2026 AI for Oncology Research Group. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Typed helpers for constructing modulated convolution layers.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Optional + +from direct.nn.conv.modulated.modulated_conv import ( + ModConv2d, + ModConv2dBias, + ModConv3d, + ModConvActivation, + ModConvTranspose2d, + ModConvTranspose3d, + ModConvType, +) +from direct.types import IntOrTuple + +__all__ = [ + "ModulationParams", + "mod_conv2d", + "mod_conv3d", + "mod_conv_transpose2d", + "mod_conv_transpose3d", +] + + +def mod_conv2d( + in_channels: int, + out_channels: int, + *, + kernel_size: IntOrTuple, + modulation_params: Optional[ModulationParams] = None, + stride: IntOrTuple = 1, + padding: IntOrTuple = 0, + dilation: IntOrTuple = 1, + bias: ModConv2dBias = ModConv2dBias.PARAM, +) -> ModConv2d: + """See :class:`ModConv2d`.""" + params = modulation_params or ModulationParams() + return ModConv2d( + in_channels, + out_channels, + kernel_size=kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + bias=bias, + modulation=params.modulation, + aux_in_features=params.aux_in_features, + fc_hidden_features=params.fc_hidden_features, + fc_bias=params.fc_bias, + fc_groups=params.fc_groups, + fc_activation=params.fc_activation, + num_weights=params.num_weights, + ) + + +def mod_conv_transpose2d( + in_channels: int, + out_channels: int, + *, + kernel_size: IntOrTuple, + modulation_params: Optional[ModulationParams] = None, + stride: IntOrTuple = 1, + padding: IntOrTuple = 0, + dilation: IntOrTuple = 1, + bias: ModConv2dBias = ModConv2dBias.PARAM, +) -> ModConvTranspose2d: + """See :class:`ModConvTranspose2d`.""" + params = modulation_params or ModulationParams() + return ModConvTranspose2d( + in_channels, + out_channels, + kernel_size=kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + bias=bias, + modulation=params.modulation, + aux_in_features=params.aux_in_features, + fc_hidden_features=params.fc_hidden_features, + fc_bias=params.fc_bias, + fc_groups=params.fc_groups, + fc_activation=params.fc_activation, + num_weights=params.num_weights, + ) + + +def mod_conv3d( + in_channels: int, + out_channels: int, + *, + kernel_size: IntOrTuple, + modulation_params: Optional[ModulationParams] = None, + stride: IntOrTuple = 1, + padding: IntOrTuple = 0, + dilation: IntOrTuple = 1, + bias: ModConv2dBias = ModConv2dBias.PARAM, +) -> ModConv3d: + """See :class:`ModConv3d`.""" + params = modulation_params or ModulationParams() + return ModConv3d( + in_channels, + out_channels, + kernel_size=kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + bias=bias, + modulation=params.modulation, + aux_in_features=params.aux_in_features, + fc_hidden_features=params.fc_hidden_features, + fc_bias=params.fc_bias, + fc_groups=params.fc_groups, + fc_activation=params.fc_activation, + num_weights=params.num_weights, + ) + + +def mod_conv_transpose3d( + in_channels: int, + out_channels: int, + *, + kernel_size: IntOrTuple, + modulation_params: Optional[ModulationParams] = None, + stride: IntOrTuple = 1, + padding: IntOrTuple = 0, + dilation: IntOrTuple = 1, + bias: ModConv2dBias = ModConv2dBias.PARAM, +) -> ModConvTranspose3d: + """See :class:`ModConvTranspose3d`.""" + params = modulation_params or ModulationParams() + return ModConvTranspose3d( + in_channels, + out_channels, + kernel_size=kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + bias=bias, + modulation=params.modulation, + aux_in_features=params.aux_in_features, + fc_hidden_features=params.fc_hidden_features, + fc_bias=params.fc_bias, + fc_groups=params.fc_groups, + fc_activation=params.fc_activation, + num_weights=params.num_weights, + ) + + +@dataclass(frozen=True) +class ModulationParams: + """Shared modulation settings for modulated convolution layers.""" + + modulation: ModConvType = ModConvType.NONE + aux_in_features: Optional[int] = None + fc_hidden_features: Optional[int | tuple[int, ...]] = None + fc_groups: int = 1 + fc_activation: ModConvActivation = ModConvActivation.SIGMOID + num_weights: Optional[int] = None + fc_bias: Optional[bool] = True diff --git a/direct/nn/conv/modulated/modulated_conv.py b/direct/nn/conv/modulated/modulated_conv.py new file mode 100644 index 00000000..af26a45d --- /dev/null +++ b/direct/nn/conv/modulated/modulated_conv.py @@ -0,0 +1,1182 @@ +# Copyright 2026 AI for Oncology Research Group. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Modulated convolution layers based on [1]_. + +These layers extend standard convolutions with input-dependent weight modulation, +allowing the network to dynamically adjust its convolutional filters based on an +auxiliary signal (e.g., acceleration factor, coil information). + +References +---------- + +.. [1] Moriakov, N., Yiasemis, G., Sonke, J.-J. & Teuwen, J. (2026). Conditional Learned Reconstruction for + Medical Imaging. Proceedings of The 9th International Conference on Medical Imaging with Deep Learning, + PMLR 315:754-780. https://proceedings.mlr.press/v315/moriakov26a.html +""" + +from __future__ import annotations + +import math +from typing import Optional + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from direct.types import DirectEnum, IntOrTuple + +__all__ = [ + "ModConv2d", + "ModConv2dBias", + "ModConvActivation", + "ModConvType", + "ModConvTranspose2d", + "ModConv3d", + "ModConvTranspose3d", +] + + +class ModConv2dBias(DirectEnum): + LEARNED = "learned" + PARAM = "param" + NONE = "none" + + +class ModConvActivation(DirectEnum): + SIGMOID = "sigmoid" + SOFTPLUS = "softplus" + + +class ModConvType(DirectEnum): + FEATURES = "features" + PARTIAL_IN = "partial_in" + PARTIAL_OUT = "partial_out" + FULL = "full" + SUM = "sum" + NONE = "none" + + +def _as_hidden_tuple(features: int | tuple[int, ...]) -> tuple[int, ...]: + return (features,) if isinstance(features, int) else features + + +def _resolved_fc_bias(fc_bias: Optional[bool]) -> bool: + return True if fc_bias is None else fc_bias + + +def _build_modulation_mlp( + aux_in_features: int, + hidden_features: tuple[int, ...], + fc_activation: ModConvActivation, + fc_bias: bool, +) -> nn.Sequential: + layers: list[nn.Module] = [nn.Linear(aux_in_features, hidden_features[0], bias=fc_bias)] + for i in range(len(hidden_features) - 1): + layers.append(nn.PReLU()) + layers.append(nn.Linear(hidden_features[i], hidden_features[i + 1])) + if fc_activation == ModConvActivation.SIGMOID: + layers.append(nn.Sigmoid()) + elif fc_activation == ModConvActivation.SOFTPLUS: + layers.append(nn.Softplus()) + return nn.Sequential(*layers) + + +def _build_learned_bias_mlp( + aux_in_features: int, + hidden_features: tuple[int, ...], + out_channels: int, + fc_bias: bool, +) -> nn.Sequential: + bias_layers: list[nn.Module] = [nn.Linear(aux_in_features, hidden_features[0], bias=fc_bias)] + for i in range(len(hidden_features) - 1): + bias_layers.append(nn.PReLU()) + bias_layers.append( + nn.Linear( + hidden_features[i], + hidden_features[i + 1] if i != (len(hidden_features) - 2) else out_channels, + ) + ) + return nn.Sequential(*bias_layers) + + +class ModConv2d(nn.Module): + """Modulated 2D convolution based on [1]_. + + When ``modulation`` is :attr:`ModConvType.NONE` and ``bias`` is :attr:`ModConv2dBias.PARAM`, + this behaves identically to :class:`torch.nn.Conv2d`. + + When modulation is enabled, the convolutional weights are element-wise scaled by + an MLP-derived signal conditioned on an auxiliary input ``y``. + + References + ---------- + + .. [1] Moriakov, N., Yiasemis, G., Sonke, J.-J. & Teuwen, J. (2026). Conditional Learned Reconstruction for + Medical Imaging. Proceedings of The 9th International Conference on Medical Imaging with Deep Learning, + PMLR 315:754-780. https://proceedings.mlr.press/v315/moriakov26a.html + """ + + def __init__( + self, + in_channels: int, + out_channels: int, + kernel_size: IntOrTuple, + modulation: ModConvType = ModConvType.NONE, + stride: IntOrTuple = 1, + padding: IntOrTuple = 0, + dilation: IntOrTuple = 1, + bias: ModConv2dBias = ModConv2dBias.PARAM, + aux_in_features: Optional[int] = None, + fc_hidden_features: Optional[int | tuple[int, ...]] = None, + fc_bias: Optional[bool] = True, + fc_groups: int | None = 1, + fc_activation: ModConvActivation = ModConvActivation.SIGMOID, + num_weights: Optional[int] = None, + ): + """Inits :class:`ModConv2d`. + + Parameters + ---------- + in_channels : int + Number of input channels. + out_channels : int + Number of output channels. + kernel_size : int or tuple of int + Size of the convolutional kernel. + modulation : ModConvType + Modulation type. Default: ModConvType.NONE. + stride : int or tuple of int + Stride of the convolution. Default: 1. + padding : int or tuple of int + Padding added to all sides of the input. Default: 0. + dilation : int or tuple of int + Spacing between kernel elements. Default: 1. + bias : ModConv2dBias + Type of bias. Default: ModConv2dBias.PARAM. + aux_in_features : int, optional + Number of features in the auxiliary input ``y``. + fc_hidden_features : int or tuple of int, optional + Hidden features in the modulation MLP. + fc_bias : bool, optional + Whether the modulation MLP uses bias. Default: True. + fc_groups : int or None + If > 1, the MLP output is divided by fc_groups^2 and expanded via nearest interpolation. + Default: 1. + fc_activation : ModConvActivation + Activation after the MLP. Default: ModConvActivation.SIGMOID. + num_weights : int, optional + Number of weight bases for ModConvType.SUM. + """ + super().__init__() + + self.kernel_size = (kernel_size, kernel_size) if isinstance(kernel_size, int) else kernel_size + self.stride = stride + self.padding = padding + self.dilation = dilation + + self.in_channels = in_channels + self.out_channels = out_channels + + self.modulation = modulation + self.aux_in_features = aux_in_features + self.fc_hidden_features = fc_hidden_features + self.fc_bias = fc_bias + self.fc_groups = 1 if fc_groups is None else fc_groups + self.fc_activation = fc_activation + self.num_weights = num_weights + + mod_hidden: tuple[int, ...] | None = None + if modulation != ModConvType.NONE: + if aux_in_features is None: + raise ValueError("aux_in_features cannot be None when modulation is enabled.") + if fc_hidden_features is None: + raise ValueError("fc_hidden_features cannot be None when modulation is enabled.") + if fc_groups is None: + raise ValueError("fc_groups cannot be None when modulation is enabled.") + if fc_groups < 1: + raise ValueError("fc_groups must be >= 1.") + + hidden = _as_hidden_tuple(fc_hidden_features) + + if modulation == ModConvType.FEATURES: + mod_out_features = (out_channels // fc_groups) * (in_channels // fc_groups) + elif modulation == ModConvType.FULL: + mod_out_features = ( + (out_channels // fc_groups) * (in_channels // fc_groups) * self.kernel_size[0] * self.kernel_size[1] + ) + elif modulation == ModConvType.PARTIAL_OUT: + mod_out_features = self.kernel_size[0] * self.kernel_size[1] * (out_channels // fc_groups) + elif modulation == ModConvType.PARTIAL_IN: + mod_out_features = self.kernel_size[0] * self.kernel_size[1] * (in_channels // fc_groups) + else: + if (num_weights is None) or (num_weights < 1): + raise ValueError(f"ModConvType.SUM requires num_weights >= 1, got {num_weights}.") + mod_out_features = num_weights + + mod_hidden = hidden + (mod_out_features,) + self.fc = _build_modulation_mlp( + aux_in_features, + mod_hidden, + fc_activation, + _resolved_fc_bias(fc_bias), + ) + + weight_shape = (out_channels, in_channels, *self.kernel_size) + if modulation == ModConvType.SUM: + weight_shape = (num_weights,) + weight_shape + k = math.sqrt(1 / (in_channels * self.kernel_size[0] * self.kernel_size[1])) + self.weight = nn.Parameter(torch.FloatTensor(*weight_shape).uniform_(-k, k)) + + self.bias_type = bias + if bias == ModConv2dBias.PARAM: + self.bias = nn.Parameter(torch.FloatTensor(out_channels).uniform_(-k, k)) + elif bias == ModConv2dBias.LEARNED: + if mod_hidden is None or aux_in_features is None: + raise ValueError("ModConv2dBias.LEARNED requires modulation to be enabled.") + self.bias = _build_learned_bias_mlp( + aux_in_features, + mod_hidden, + out_channels, + _resolved_fc_bias(fc_bias), + ) + else: + self.bias = None + + def __repr__(self): + return ( + f"ModConv2d(in_channels={self.in_channels}, out_channels={self.out_channels}, " + f"kernel_size={self.kernel_size}, modulation={self.modulation}, " + f"stride={self.stride}, padding={self.padding}, " + f"dilation={self.dilation}, bias={self.bias_type})" + ) + + def forward(self, x: torch.Tensor, y: Optional[torch.Tensor] = None) -> torch.Tensor: + """Forward pass. + + Parameters + ---------- + x : torch.Tensor + Input of shape (N, in_channels, H, W). + y : torch.Tensor, optional + Auxiliary signal of shape (N, aux_in_features). + + Returns + ------- + torch.Tensor + Output of shape (N, out_channels, H_out, W_out). + """ + if self.modulation == ModConvType.NONE: + out = F.conv2d( + x, + self.weight, + stride=self.stride, + padding=self.padding, + dilation=self.dilation, + ) + else: + fc_out = self.fc(y) + + if self.modulation == ModConvType.SUM: + weight = (fc_out.view(x.shape[0], -1, 1, 1, 1, 1) * self.weight).sum(1) + else: + if self.modulation == ModConvType.FEATURES: + if self.fc_groups > 1: + fc_out = fc_out.view( + x.shape[0], + 1, + self.out_channels // self.fc_groups, + self.in_channels // self.fc_groups, + ) + fc_out = F.interpolate( + fc_out, + size=(self.out_channels, self.in_channels), + mode="nearest", + ) + fc_out = fc_out.view(x.shape[0], self.out_channels, self.in_channels, 1, 1) + + elif self.modulation == ModConvType.FULL: + if self.fc_groups > 1: + fc_out = fc_out.view( + x.shape[0], + self.kernel_size[0], + self.kernel_size[1], + self.out_channels // self.fc_groups, + self.in_channels // self.fc_groups, + ) + fc_out = F.interpolate( + fc_out, + size=( + self.kernel_size[1], + self.out_channels, + self.in_channels, + ), + mode="nearest", + ) + fc_out = fc_out.permute(0, 3, 4, 1, 2) + fc_out = fc_out.view( + x.shape[0], + self.out_channels, + self.in_channels, + self.kernel_size[0], + self.kernel_size[1], + ) + + elif self.modulation == ModConvType.PARTIAL_OUT: + if self.fc_groups > 1: + fc_out = fc_out.view( + x.shape[0], + self.kernel_size[0], + self.kernel_size[1], + self.out_channels // self.fc_groups, + 1, + ) + fc_out = F.interpolate( + fc_out, + size=(self.kernel_size[1], self.out_channels, 1), + mode="nearest", + ) + fc_out = fc_out.permute(0, 3, 4, 1, 2) + fc_out = fc_out.view( + x.shape[0], + self.out_channels, + 1, + self.kernel_size[0], + self.kernel_size[1], + ) + + else: # PARTIAL_IN + if self.fc_groups > 1: + fc_out = fc_out.view( + x.shape[0], + self.kernel_size[0], + self.kernel_size[1], + 1, + self.in_channels // self.fc_groups, + ) + fc_out = F.interpolate( + fc_out, + size=(self.kernel_size[1], 1, self.in_channels), + mode="nearest", + ) + fc_out = fc_out.permute(0, 3, 4, 1, 2) + fc_out = fc_out.view( + x.shape[0], + 1, + self.in_channels, + self.kernel_size[0], + self.kernel_size[1], + ) + + weight = fc_out * self.weight + + out = torch.cat( + [ + F.conv2d( + x[i : i + 1], + weight[i], + stride=self.stride, + padding=self.padding, + dilation=self.dilation, + ) + for i in range(x.shape[0]) + ], + 0, + ) + + if self.bias is not None: + if isinstance(self.bias, nn.parameter.Parameter): + bias = self.bias.view(1, -1, 1, 1) + else: + bias = self.bias(y).view(x.shape[0], -1, 1, 1) + out = out + bias + + return out + + +class ModConvTranspose2d(nn.Module): + """Modulated 2D transposed convolution. + + Transpose variant of :class:`ModConv2d` supporting the same modulation types. + """ + + def __init__( + self, + in_channels: int, + out_channels: int, + kernel_size: IntOrTuple, + modulation: ModConvType = ModConvType.NONE, + stride: IntOrTuple = 1, + padding: IntOrTuple = 0, + dilation: IntOrTuple = 1, + bias: ModConv2dBias = ModConv2dBias.PARAM, + aux_in_features: Optional[int] = None, + fc_hidden_features: Optional[int | tuple[int, ...]] = None, + fc_bias: Optional[bool] = True, + fc_groups: int | None = 1, + fc_activation: ModConvActivation = ModConvActivation.SIGMOID, + num_weights: Optional[int] = None, + ): + super().__init__() + + self.kernel_size = (kernel_size, kernel_size) if isinstance(kernel_size, int) else kernel_size + self.stride = stride + self.padding = padding + self.dilation = dilation + + self.in_channels = in_channels + self.out_channels = out_channels + + self.modulation = modulation + self.aux_in_features = aux_in_features + self.fc_hidden_features = fc_hidden_features + self.fc_bias = fc_bias + self.fc_groups = 1 if fc_groups is None else fc_groups + self.fc_activation = fc_activation + self.num_weights = num_weights + + mod_hidden: tuple[int, ...] | None = None + if modulation != ModConvType.NONE: + if aux_in_features is None: + raise ValueError("aux_in_features cannot be None when modulation is enabled.") + if fc_hidden_features is None: + raise ValueError("fc_hidden_features cannot be None when modulation is enabled.") + if fc_groups is None: + raise ValueError("fc_groups cannot be None when modulation is enabled.") + if fc_groups < 1: + raise ValueError("fc_groups must be >= 1.") + + hidden = _as_hidden_tuple(fc_hidden_features) + + if modulation == ModConvType.FEATURES: + mod_out_features = (in_channels // fc_groups) * (out_channels // fc_groups) + elif modulation == ModConvType.FULL: + mod_out_features = ( + (in_channels // fc_groups) * (out_channels // fc_groups) * self.kernel_size[0] * self.kernel_size[1] + ) + elif modulation == ModConvType.PARTIAL_OUT: + mod_out_features = self.kernel_size[0] * self.kernel_size[1] * (out_channels // fc_groups) + elif modulation == ModConvType.PARTIAL_IN: + mod_out_features = self.kernel_size[0] * self.kernel_size[1] * (in_channels // fc_groups) + else: + if (num_weights is None) or (num_weights < 1): + raise ValueError(f"ModConvType.SUM requires num_weights >= 1, got {num_weights}.") + mod_out_features = num_weights + + mod_hidden = hidden + (mod_out_features,) + self.fc = _build_modulation_mlp( + aux_in_features, + mod_hidden, + fc_activation, + _resolved_fc_bias(fc_bias), + ) + + weight_shape = (in_channels, out_channels, *self.kernel_size) + if modulation == ModConvType.SUM: + weight_shape = (num_weights,) + weight_shape + k = math.sqrt(1 / (out_channels * self.kernel_size[0] * self.kernel_size[1])) + self.weight = nn.Parameter(torch.FloatTensor(*weight_shape).uniform_(-k, k)) + + self.bias_type = bias + if bias == ModConv2dBias.PARAM: + self.bias = nn.Parameter(torch.FloatTensor(out_channels).uniform_(-k, k)) + elif bias == ModConv2dBias.LEARNED: + if mod_hidden is None or aux_in_features is None: + raise ValueError("ModConv2dBias.LEARNED requires modulation to be enabled.") + self.bias = _build_learned_bias_mlp( + aux_in_features, + mod_hidden, + out_channels, + _resolved_fc_bias(fc_bias), + ) + else: + self.bias = None + + def __repr__(self): + return ( + f"ModConvTranspose2d(in_channels={self.in_channels}, out_channels={self.out_channels}, " + f"kernel_size={self.kernel_size}, modulation={self.modulation}, " + f"stride={self.stride}, padding={self.padding}, dilation={self.dilation}, bias={self.bias_type})" + ) + + def forward(self, x: torch.Tensor, y: Optional[torch.Tensor] = None) -> torch.Tensor: + """Forward pass. + + Parameters + ---------- + x : torch.Tensor + Input of shape (N, in_channels, H, W). + y : torch.Tensor, optional + Auxiliary signal of shape (N, aux_in_features). + + Returns + ------- + torch.Tensor + Output of shape (N, out_channels, H_out, W_out). + """ + if self.modulation == ModConvType.NONE: + out = F.conv_transpose2d( + x, + self.weight, + stride=self.stride, + padding=self.padding, + dilation=self.dilation, + ) + else: + fc_out = self.fc(y) + + if self.modulation == ModConvType.SUM: + weight = (fc_out.view(x.shape[0], -1, 1, 1, 1, 1) * self.weight).sum(1) + else: + if self.modulation == ModConvType.FEATURES: + if self.fc_groups > 1: + fc_out = fc_out.view( + x.shape[0], + 1, + self.in_channels // self.fc_groups, + self.out_channels // self.fc_groups, + ) + fc_out = F.interpolate( + fc_out, + size=(self.in_channels, self.out_channels), + mode="nearest", + ) + fc_out = fc_out.view(x.shape[0], self.in_channels, self.out_channels, 1, 1) + + elif self.modulation == ModConvType.FULL: + if self.fc_groups > 1: + fc_out = fc_out.view( + x.shape[0], + self.kernel_size[0], + self.kernel_size[1], + self.in_channels // self.fc_groups, + self.out_channels // self.fc_groups, + ) + fc_out = F.interpolate( + fc_out, + size=( + self.kernel_size[1], + self.in_channels, + self.out_channels, + ), + mode="nearest", + ) + fc_out = fc_out.permute(0, 3, 4, 1, 2) + fc_out = fc_out.view( + x.shape[0], + self.in_channels, + self.out_channels, + self.kernel_size[0], + self.kernel_size[1], + ) + + elif self.modulation == ModConvType.PARTIAL_OUT: + if self.fc_groups > 1: + fc_out = fc_out.view( + x.shape[0], + self.kernel_size[0], + self.kernel_size[1], + 1, + self.out_channels // self.fc_groups, + ) + fc_out = F.interpolate( + fc_out, + size=(self.kernel_size[1], 1, self.out_channels), + mode="nearest", + ) + fc_out = fc_out.permute(0, 3, 4, 1, 2) + fc_out = fc_out.view( + x.shape[0], + 1, + self.out_channels, + self.kernel_size[0], + self.kernel_size[1], + ) + + else: # PARTIAL_IN + if self.fc_groups > 1: + fc_out = fc_out.view( + x.shape[0], + self.kernel_size[0], + self.kernel_size[1], + self.in_channels // self.fc_groups, + 1, + ) + fc_out = F.interpolate( + fc_out, + size=(self.kernel_size[1], self.in_channels, 1), + mode="nearest", + ) + fc_out = fc_out.permute(0, 3, 4, 1, 2) + fc_out = fc_out.view( + x.shape[0], + self.in_channels, + 1, + self.kernel_size[0], + self.kernel_size[1], + ) + + weight = fc_out * self.weight + + out = torch.cat( + [ + F.conv_transpose2d( + x[i : i + 1], + weight[i], + stride=self.stride, + padding=self.padding, + dilation=self.dilation, + ) + for i in range(x.shape[0]) + ], + 0, + ) + + if self.bias is not None: + if isinstance(self.bias, nn.parameter.Parameter): + bias = self.bias.view(1, -1, 1, 1) + else: + bias = self.bias(y).view(x.shape[0], -1, 1, 1) + out = out + bias + + return out + + +class ModConv3d(nn.Module): + """Modulated 3D convolution. + + 3D extension of :class:`ModConv2d` supporting the same modulation types. + """ + + def __init__( + self, + in_channels: int, + out_channels: int, + kernel_size: IntOrTuple, + modulation: ModConvType = ModConvType.NONE, + stride: IntOrTuple = 1, + padding: IntOrTuple = 0, + dilation: IntOrTuple = 1, + bias: ModConv2dBias = ModConv2dBias.PARAM, + aux_in_features: Optional[int] = None, + fc_hidden_features: Optional[int | tuple[int, ...]] = None, + fc_bias: Optional[bool] = True, + fc_groups: int | None = 1, + fc_activation: ModConvActivation = ModConvActivation.SIGMOID, + num_weights: Optional[int] = None, + ): + super().__init__() + + if isinstance(kernel_size, int): + self.kernel_size = (kernel_size, kernel_size, kernel_size) + elif len(kernel_size) == 2: + self.kernel_size = (kernel_size[0], kernel_size[0], kernel_size[1]) + else: + self.kernel_size = kernel_size + self.stride = stride + self.padding = padding + self.dilation = dilation + + self.in_channels = in_channels + self.out_channels = out_channels + + self.modulation = modulation + self.aux_in_features = aux_in_features + self.fc_hidden_features = fc_hidden_features + self.fc_bias = fc_bias + self.fc_groups = 1 if fc_groups is None else fc_groups + self.fc_activation = fc_activation + self.num_weights = num_weights + + mod_hidden: tuple[int, ...] | None = None + if modulation != ModConvType.NONE: + if aux_in_features is None: + raise ValueError("aux_in_features cannot be None when modulation is enabled.") + if fc_hidden_features is None: + raise ValueError("fc_hidden_features cannot be None when modulation is enabled.") + if fc_groups is None: + raise ValueError("fc_groups cannot be None when modulation is enabled.") + if fc_groups < 1: + raise ValueError("fc_groups must be >= 1.") + + hidden = _as_hidden_tuple(fc_hidden_features) + + if modulation == ModConvType.FEATURES: + mod_out_features = (out_channels // fc_groups) * (in_channels // fc_groups) + elif modulation == ModConvType.FULL: + mod_out_features = ( + (out_channels // fc_groups) + * (in_channels // fc_groups) + * self.kernel_size[0] + * self.kernel_size[1] + * self.kernel_size[2] + ) + elif modulation == ModConvType.PARTIAL_OUT: + mod_out_features = ( + self.kernel_size[0] * self.kernel_size[1] * self.kernel_size[2] * (out_channels // fc_groups) + ) + elif modulation == ModConvType.PARTIAL_IN: + mod_out_features = ( + self.kernel_size[0] * self.kernel_size[1] * self.kernel_size[2] * (in_channels // fc_groups) + ) + else: + if (num_weights is None) or (num_weights < 1): + raise ValueError(f"ModConvType.SUM requires num_weights >= 1, got {num_weights}.") + mod_out_features = num_weights + + mod_hidden = hidden + (mod_out_features,) + self.fc = _build_modulation_mlp( + aux_in_features, + mod_hidden, + fc_activation, + _resolved_fc_bias(fc_bias), + ) + + weight_shape = (out_channels, in_channels, *self.kernel_size) + if modulation == ModConvType.SUM: + weight_shape = (num_weights,) + weight_shape + k = math.sqrt(1 / (in_channels * self.kernel_size[0] * self.kernel_size[1] * self.kernel_size[2])) + self.weight = nn.Parameter(torch.FloatTensor(*weight_shape).uniform_(-k, k)) + + self.bias_type = bias + if bias == ModConv2dBias.PARAM: + self.bias = nn.Parameter(torch.FloatTensor(out_channels).uniform_(-k, k)) + elif bias == ModConv2dBias.LEARNED: + if mod_hidden is None or aux_in_features is None: + raise ValueError("ModConv2dBias.LEARNED requires modulation to be enabled.") + self.bias = _build_learned_bias_mlp( + aux_in_features, + mod_hidden, + out_channels, + _resolved_fc_bias(fc_bias), + ) + else: + self.bias = None + + def __repr__(self): + return ( + f"ModConv3d(in_channels={self.in_channels}, out_channels={self.out_channels}, " + f"kernel_size={self.kernel_size}, modulation={self.modulation}, " + f"stride={self.stride}, padding={self.padding}, dilation={self.dilation}, bias={self.bias_type})" + ) + + def forward(self, x: torch.Tensor, y: Optional[torch.Tensor] = None) -> torch.Tensor: + """Forward pass. + + Parameters + ---------- + x : torch.Tensor + Input of shape (N, in_channels, D, H, W). + y : torch.Tensor, optional + Auxiliary signal of shape (N, aux_in_features). + + Returns + ------- + torch.Tensor + Output of shape (N, out_channels, D_out, H_out, W_out). + """ + if self.modulation == ModConvType.NONE: + out = F.conv3d( + x, + self.weight, + stride=self.stride, + padding=self.padding, + dilation=self.dilation, + ) + else: + fc_out = self.fc(y) + + if self.modulation == ModConvType.SUM: + weight = (fc_out.view(x.shape[0], -1, 1, 1, 1, 1, 1) * self.weight).sum(1) + else: + if self.modulation == ModConvType.FEATURES: + if self.fc_groups > 1: + fc_out = fc_out.view( + x.shape[0], + 1, + self.out_channels // self.fc_groups, + self.in_channels // self.fc_groups, + ) + fc_out = F.interpolate( + fc_out, + size=(self.out_channels, self.in_channels), + mode="nearest", + ) + fc_out = fc_out.view(x.shape[0], self.out_channels, self.in_channels, 1, 1, 1) + + elif self.modulation == ModConvType.FULL: + if self.fc_groups > 1: + fc_out = fc_out.view( + x.shape[0], + self.kernel_size[0], + self.kernel_size[1], + self.kernel_size[2], + self.out_channels // self.fc_groups, + self.in_channels // self.fc_groups, + ) + fc_out = fc_out.permute(0, 4, 5, 1, 2, 3) + out_ch_expand = self.out_channels // self.fc_groups + in_ch_expand = self.in_channels // self.fc_groups + if out_ch_expand < self.out_channels or in_ch_expand < self.in_channels: + fc_out = fc_out.repeat(1, self.fc_groups, self.fc_groups, 1, 1, 1) + fc_out = fc_out[:, : self.out_channels, : self.in_channels, :, :, :] + fc_out = fc_out.view( + x.shape[0], + self.out_channels, + self.in_channels, + self.kernel_size[0], + self.kernel_size[1], + self.kernel_size[2], + ) + else: + fc_out = fc_out.view( + x.shape[0], + self.out_channels, + self.in_channels, + self.kernel_size[0], + self.kernel_size[1], + self.kernel_size[2], + ) + + elif self.modulation == ModConvType.PARTIAL_OUT: + if self.fc_groups > 1: + fc_out = fc_out.view( + x.shape[0], + self.kernel_size[0], + self.kernel_size[1], + self.kernel_size[2], + self.out_channels // self.fc_groups, + 1, + ) + fc_out = fc_out.permute(0, 4, 5, 1, 2, 3) + fc_out = fc_out.view( + x.shape[0], + self.out_channels, + 1, + self.kernel_size[0], + self.kernel_size[1], + self.kernel_size[2], + ) + + else: # PARTIAL_IN + if self.fc_groups > 1: + fc_out = fc_out.view( + x.shape[0], + self.kernel_size[0], + self.kernel_size[1], + self.kernel_size[2], + 1, + self.in_channels // self.fc_groups, + ) + fc_out = fc_out.permute(0, 4, 5, 1, 2, 3) + fc_out = fc_out.view( + x.shape[0], + 1, + self.in_channels, + self.kernel_size[0], + self.kernel_size[1], + self.kernel_size[2], + ) + + weight = fc_out * self.weight + + out = torch.cat( + [ + F.conv3d( + x[i : i + 1], + weight[i], + stride=self.stride, + padding=self.padding, + dilation=self.dilation, + ) + for i in range(x.shape[0]) + ], + 0, + ) + + if self.bias is not None: + if isinstance(self.bias, nn.parameter.Parameter): + bias = self.bias.view(1, -1, 1, 1, 1) + else: + bias = self.bias(y).view(x.shape[0], -1, 1, 1, 1) + out = out + bias + + return out + + +class ModConvTranspose3d(nn.Module): + """Modulated 3D transposed convolution. + + 3D transpose variant of :class:`ModConv3d`. + """ + + def __init__( + self, + in_channels: int, + out_channels: int, + kernel_size: IntOrTuple, + modulation: ModConvType = ModConvType.NONE, + stride: IntOrTuple = 1, + padding: IntOrTuple = 0, + dilation: IntOrTuple = 1, + bias: ModConv2dBias = ModConv2dBias.PARAM, + aux_in_features: Optional[int] = None, + fc_hidden_features: Optional[int | tuple[int, ...]] = None, + fc_bias: Optional[bool] = True, + fc_groups: int | None = 1, + fc_activation: ModConvActivation = ModConvActivation.SIGMOID, + num_weights: Optional[int] = None, + ): + super().__init__() + + if isinstance(kernel_size, int): + self.kernel_size = (kernel_size, kernel_size, kernel_size) + elif len(kernel_size) == 2: + self.kernel_size = (kernel_size[0], kernel_size[0], kernel_size[1]) + else: + self.kernel_size = kernel_size + self.stride = stride + self.padding = padding + self.dilation = dilation + + self.in_channels = in_channels + self.out_channels = out_channels + + self.modulation = modulation + self.aux_in_features = aux_in_features + self.fc_hidden_features = fc_hidden_features + self.fc_bias = fc_bias + self.fc_groups = 1 if fc_groups is None else fc_groups + self.fc_activation = fc_activation + self.num_weights = num_weights + + mod_hidden: tuple[int, ...] | None = None + if modulation != ModConvType.NONE: + if aux_in_features is None: + raise ValueError("aux_in_features cannot be None when modulation is enabled.") + if fc_hidden_features is None: + raise ValueError("fc_hidden_features cannot be None when modulation is enabled.") + if fc_groups is None: + raise ValueError("fc_groups cannot be None when modulation is enabled.") + if fc_groups < 1: + raise ValueError("fc_groups must be >= 1.") + + hidden = _as_hidden_tuple(fc_hidden_features) + + if modulation == ModConvType.FEATURES: + mod_out_features = (in_channels // fc_groups) * (out_channels // fc_groups) + elif modulation == ModConvType.FULL: + mod_out_features = ( + (in_channels // fc_groups) + * (out_channels // fc_groups) + * self.kernel_size[0] + * self.kernel_size[1] + * self.kernel_size[2] + ) + elif modulation == ModConvType.PARTIAL_OUT: + mod_out_features = ( + self.kernel_size[0] * self.kernel_size[1] * self.kernel_size[2] * (out_channels // fc_groups) + ) + elif modulation == ModConvType.PARTIAL_IN: + mod_out_features = ( + self.kernel_size[0] * self.kernel_size[1] * self.kernel_size[2] * (in_channels // fc_groups) + ) + else: + if (num_weights is None) or (num_weights < 1): + raise ValueError(f"ModConvType.SUM requires num_weights >= 1, got {num_weights}.") + mod_out_features = num_weights + + mod_hidden = hidden + (mod_out_features,) + self.fc = _build_modulation_mlp( + aux_in_features, + mod_hidden, + fc_activation, + _resolved_fc_bias(fc_bias), + ) + + weight_shape = (in_channels, out_channels, *self.kernel_size) + if modulation == ModConvType.SUM: + weight_shape = (num_weights,) + weight_shape + k = math.sqrt(1 / (out_channels * self.kernel_size[0] * self.kernel_size[1] * self.kernel_size[2])) + self.weight = nn.Parameter(torch.FloatTensor(*weight_shape).uniform_(-k, k)) + + self.bias_type = bias + if bias == ModConv2dBias.PARAM: + self.bias = nn.Parameter(torch.FloatTensor(out_channels).uniform_(-k, k)) + elif bias == ModConv2dBias.LEARNED: + if mod_hidden is None or aux_in_features is None: + raise ValueError("ModConv2dBias.LEARNED requires modulation to be enabled.") + self.bias = _build_learned_bias_mlp( + aux_in_features, + mod_hidden, + out_channels, + _resolved_fc_bias(fc_bias), + ) + else: + self.bias = None + + def __repr__(self): + return ( + f"ModConvTranspose3d(in_channels={self.in_channels}, out_channels={self.out_channels}, " + f"kernel_size={self.kernel_size}, modulation={self.modulation}, " + f"stride={self.stride}, padding={self.padding}, dilation={self.dilation}, bias={self.bias_type})" + ) + + def forward(self, x: torch.Tensor, y: Optional[torch.Tensor] = None) -> torch.Tensor: + """Forward pass. + + Parameters + ---------- + x : torch.Tensor + Input of shape (N, in_channels, D, H, W). + y : torch.Tensor, optional + Auxiliary signal of shape (N, aux_in_features). + + Returns + ------- + torch.Tensor + Output of shape (N, out_channels, D_out, H_out, W_out). + """ + if self.modulation == ModConvType.NONE: + out = F.conv_transpose3d( + x, + self.weight, + stride=self.stride, + padding=self.padding, + dilation=self.dilation, + ) + else: + fc_out = self.fc(y) + + if self.modulation == ModConvType.SUM: + weight = (fc_out.view(x.shape[0], -1, 1, 1, 1, 1, 1) * self.weight).sum(1) + else: + if self.modulation == ModConvType.FEATURES: + if self.fc_groups > 1: + fc_out = fc_out.view( + x.shape[0], + 1, + self.in_channels // self.fc_groups, + self.out_channels // self.fc_groups, + ) + fc_out = F.interpolate( + fc_out, + size=(self.in_channels, self.out_channels), + mode="nearest", + ) + fc_out = fc_out.view(x.shape[0], self.in_channels, self.out_channels, 1, 1, 1) + + elif self.modulation == ModConvType.FULL: + if self.fc_groups > 1: + fc_out = fc_out.view( + x.shape[0], + self.kernel_size[0], + self.kernel_size[1], + self.kernel_size[2], + self.in_channels // self.fc_groups, + self.out_channels // self.fc_groups, + ) + fc_out = fc_out.permute(0, 4, 5, 1, 2, 3) + in_ch_expand = self.in_channels // self.fc_groups + out_ch_expand = self.out_channels // self.fc_groups + if out_ch_expand < self.out_channels or in_ch_expand < self.in_channels: + fc_out = fc_out.repeat(1, self.fc_groups, self.fc_groups, 1, 1, 1) + fc_out = fc_out[:, : self.in_channels, : self.out_channels, :, :, :] + fc_out = fc_out.view( + x.shape[0], + self.in_channels, + self.out_channels, + self.kernel_size[0], + self.kernel_size[1], + self.kernel_size[2], + ) + else: + fc_out = fc_out.view( + x.shape[0], + self.in_channels, + self.out_channels, + self.kernel_size[0], + self.kernel_size[1], + self.kernel_size[2], + ) + + elif self.modulation == ModConvType.PARTIAL_OUT: + if self.fc_groups > 1: + fc_out = fc_out.view( + x.shape[0], + self.kernel_size[0], + self.kernel_size[1], + self.kernel_size[2], + 1, + self.out_channels // self.fc_groups, + ) + fc_out = fc_out.permute(0, 4, 5, 1, 2, 3) + if self.out_channels // self.fc_groups < self.out_channels: + fc_out = fc_out.repeat(1, 1, self.fc_groups, 1, 1, 1) + fc_out = fc_out[:, :, : self.out_channels, :, :, :] + fc_out = fc_out.view( + x.shape[0], + 1, + self.out_channels, + self.kernel_size[0], + self.kernel_size[1], + self.kernel_size[2], + ) + + else: # PARTIAL_IN + if self.fc_groups > 1: + fc_out = fc_out.view( + x.shape[0], + self.kernel_size[0], + self.kernel_size[1], + self.kernel_size[2], + self.in_channels // self.fc_groups, + 1, + ) + fc_out = fc_out.permute(0, 4, 5, 1, 2, 3) + if self.in_channels // self.fc_groups < self.in_channels: + fc_out = fc_out.repeat(1, self.fc_groups, 1, 1, 1, 1) + fc_out = fc_out[:, : self.in_channels, :, :, :, :] + fc_out = fc_out.view( + x.shape[0], + self.in_channels, + 1, + self.kernel_size[0], + self.kernel_size[1], + self.kernel_size[2], + ) + + weight = fc_out * self.weight + + out = torch.cat( + [ + F.conv_transpose3d( + x[i : i + 1], + weight[i], + stride=self.stride, + padding=self.padding, + dilation=self.dilation, + ) + for i in range(x.shape[0]) + ], + 0, + ) + + if self.bias is not None: + if isinstance(self.bias, nn.parameter.Parameter): + bias = self.bias.view(1, -1, 1, 1, 1) + else: + bias = self.bias(y).view(x.shape[0], -1, 1, 1, 1) + out = out + bias + + return out diff --git a/direct/nn/crossdomain/crossdomain.py b/direct/nn/crossdomain/crossdomain.py index 0ca0fd5d..f977ff4f 100644 --- a/direct/nn/crossdomain/crossdomain.py +++ b/direct/nn/crossdomain/crossdomain.py @@ -17,6 +17,7 @@ import torch.nn as nn import direct.data.transforms as T +from direct.nn.conv.modulated import ModConvType from direct.types import FFTOperator @@ -33,6 +34,7 @@ def __init__( image_buffer_size: int = 1, kspace_buffer_size: int = 1, normalize_image: bool = False, + conv_modulation: ModConvType = ModConvType.NONE, **kwargs, ): """Inits CrossDomainNetwork. @@ -81,6 +83,7 @@ def __init__( self.image_buffer_size = image_buffer_size self.normalize_image = normalize_image + self.conv_modulation = conv_modulation self._coil_dim = 1 self._complex_dim = -1 @@ -94,6 +97,7 @@ def kspace_correction( sampling_mask: torch.Tensor, sensitivity_map: torch.Tensor, masked_kspace: torch.Tensor, + auxiliary_data: Optional[torch.Tensor] = None, ) -> torch.Tensor: forward_buffer = torch.cat( [ @@ -109,9 +113,11 @@ def kspace_correction( kspace_buffer = torch.cat([kspace_buffer, forward_buffer, masked_kspace], self._complex_dim) if self.kspace_model_list is not None: - kspace_buffer = self.kspace_model_list[block_idx](kspace_buffer.permute(0, 1, 4, 2, 3)).permute( - 0, 1, 3, 4, 2 - ) + kspace_input = kspace_buffer.permute(0, 1, 4, 2, 3) + if self.conv_modulation != ModConvType.NONE: + kspace_buffer = self.kspace_model_list[block_idx](kspace_input, auxiliary_data).permute(0, 1, 3, 4, 2) + else: + kspace_buffer = self.kspace_model_list[block_idx](kspace_input).permute(0, 1, 3, 4, 2) else: kspace_buffer = kspace_buffer[..., :2] - kspace_buffer[..., 2:4] @@ -124,6 +130,7 @@ def image_correction( kspace_buffer: torch.Tensor, sampling_mask: torch.Tensor, sensitivity_map: torch.Tensor, + auxiliary_data: Optional[torch.Tensor] = None, ) -> torch.Tensor: backward_buffer = torch.cat( [ @@ -134,22 +141,34 @@ def image_correction( ) image_buffer = torch.cat([image_buffer, backward_buffer], self._complex_dim).permute(0, 3, 1, 2) - image_buffer = self.image_model_list[block_idx](image_buffer).permute(0, 2, 3, 1) + if self.conv_modulation != ModConvType.NONE: + image_buffer = self.image_model_list[block_idx](image_buffer, auxiliary_data).permute(0, 2, 3, 1) + else: + image_buffer = self.image_model_list[block_idx](image_buffer).permute(0, 2, 3, 1) return image_buffer def _forward_operator( - self, image: torch.Tensor, sampling_mask: torch.Tensor, sensitivity_map: torch.Tensor + self, + image: torch.Tensor, + sampling_mask: torch.Tensor, + sensitivity_map: torch.Tensor, ) -> torch.Tensor: forward = torch.where( sampling_mask == 0, torch.tensor([0.0], dtype=image.dtype).to(image.device), - self.forward_operator(T.expand_operator(image, sensitivity_map, self._coil_dim), dim=self._spatial_dims), + self.forward_operator( + T.expand_operator(image, sensitivity_map, self._coil_dim), + dim=self._spatial_dims, + ), ) return forward def _backward_operator( - self, kspace: torch.Tensor, sampling_mask: torch.Tensor, sensitivity_map: torch.Tensor + self, + kspace: torch.Tensor, + sampling_mask: torch.Tensor, + sensitivity_map: torch.Tensor, ) -> torch.Tensor: backward = T.reduce_operator( self.backward_operator( @@ -171,6 +190,7 @@ def forward( sampling_mask: torch.Tensor, sensitivity_map: torch.Tensor, scaling_factor: Optional[torch.Tensor] = None, + auxiliary_data: Optional[torch.Tensor] = None, ) -> torch.Tensor: """Computes the forward pass of :class:`CrossDomainNetwork`. @@ -204,12 +224,23 @@ def forward( for block_domain in self.domain_sequence: if block_domain == "K": kspace_buffer = self.kspace_correction( - kspace_block_idx, image_buffer, kspace_buffer, sampling_mask, sensitivity_map, masked_kspace + kspace_block_idx, + image_buffer, + kspace_buffer, + sampling_mask, + sensitivity_map, + masked_kspace, + auxiliary_data, ) kspace_block_idx += 1 else: image_buffer = self.image_correction( - image_block_idx, image_buffer, kspace_buffer, sampling_mask, sensitivity_map + image_block_idx, + image_buffer, + kspace_buffer, + sampling_mask, + sensitivity_map, + auxiliary_data, ) image_block_idx += 1 diff --git a/direct/nn/crossdomain/multicoil.py b/direct/nn/crossdomain/multicoil.py index dc86e37b..bd6cdc9a 100644 --- a/direct/nn/crossdomain/multicoil.py +++ b/direct/nn/crossdomain/multicoil.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from typing import Optional + import torch import torch.nn as nn @@ -41,16 +43,19 @@ def __init__(self, model: nn.Module, coil_dim: int = 1, coil_to_batch: bool = Fa self.coil_to_batch = coil_to_batch self._coil_dim = coil_dim - def _compute_model_per_coil(self, data: torch.Tensor) -> torch.Tensor: + def _compute_model_per_coil(self, data: torch.Tensor, y: Optional[torch.Tensor] = None) -> torch.Tensor: output = [] for idx in range(data.size(self._coil_dim)): subselected_data = data.select(self._coil_dim, idx) - output.append(self.model(subselected_data)) + if y is not None: + output.append(self.model(subselected_data, y)) + else: + output.append(self.model(subselected_data)) return torch.stack(output, dim=self._coil_dim) - def forward(self, x: torch.Tensor) -> torch.Tensor: + def forward(self, x: torch.Tensor, y: Optional[torch.Tensor] = None) -> torch.Tensor: """Performs the forward pass of MultiCoil. Parameters @@ -68,9 +73,12 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: batch, coil, height, width, channels = x.size() x = x.reshape(batch * coil, height, width, channels).permute(0, 3, 1, 2).contiguous() - x = self.model(x).permute(0, 2, 3, 1) + if y is not None: + x = self.model(x, y).permute(0, 2, 3, 1) + else: + x = self.model(x).permute(0, 2, 3, 1) x = x.reshape(batch, coil, height, width, -1) else: - x = self._compute_model_per_coil(x).contiguous() + x = self._compute_model_per_coil(x, y).contiguous() return x diff --git a/direct/nn/didn/didn.py b/direct/nn/didn/didn.py index 6d6495af..5f894598 100644 --- a/direct/nn/didn/didn.py +++ b/direct/nn/didn/didn.py @@ -11,12 +11,23 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Tuple, Union +from __future__ import annotations + +from typing import Optional, Tuple, Union import torch import torch.nn as nn import torch.nn.functional as F +from direct.nn.conv.modulated import ( + ModConv2d, + ModConv2dBias, + ModConvActivation, + ModConvType, + ModulationParams, + mod_conv2d, +) + class Subpixel(nn.Module): """Subpixel convolution layer for up-scaling of low resolution features at super-resolution as implemented in [1]_. @@ -24,7 +35,8 @@ class Subpixel(nn.Module): References ---------- - .. [1] Yu, Songhyun, et al. “Deep Iterative Down-Up CNN for Image Denoising.” 2019 IEEE/CVF Conference on Computer Vision and Pattern Recognition Workshops (CVPRW), 2019, pp. 2095–103. IEEE Xplore, https://doi.org/10.1109/CVPRW.2019.00262. + .. [1] Yu, Songhyun, et al. "Deep Iterative Down-Up CNN for Image Denoising." + CVPRW, 2019. https://doi.org/10.1109/CVPRW.2019.00262. """ def __init__( @@ -34,6 +46,13 @@ def __init__( upscale_factor: int, kernel_size: Union[int, Tuple[int, int]], padding: int = 0, + modulation: ModConvType = ModConvType.NONE, + modulation_params: Optional[ModulationParams] = None, + aux_in_features: Optional[int] = None, + fc_hidden_features: Optional[tuple[int] | int] = None, + fc_groups: int = 1, + fc_activation: ModConvActivation = ModConvActivation.SIGMOID, + num_weights: Optional[int] = None, ): """Inits :class:`Subpixel`. @@ -49,19 +68,52 @@ def __init__( Convolution kernel size. padding: int Padding size. Default: 0. + modulation : ModConvType + Modulation type. Default: ModConvType.NONE. + aux_in_features : int, optional + Auxiliary input features for modulation. + fc_hidden_features : int or tuple of int, optional + Hidden features for modulation MLP. + fc_groups : int + Groups for modulation MLP. Default: 1. + fc_activation : ModConvActivation + Activation for modulation MLP. Default: ModConvActivation.SIGMOID. + num_weights : int, optional + Number of weight bases for ModConvType.SUM. """ super().__init__() - self.conv = nn.Conv2d(in_channels, out_channels * upscale_factor**2, kernel_size=kernel_size, padding=padding) + if modulation_params is None: + modulation_params = ModulationParams( + modulation=modulation, + aux_in_features=aux_in_features, + fc_hidden_features=fc_hidden_features, + fc_groups=fc_groups, + fc_activation=fc_activation, + num_weights=num_weights, + ) + self.modulation = modulation_params.modulation + self.conv = mod_conv2d( + in_channels, + out_channels * upscale_factor**2, + kernel_size=kernel_size, + padding=padding, + bias=ModConv2dBias.PARAM, + modulation_params=modulation_params, + ) self.pixelshuffle = nn.PixelShuffle(upscale_factor) - def forward(self, x: torch.Tensor) -> torch.Tensor: + def forward(self, x: torch.Tensor, y: Optional[torch.Tensor] = None) -> torch.Tensor: """Computes :class:`Subpixel` convolution on input torch.Tensor ``x``. Parameters ---------- x: torch.Tensor Input tensor. + y: torch.Tensor, optional + Auxiliary signal for modulation. """ + if self.modulation != ModConvType.NONE: + return self.pixelshuffle(self.conv(x, y)) return self.pixelshuffle(self.conv(x)) @@ -71,10 +123,22 @@ class ReconBlock(nn.Module): References ---------- - .. [1] Yu, Songhyun, et al. “Deep Iterative Down-Up CNN for Image Denoising.” 2019 IEEE/CVF Conference on Computer Vision and Pattern Recognition Workshops (CVPRW), 2019, pp. 2095–103. IEEE Xplore, https://doi.org/10.1109/CVPRW.2019.00262. + .. [1] Yu, Songhyun, et al. "Deep Iterative Down-Up CNN for Image Denoising." + CVPRW, 2019. https://doi.org/10.1109/CVPRW.2019.00262. """ - def __init__(self, in_channels: int, num_convs: int): + def __init__( + self, + in_channels: int, + num_convs: int, + modulation: ModConvType = ModConvType.NONE, + modulation_params: Optional[ModulationParams] = None, + aux_in_features: Optional[int] = None, + fc_hidden_features: Optional[tuple[int] | int] = None, + fc_groups: int = 1, + fc_activation: ModConvActivation = ModConvActivation.SIGMOID, + num_weights: Optional[int] = None, + ): """Inits :class:`ReconBlock`. Parameters @@ -83,34 +147,64 @@ def __init__(self, in_channels: int, num_convs: int): Number of input channels. num_convs: int Number of convolution blocks. + modulation : ModConvType + Modulation type. Default: ModConvType.NONE. + aux_in_features : int, optional + Auxiliary input features for modulation. + fc_hidden_features : int or tuple of int, optional + Hidden features for modulation MLP. + fc_groups : int + Groups for modulation MLP. Default: 1. + fc_activation : ModConvActivation + Activation for modulation MLP. Default: ModConvActivation.SIGMOID. + num_weights : int, optional + Number of weight bases for ModConvType.SUM. """ super().__init__() - self.convs = nn.ModuleList( - [ - nn.Sequential( - *[ - nn.Conv2d(in_channels=in_channels, out_channels=in_channels, kernel_size=3, padding=1), - nn.PReLU(), - ] - ) - for _ in range(num_convs - 1) - ] - ) - self.convs.append(nn.Conv2d(in_channels=in_channels, out_channels=in_channels, kernel_size=3, padding=1)) + if modulation_params is None: + modulation_params = ModulationParams( + modulation=modulation, + aux_in_features=aux_in_features, + fc_hidden_features=fc_hidden_features, + fc_groups=fc_groups, + fc_activation=fc_activation, + num_weights=num_weights, + ) + self.modulation = modulation_params.modulation self.num_convs = num_convs + self.activations = nn.ModuleList([nn.PReLU() for _ in range(num_convs - 1)]) + + self.convs = nn.ModuleList() + for _ in range(num_convs): + self.convs.append( + mod_conv2d( + in_channels=in_channels, + out_channels=in_channels, + kernel_size=3, + padding=1, + bias=ModConv2dBias.PARAM, + modulation_params=modulation_params, + ) + ) - def forward(self, input_data: torch.Tensor) -> torch.Tensor: + def forward(self, input_data: torch.Tensor, y: Optional[torch.Tensor] = None) -> torch.Tensor: """Computes num_convs convolutions followed by PReLU activation on `input_data`. Parameters ---------- input_data: torch.Tensor Input tensor. + y: torch.Tensor, optional + Auxiliary signal for modulation. """ - output = input_data.clone() for idx in range(self.num_convs): - output = self.convs[idx](output) + if self.modulation != ModConvType.NONE: + output = self.convs[idx](output, y) + else: + output = self.convs[idx](output) + if idx < self.num_convs - 1: + output = self.activations[idx](output) return input_data + output @@ -121,13 +215,21 @@ class DUB(nn.Module): References ---------- - .. [1] Yu, Songhyun, et al. “Deep Iterative Down-Up CNN for Image Denoising.” 2019 IEEE/CVF Conference on Computer Vision and Pattern Recognition Workshops (CVPRW), 2019, pp. 2095–103. IEEE Xplore, https://doi.org/10.1109/CVPRW.2019.00262. + .. [1] Yu, Songhyun, et al. "Deep Iterative Down-Up CNN for Image Denoising." + CVPRW, 2019. https://doi.org/10.1109/CVPRW.2019.00262. """ def __init__( self, in_channels: int, out_channels: int, + modulation: ModConvType = ModConvType.NONE, + modulation_params: Optional[ModulationParams] = None, + aux_in_features: Optional[int] = None, + fc_hidden_features: Optional[tuple[int] | int] = None, + fc_groups: int = 1, + fc_activation: ModConvActivation = ModConvActivation.SIGMOID, + num_weights: Optional[int] = None, ): """Inits :class:`DUB`. @@ -137,72 +239,129 @@ def __init__( Number of input channels. out_channels: int Number of output channels. + modulation : ModConvType + Modulation type. Default: ModConvType.NONE. + aux_in_features : int, optional + Auxiliary input features for modulation. + fc_hidden_features : int or tuple of int, optional + Hidden features for modulation MLP. + fc_groups : int + Groups for modulation MLP. Default: 1. + fc_activation : ModConvActivation + Activation for modulation MLP. Default: ModConvActivation.SIGMOID. + num_weights : int, optional + Number of weight bases for ModConvType.SUM. """ super().__init__() self.in_channels = in_channels self.out_channels = out_channels + if modulation_params is None: + modulation_params = ModulationParams( + modulation=modulation, + aux_in_features=aux_in_features, + fc_hidden_features=fc_hidden_features, + fc_groups=fc_groups, + fc_activation=fc_activation, + num_weights=num_weights, + ) + self.modulation = modulation_params.modulation + + # Scale 1 - down path + self.conv1_1_a = mod_conv2d( + in_channels, in_channels, kernel_size=3, padding=1, modulation_params=modulation_params + ) + self.conv1_1_a_act = nn.PReLU() + self.conv1_1_b = mod_conv2d( + in_channels, in_channels, kernel_size=3, padding=1, modulation_params=modulation_params + ) + self.conv1_1_b_act = nn.PReLU() + self.down1 = mod_conv2d( + in_channels, + in_channels * 2, + kernel_size=3, + stride=2, + padding=1, + modulation_params=modulation_params, + ) - # Scale 1 - self.conv1_1 = nn.Sequential(*[nn.Conv2d(in_channels, in_channels, kernel_size=3, padding=1), nn.PReLU()] * 2) - self.down1 = nn.Conv2d(in_channels, in_channels * 2, kernel_size=3, stride=2, padding=1) - # Scale 2 - self.conv2_1 = nn.Sequential( - *[nn.Conv2d(in_channels * 2, in_channels * 2, kernel_size=3, padding=1), nn.PReLU()] + # Scale 2 - down path + self.conv2_1 = mod_conv2d( + in_channels * 2, + in_channels * 2, + kernel_size=3, + padding=1, + modulation_params=modulation_params, ) - self.down2 = nn.Conv2d(in_channels * 2, in_channels * 4, kernel_size=3, stride=2, padding=1) - # Scale 3 - self.conv3_1 = nn.Sequential( - *[ - nn.Conv2d(in_channels * 4, in_channels * 4, kernel_size=3, padding=1), - nn.PReLU(), - ] + self.conv2_1_act = nn.PReLU() + self.down2 = mod_conv2d( + in_channels * 2, + in_channels * 4, + kernel_size=3, + stride=2, + padding=1, + modulation_params=modulation_params, ) - self.up1 = nn.Sequential( - *[ - # nn.Conv2d(in_channels * 4, in_channels * 8, kernel_size=1), - Subpixel(in_channels * 4, in_channels * 2, 2, 1, 0) - ] + + # Scale 3 - bottom + self.conv3_1 = mod_conv2d( + in_channels * 4, + in_channels * 4, + kernel_size=3, + padding=1, + modulation_params=modulation_params, ) - # Scale 2 - self.conv_agg_1 = nn.Conv2d(in_channels * 4, in_channels * 2, kernel_size=1) - self.conv2_2 = nn.Sequential( - *[ - nn.Conv2d(in_channels * 2, in_channels * 2, kernel_size=3, padding=1), - nn.PReLU(), - ] + self.conv3_1_act = nn.PReLU() + self.up1 = Subpixel(in_channels * 4, in_channels * 2, 2, 1, 0, modulation_params=modulation_params) + + # Scale 2 - up path + self.conv_agg_1 = mod_conv2d( + in_channels * 4, + in_channels * 2, + kernel_size=1, + padding=0, + modulation_params=modulation_params, ) - self.up2 = nn.Sequential( - *[ - # nn.Conv2d(in_channels * 2, in_channels * 4, kernel_size=1), - Subpixel(in_channels * 2, in_channels, 2, 1, 0) - ] + self.conv2_2 = mod_conv2d( + in_channels * 2, + in_channels * 2, + kernel_size=3, + padding=1, + modulation_params=modulation_params, ) - # Scale 1 - self.conv_agg_2 = nn.Conv2d(in_channels * 2, in_channels, kernel_size=1) - self.conv1_2 = nn.Sequential(*[nn.Conv2d(in_channels, in_channels, kernel_size=3, padding=1), nn.PReLU()] * 2) - self.conv_out = nn.Sequential(*[nn.Conv2d(in_channels, in_channels, kernel_size=3, padding=1), nn.PReLU()]) + self.conv2_2_act = nn.PReLU() + self.up2 = Subpixel(in_channels * 2, in_channels, 2, 1, 0, modulation_params=modulation_params) - @staticmethod - def pad(x: torch.Tensor) -> torch.Tensor: - """Pads input to height and width dimensions if odd. + # Scale 1 - up path + self.conv_agg_2 = mod_conv2d( + in_channels * 2, in_channels, kernel_size=1, padding=0, modulation_params=modulation_params + ) + self.conv1_2_a = mod_conv2d( + in_channels, in_channels, kernel_size=3, padding=1, modulation_params=modulation_params + ) + self.conv1_2_a_act = nn.PReLU() + self.conv1_2_b = mod_conv2d( + in_channels, in_channels, kernel_size=3, padding=1, modulation_params=modulation_params + ) + self.conv1_2_b_act = nn.PReLU() + self.conv_out = mod_conv2d( + in_channels, in_channels, kernel_size=3, padding=1, modulation_params=modulation_params + ) + self.conv_out_act = nn.PReLU() - Parameters - ---------- - x: torch.Tensor - Input to pad. + def _conv(self, layer: ModConv2d, x: torch.Tensor, y: Optional[torch.Tensor]) -> torch.Tensor: + if self.modulation != ModConvType.NONE: + return layer(x, y) + return layer(x) - Returns - ------- - x: torch.Tensor - Padded tensor. - """ + @staticmethod + def pad(x: torch.Tensor) -> torch.Tensor: + """Pads input to height and width dimensions if odd.""" padding = [0, 0, 0, 0] - if x.shape[-2] % 2 != 0: - padding[3] = 1 # Padding right - width + padding[3] = 1 if x.shape[-1] % 2 != 0: - padding[1] = 1 # Padding bottom - height + padding[1] = 1 if sum(padding) != 0: x = F.pad(x, padding, "reflect") return x @@ -214,7 +373,7 @@ def crop_to_shape(x: torch.Tensor, shape: Tuple[int, int]) -> torch.Tensor: Parameters ---------- x: torch.Tensor - Input tensor with shape (\*, H, W). + Input tensor with shape (\\*, H, W). shape: Tuple(int, int) Crop shape corresponding to H, W. @@ -224,19 +383,21 @@ def crop_to_shape(x: torch.Tensor, shape: Tuple[int, int]) -> torch.Tensor: Cropped tensor. """ h, w = x.shape[-2:] - if h > shape[0]: x = x[:, :, : shape[0], :] if w > shape[1]: x = x[:, :, :, : shape[1]] return x - def forward(self, x: torch.Tensor) -> torch.Tensor: - """ + def forward(self, x: torch.Tensor, y: Optional[torch.Tensor] = None) -> torch.Tensor: + """Forward pass. + Parameters ---------- x: torch.Tensor Input tensor. + y: torch.Tensor, optional + Auxiliary signal for modulation. Returns ------- @@ -244,20 +405,41 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: DUB output. """ x1 = self.pad(x.clone()) - x1 = x1 + self.conv1_1(x1) - x2 = self.down1(x1) - x2 = x2 + self.conv2_1(x2) - out = self.down2(x2) - out = out + self.conv3_1(out) - out = self.up1(out) + x1 = x1 + self.conv1_1_b_act( + self._conv(self.conv1_1_b, self.conv1_1_a_act(self._conv(self.conv1_1_a, x1, y)), y) + ) + x2 = self._conv(self.down1, x1, y) + x2 = x2 + self.conv2_1_act(self._conv(self.conv2_1, x2, y)) + out = self._conv(self.down2, x2, y) + out = out + self.conv3_1_act(self._conv(self.conv3_1, out, y)) + + if self.modulation != ModConvType.NONE: + out = self.up1(out, y) + else: + out = self.up1(out) + out = torch.cat([x2, self.crop_to_shape(out, (x2.shape[-2], x2.shape[-1]))], dim=1) - out = self.conv_agg_1(out) - out = out + self.conv2_2(out) - out = self.up2(out) + out = self._conv(self.conv_agg_1, out, y) + out = out + self.conv2_2_act(self._conv(self.conv2_2, out, y)) + + if self.modulation != ModConvType.NONE: + out = self.up2(out, y) + else: + out = self.up2(out) + out = torch.cat([x1, self.crop_to_shape(out, (x1.shape[-2], x1.shape[-1]))], dim=1) - out = self.conv_agg_2(out) - out = out + self.conv1_2(out) - out = x + self.crop_to_shape(self.conv_out(out), (x.shape[-2], x.shape[-1])) + out = self._conv(self.conv_agg_2, out, y) + out = out + self.conv1_2_b_act( + self._conv( + self.conv1_2_b, + self.conv1_2_a_act(self._conv(self.conv1_2_a, out, y)), + y, + ) + ) + out = x + self.crop_to_shape( + self.conv_out_act(self._conv(self.conv_out, out, y)), + (x.shape[-2], x.shape[-1]), + ) return out @@ -267,7 +449,8 @@ class DIDN(nn.Module): References ---------- - .. [1] Yu, Songhyun, et al. “Deep Iterative Down-Up CNN for Image Denoising.” 2019 IEEE/CVF Conference on Computer Vision and Pattern Recognition Workshops (CVPRW), 2019, pp. 2095–103. IEEE Xplore, https://doi.org/10.1109/CVPRW.2019.00262. + .. [1] Yu, Songhyun, et al. "Deep Iterative Down-Up CNN for Image Denoising." + CVPRW, 2019. https://doi.org/10.1109/CVPRW.2019.00262. """ def __init__( @@ -278,6 +461,13 @@ def __init__( num_dubs: int = 6, num_convs_recon: int = 9, skip_connection: bool = False, + modulation: ModConvType = ModConvType.NONE, + modulation_params: Optional[ModulationParams] = None, + aux_in_features: Optional[int] = None, + fc_hidden_features: Optional[tuple[int] | int] = None, + fc_groups: int = 1, + fc_activation: ModConvActivation = ModConvActivation.SIGMOID, + num_weights: Optional[int] = None, ): """Inits :class:`DIDN`. @@ -295,44 +485,94 @@ def __init__( Number of ReconBlock convolutions. Default: 9. skip_connection: bool Use skip connection. Default: False. + modulation : ModConvType + Modulation type. Default: ModConvType.NONE. + aux_in_features : int, optional + Auxiliary input features for modulation. + fc_hidden_features : int or tuple of int, optional + Hidden features for modulation MLP. + fc_groups : int + Groups for modulation MLP. Default: 1. + fc_activation : ModConvActivation + Activation for modulation MLP. Default: ModConvActivation.SIGMOID. + num_weights : int, optional + Number of weight bases for ModConvType.SUM. """ super().__init__() - self.conv_in = nn.Sequential( - *[nn.Conv2d(in_channels=in_channels, out_channels=hidden_channels, kernel_size=3, padding=1), nn.PReLU()] + if modulation_params is None: + modulation_params = ModulationParams( + modulation=modulation, + aux_in_features=aux_in_features, + fc_hidden_features=fc_hidden_features, + fc_groups=fc_groups, + fc_activation=fc_activation, + num_weights=num_weights, + ) + self.modulation = modulation_params.modulation + + self.conv_in = mod_conv2d( + in_channels=in_channels, + out_channels=hidden_channels, + kernel_size=3, + padding=1, + modulation_params=modulation_params, ) - self.down = nn.Conv2d( + self.conv_in_act = nn.PReLU() + self.down = mod_conv2d( in_channels=hidden_channels, out_channels=hidden_channels, kernel_size=3, stride=2, padding=1, + modulation_params=modulation_params, ) self.dubs = nn.ModuleList( - [DUB(in_channels=hidden_channels, out_channels=hidden_channels) for _ in range(num_dubs)] - ) - self.recon_block = ReconBlock(in_channels=hidden_channels, num_convs=num_convs_recon) - self.recon_agg = nn.Conv2d(in_channels=hidden_channels * num_dubs, out_channels=hidden_channels, kernel_size=1) - self.conv = nn.Sequential( - *[ - nn.Conv2d( + [ + DUB( in_channels=hidden_channels, out_channels=hidden_channels, - kernel_size=3, - padding=1, - ), - nn.PReLU(), + modulation_params=modulation_params, + ) + for _ in range(num_dubs) ] ) - self.up2 = Subpixel(hidden_channels, hidden_channels, 2, 1) - self.conv_out = nn.Conv2d( + self.recon_blocks = nn.ModuleList( + [ + ReconBlock(in_channels=hidden_channels, num_convs=num_convs_recon, modulation_params=modulation_params) + for _ in range(num_dubs) + ] + ) + self.recon_agg = mod_conv2d( + in_channels=hidden_channels * num_dubs, + out_channels=hidden_channels, + kernel_size=1, + padding=0, + modulation_params=modulation_params, + ) + self.conv = mod_conv2d( + in_channels=hidden_channels, + out_channels=hidden_channels, + kernel_size=3, + padding=1, + modulation_params=modulation_params, + ) + self.conv_act = nn.PReLU() + self.up2 = Subpixel(hidden_channels, hidden_channels, 2, 1, modulation_params=modulation_params) + self.conv_out = mod_conv2d( in_channels=hidden_channels, out_channels=out_channels, kernel_size=3, padding=1, + modulation_params=modulation_params, ) self.num_dubs = num_dubs self.skip_connection = (in_channels == out_channels) and skip_connection + def _conv(self, layer: ModConv2d, x: torch.Tensor, y: Optional[torch.Tensor]) -> torch.Tensor: + if self.modulation != ModConvType.NONE: + return layer(x, y) + return layer(x) + @staticmethod def crop_to_shape(x: torch.Tensor, shape: Tuple[int, int]) -> torch.Tensor: """Crops ``x`` to specified shape. @@ -340,7 +580,7 @@ def crop_to_shape(x: torch.Tensor, shape: Tuple[int, int]) -> torch.Tensor: Parameters ---------- x: torch.Tensor - Input tensor with shape (\*, H, W). + Input tensor with shape (\\*, H, W). shape: Tuple(int, int) Crop shape corresponding to H, W. @@ -350,20 +590,21 @@ def crop_to_shape(x: torch.Tensor, shape: Tuple[int, int]) -> torch.Tensor: Cropped tensor. """ h, w = x.shape[-2:] - if h > shape[0]: x = x[:, :, : shape[0], :] if w > shape[1]: x = x[:, :, :, : shape[1]] return x - def forward(self, x: torch.Tensor, channel_dim: int = 1) -> torch.Tensor: + def forward(self, x: torch.Tensor, y: Optional[torch.Tensor] = None, channel_dim: int = 1) -> torch.Tensor: """Takes as input a torch.Tensor `x` and computes DIDN(x). Parameters ---------- x: torch.Tensor Input tensor. + y: torch.Tensor, optional + Auxiliary signal for modulation of shape (N, aux_in_features). channel_dim: int Channel dimension. Default: 1. @@ -372,19 +613,24 @@ def forward(self, x: torch.Tensor, channel_dim: int = 1) -> torch.Tensor: out: torch.Tensor DIDN output tensor. """ - out = self.conv_in(x) - out = self.down(out) + out = self.conv_in_act(self._conv(self.conv_in, x, y)) + out = self._conv(self.down, out, y) dub_outs = [] for dub in self.dubs: - out = dub(out) + out = dub(out, y) dub_outs.append(out) - out = [self.recon_block(dub_out) for dub_out in dub_outs] - out = self.recon_agg(torch.cat(out, dim=channel_dim)) - out = self.conv(out) - out = self.up2(out) - out = self.conv_out(out) + out = [self.recon_blocks[i](dub_outs[i], y) for i in range(self.num_dubs)] + out = self._conv(self.recon_agg, torch.cat(out, dim=channel_dim), y) + out = self.conv_act(self._conv(self.conv, out, y)) + + if self.modulation != ModConvType.NONE: + out = self.up2(out, y) + else: + out = self.up2(out) + + out = self._conv(self.conv_out, out, y) out = self.crop_to_shape(out, (x.shape[-2], x.shape[-1])) if self.skip_connection: diff --git a/direct/nn/get_nn_model_config.py b/direct/nn/get_nn_model_config.py index 286ba227..79f25c41 100644 --- a/direct/nn/get_nn_model_config.py +++ b/direct/nn/get_nn_model_config.py @@ -16,7 +16,9 @@ from torch import nn from direct.constants import COMPLEX_SIZE +from direct.nn.adain.adain import NormType from direct.nn.conv.conv import Conv2d +from direct.nn.conv.modulated import ModConvActivation, ModConvType from direct.nn.didn.didn import DIDN from direct.nn.resnet.resnet import ResNet from direct.nn.types import ActivationType, ModelName @@ -42,9 +44,15 @@ def _get_relu_activation(activation: ActivationType = ActivationType.RELU, **kwa def _get_model_config( - model_architecture_name: ModelName, in_channels: int = COMPLEX_SIZE, out_channels: int = COMPLEX_SIZE, **kwargs + model_architecture_name: ModelName, + in_channels: int = COMPLEX_SIZE, + out_channels: int = COMPLEX_SIZE, + **kwargs, ) -> tuple[type[nn.Module], dict[str, object]]: - model_kwargs: dict[str, object] = {"in_channels": in_channels, "out_channels": out_channels} + model_kwargs: dict[str, object] = { + "in_channels": in_channels, + "out_channels": out_channels, + } if model_architecture_name in ["unet", "normunet"]: model_architecture = UnetModel2d if model_architecture_name == "unet" else NormUnetModel2d model_kwargs.update( @@ -52,6 +60,15 @@ def _get_model_config( "num_filters": kwargs.get("unet_num_filters", 32), "num_pool_layers": kwargs.get("unet_num_pool_layers", 4), "dropout_probability": kwargs.get("unet_dropout", 0.0), + "modulation": kwargs.get("modulation", ModConvType.NONE), + "aux_in_features": kwargs.get("aux_in_features"), + "fc_hidden_features": kwargs.get("fc_hidden_features"), + "fc_groups": kwargs.get("fc_groups", 1), + "fc_activation": kwargs.get("fc_activation", ModConvActivation.SIGMOID), + "num_weights": kwargs.get("num_weights"), + "modulation_at_input": kwargs.get("modulation_at_input", False), + "norm_type": kwargs.get("unet_norm_type", NormType.INSTANCE), + "adain_hidden_features": kwargs.get("unet_adain_hidden_features"), } ) elif model_architecture_name == "resnet": diff --git a/direct/nn/iterdualnet/config.py b/direct/nn/iterdualnet/config.py index 2ec2d697..a4914bcd 100644 --- a/direct/nn/iterdualnet/config.py +++ b/direct/nn/iterdualnet/config.py @@ -12,8 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. from dataclasses import dataclass +from typing import Optional from direct.config.defaults import ModelConfig +from direct.nn.conv.modulated import ModConvActivation, ModConvType @dataclass @@ -30,3 +32,11 @@ class IterDualNetConfig(ModelConfig): image_no_parameter_sharing: bool = True kspace_no_parameter_sharing: bool = False compute_per_coil: bool = True + conv_modulation: ModConvType = ModConvType.NONE + aux_in_features: Optional[int] = None + auxiliary_features: Optional[tuple[str, ...]] = None + log_aux: bool = False + fc_hidden_features: Optional[tuple[int, ...]] = None + fc_groups: int = 1 + fc_activation: ModConvActivation = ModConvActivation.SIGMOID + num_weights: Optional[int] = None diff --git a/direct/nn/iterdualnet/iterdualnet.py b/direct/nn/iterdualnet/iterdualnet.py index 93da7b31..761bcd86 100644 --- a/direct/nn/iterdualnet/iterdualnet.py +++ b/direct/nn/iterdualnet/iterdualnet.py @@ -12,11 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import annotations + +from typing import Optional + import torch from torch import nn import direct.data.transforms as T from direct.constants import COMPLEX_SIZE +from direct.nn.conv.modulated import ModConvActivation, ModConvType, ModulationParams from direct.nn.unet.unet_2d import NormUnetModel2d, UnetModel2d from direct.types import FFTOperator @@ -33,6 +38,15 @@ class IterDualNet(nn.Module): by unrolling a gradient descent scheme where :math:`\mathcal{E}` and :math:`\mathcal{R}` are the expand and reduce operators which use the sensitivity maps. :math:`D_I` and :math:`D_F` are trainable U-Nets operating in the image and k-space domain. + + Supports conditional weight modulation as proposed in [1]_. + + References + ---------- + + .. [1] Moriakov, N., Yiasemis, G., Sonke, J.-J. & Teuwen, J. (2026). Conditional Learned Reconstruction for + Medical Imaging. Proceedings of The 9th International Conference on Medical Imaging with Deep Learning, + PMLR 315:754-780. https://proceedings.mlr.press/v315/moriakov26a.html """ def __init__( @@ -45,6 +59,12 @@ def __init__( image_no_parameter_sharing: bool = True, kspace_no_parameter_sharing: bool = True, compute_per_coil: bool = True, + conv_modulation: ModConvType = ModConvType.NONE, + aux_in_features: Optional[int] = None, + fc_hidden_features: Optional[tuple[int] | int] = None, + fc_groups: int = 1, + fc_activation: ModConvActivation = ModConvActivation.SIGMOID, + num_weights: Optional[int] = None, **kwargs, ): """Inits :class:`IterDualNet`. @@ -67,6 +87,18 @@ def __init__( If False, a single kspace model will be shared across all iterations. Default: True. compute_per_coil : bool If True :math:`f` will be transformed into a multi-coil kspace. + conv_modulation : ModConvType + Modulation type for convolutional layers. Default: ModConvType.NONE. + aux_in_features : int, optional + Number of features in the auxiliary input for modulation. + fc_hidden_features : int or tuple of int, optional + Hidden features in the modulation MLP. + fc_groups : int + Groups for modulation MLP output. Default: 1. + fc_activation : ModConvActivation + Activation after modulation MLP. Default: ModConvActivation.SIGMOID. + num_weights : int, optional + Number of weight bases for ModConvType.SUM. kwargs : dict Kwargs for unet models. """ @@ -75,12 +107,22 @@ def __init__( self.forward_operator = forward_operator self.backward_operator = backward_operator self.num_iter = num_iter + self.conv_modulation = conv_modulation self.image_no_parameter_sharing = image_no_parameter_sharing self.kspace_no_parameter_sharing = kspace_no_parameter_sharing image_unet_architecture = NormUnetModel2d if image_normunet else UnetModel2d kspace_unet_architecture = NormUnetModel2d if kspace_normunet else UnetModel2d + modulation_params = ModulationParams( + modulation=conv_modulation, + aux_in_features=aux_in_features, + fc_hidden_features=fc_hidden_features, + fc_groups=fc_groups, + fc_activation=fc_activation, + num_weights=num_weights, + ) + self.image_block_list = nn.ModuleList() self.kspace_block_list = nn.ModuleList() @@ -92,6 +134,7 @@ def __init__( num_filters=kwargs.get("image_unet_num_filters", 8), num_pool_layers=kwargs.get("image_unet_num_pool_layers", 4), dropout_probability=kwargs.get("image_unet_dropout", 0.0), + modulation_params=modulation_params, ) ) for _ in range(self.num_iter if self.kspace_no_parameter_sharing else 1): @@ -102,6 +145,7 @@ def __init__( num_filters=kwargs.get("kspace_unet_num_filters", 8), num_pool_layers=kwargs.get("kspace_unet_num_pool_layers", 4), dropout_probability=kwargs.get("kspace_unet_dropout", 0.0), + modulation_params=modulation_params, ) ) self.compute_per_coil = compute_per_coil @@ -114,44 +158,87 @@ def __init__( self._complex_dim = -1 self._spatial_dims = (2, 3) - def _image_model(self, image: torch.Tensor, step: int) -> torch.Tensor: + def _image_model( + self, + image: torch.Tensor, + step: int, + auxiliary_data: Optional[torch.Tensor] = None, + ) -> torch.Tensor: image = image.permute(0, 3, 1, 2) block_idx = step if self.image_no_parameter_sharing else 0 + if self.conv_modulation != ModConvType.NONE: + return self.image_block_list[block_idx](image, auxiliary_data).permute(0, 2, 3, 1).contiguous() return self.image_block_list[block_idx](image).permute(0, 2, 3, 1).contiguous() - def _kspace_model(self, kspace: torch.Tensor, step: int) -> torch.Tensor: + def _kspace_model( + self, + kspace: torch.Tensor, + step: int, + auxiliary_data: Optional[torch.Tensor] = None, + ) -> torch.Tensor: block_idx = step if self.kspace_no_parameter_sharing else 0 if self.compute_per_coil: kspace = ( - self._compute_model_per_coil(self.kspace_block_list[block_idx], kspace.permute(0, 1, 4, 2, 3)) + self._compute_model_per_coil( + self.kspace_block_list[block_idx], + kspace.permute(0, 1, 4, 2, 3), + auxiliary_data, + ) .permute(0, 1, 3, 4, 2) .contiguous() ) else: - kspace = self.kspace_block_list[block_idx](kspace.permute(0, 3, 1, 2)).permute(0, 2, 3, 1).contiguous() + if self.conv_modulation != ModConvType.NONE: + kspace = ( + self.kspace_block_list[block_idx](kspace.permute(0, 3, 1, 2), auxiliary_data) + .permute(0, 2, 3, 1) + .contiguous() + ) + else: + kspace = self.kspace_block_list[block_idx](kspace.permute(0, 3, 1, 2)).permute(0, 2, 3, 1).contiguous() return kspace - def _compute_model_per_coil(self, model: nn.Module, data: torch.Tensor) -> torch.Tensor: + def _compute_model_per_coil( + self, + model: nn.Module, + data: torch.Tensor, + auxiliary_data: Optional[torch.Tensor] = None, + ) -> torch.Tensor: output = [] for idx in range(data.size(self._coil_dim)): subselected_data = data.select(self._coil_dim, idx) - output.append(model(subselected_data)) + if self.conv_modulation != ModConvType.NONE: + output.append(model(subselected_data, auxiliary_data)) + else: + output.append(model(subselected_data)) return torch.stack(output, dim=self._coil_dim) def _forward_operator( - self, image: torch.Tensor, sampling_mask: torch.Tensor, sensitivity_map: torch.Tensor + self, + image: torch.Tensor, + sampling_mask: torch.Tensor, + sensitivity_map: torch.Tensor, ) -> torch.Tensor: return T.apply_mask( - self.forward_operator(T.expand_operator(image, sensitivity_map, self._coil_dim), dim=self._spatial_dims), + self.forward_operator( + T.expand_operator(image, sensitivity_map, self._coil_dim), + dim=self._spatial_dims, + ), sampling_mask, return_mask=False, ) def _backward_operator( - self, kspace: torch.Tensor, sampling_mask: torch.Tensor, sensitivity_map: torch.Tensor + self, + kspace: torch.Tensor, + sampling_mask: torch.Tensor, + sensitivity_map: torch.Tensor, ) -> torch.Tensor: return T.reduce_operator( - self.backward_operator(T.apply_mask(kspace, sampling_mask, return_mask=False), self._spatial_dims), + self.backward_operator( + T.apply_mask(kspace, sampling_mask, return_mask=False), + self._spatial_dims, + ), sensitivity_map, self._coil_dim, ) @@ -161,6 +248,7 @@ def forward( masked_kspace: torch.Tensor, sampling_mask: torch.Tensor, sensitivity_map: torch.Tensor, + auxiliary_data: Optional[torch.Tensor] = None, ) -> torch.Tensor: """Computes forward pass of :class:`IterDualNet`. @@ -172,6 +260,8 @@ def forward( Sampling mask of shape (N, 1, height, width, 1). sensitivity_map: torch.Tensor Sensitivity map of shape (N, coil, height, width, complex=2). + auxiliary_data: torch.Tensor, optional + Auxiliary data for modulation of shape (N, aux_in_features). Returns ------- @@ -179,16 +269,21 @@ def forward( Output image of shape (N, height, width, complex=2). """ x = T.reduce_operator( - self.backward_operator(masked_kspace, self._spatial_dims), sensitivity_map, self._coil_dim + self.backward_operator(masked_kspace, self._spatial_dims), + sensitivity_map, + self._coil_dim, ) for step in range(self.num_iter): f = ( - self.forward_operator(T.expand_operator(x, sensitivity_map, self._coil_dim), dim=self._spatial_dims) + self.forward_operator( + T.expand_operator(x, sensitivity_map, self._coil_dim), + dim=self._spatial_dims, + ) if self.compute_per_coil else self.forward_operator(x, dim=[d - 1 for d in self._spatial_dims]) ) - kspace_model_out = self._kspace_model(f, step) + kspace_model_out = self._kspace_model(f, step, auxiliary_data) kspace_model_out = ( T.reduce_operator( self.backward_operator(kspace_model_out, self._spatial_dims), @@ -199,7 +294,7 @@ def forward( else self.backward_operator(kspace_model_out, dim=[d - 1 for d in self._spatial_dims]) ) - img_model_out = self._image_model(x, step) + img_model_out = self._image_model(x, step, auxiliary_data) dc_out = self._backward_operator( self._forward_operator(x, sampling_mask, sensitivity_map) - masked_kspace, diff --git a/direct/nn/iterdualnet/iterdualnet_engine.py b/direct/nn/iterdualnet/iterdualnet_engine.py index 23dda7c5..6590c6ac 100644 --- a/direct/nn/iterdualnet/iterdualnet_engine.py +++ b/direct/nn/iterdualnet/iterdualnet_engine.py @@ -66,6 +66,7 @@ def forward_function(self, data: Dict[str, Any]) -> Tuple[torch.Tensor, None]: masked_kspace=data["masked_kspace"], sampling_mask=data["sampling_mask"], sensitivity_map=data["sensitivity_map"], + auxiliary_data=self.auxiliary_data_from(data), ) # shape (batch, height, width) output_kspace = None diff --git a/direct/nn/jointicnet/config.py b/direct/nn/jointicnet/config.py index 961769ed..392646e9 100644 --- a/direct/nn/jointicnet/config.py +++ b/direct/nn/jointicnet/config.py @@ -12,8 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. from dataclasses import dataclass +from typing import Optional from direct.config.defaults import ModelConfig +from direct.nn.conv.modulated import ModConvActivation, ModConvType @dataclass @@ -29,3 +31,11 @@ class JointICNetConfig(ModelConfig): sens_unet_num_filters: int = 8 sens_unet_num_pool_layers: int = 4 sens_unet_dropout: float = 0.0 + conv_modulation: ModConvType = ModConvType.NONE + aux_in_features: Optional[int] = None + auxiliary_features: Optional[tuple[str, ...]] = None + log_aux: bool = False + fc_hidden_features: Optional[tuple[int, ...]] = None + fc_groups: int = 1 + fc_activation: ModConvActivation = ModConvActivation.SIGMOID + num_weights: Optional[int] = None diff --git a/direct/nn/jointicnet/jointicnet.py b/direct/nn/jointicnet/jointicnet.py index ec20a4dd..d72a22ea 100644 --- a/direct/nn/jointicnet/jointicnet.py +++ b/direct/nn/jointicnet/jointicnet.py @@ -12,10 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import annotations + +from typing import Optional + import torch import torch.nn as nn import direct.data.transforms as T +from direct.nn.conv.modulated import ModConvActivation, ModConvType, ModulationParams from direct.nn.unet.unet_2d import NormUnetModel2d, UnetModel2d from direct.types import FFTOperator @@ -24,9 +29,17 @@ class JointICNet(nn.Module): """Joint Deep Model-Based MR Image and Coil Sensitivity Reconstruction Network (Joint-ICNet) implementation as presented in [1]_. + Supports conditional weight modulation as proposed in [2]_. + References ---------- - .. [1] Jun, Yohan, et al. “Joint Deep Model-Based MR Image and Coil Sensitivity Reconstruction Network (Joint-ICNet) for Fast MRI.” 2021 IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR), IEEE, 2021, pp. 5266–75. DOI.org (Crossref), https://doi.org/10.1109/CVPR46437.2021.00523. + .. [1] Jun, Yohan, et al. "Joint Deep Model-Based MR Image and Coil Sensitivity Reconstruction Network + (Joint-ICNet) for Fast MRI." 2021 IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR), + IEEE, 2021, pp. 5266-75. https://doi.org/10.1109/CVPR46437.2021.00523. + + .. [2] Moriakov, N., Yiasemis, G., Sonke, J.-J. & Teuwen, J. (2026). Conditional Learned Reconstruction for + Medical Imaging. Proceedings of The 9th International Conference on Medical Imaging with Deep Learning, + PMLR 315:754-780. https://proceedings.mlr.press/v315/moriakov26a.html """ def __init__( @@ -35,6 +48,12 @@ def __init__( backward_operator: FFTOperator, num_iter: int = 10, use_norm_unet: bool = False, + conv_modulation: ModConvType = ModConvType.NONE, + aux_in_features: Optional[int] = None, + fc_hidden_features: Optional[tuple[int] | int] = None, + fc_groups: int = 1, + fc_activation: ModConvActivation = ModConvActivation.SIGMOID, + num_weights: Optional[int] = None, **kwargs, ): """Inits :class:`JointICNet`. @@ -49,6 +68,18 @@ def __init__( Number of unrolled iterations. Default: 10. use_norm_unet: bool If True, a Normalized U-Net is used. Default: False. + conv_modulation : ModConvType + Modulation type for convolutional layers. Default: ModConvType.NONE. + aux_in_features : int, optional + Number of features in the auxiliary input for modulation. + fc_hidden_features : int or tuple of int, optional + Hidden features in the modulation MLP. + fc_groups : int + Groups for modulation MLP output. Default: 1. + fc_activation : ModConvActivation + Activation after modulation MLP. Default: ModConvActivation.SIGMOID. + num_weights : int, optional + Number of weight bases for ModConvType.SUM. kwargs: dict Image, k-space and sensitivity-map U-Net models keyword-arguments. """ @@ -57,15 +88,26 @@ def __init__( self.forward_operator = forward_operator self.backward_operator = backward_operator self.num_iter = num_iter + self.conv_modulation = conv_modulation unet_architecture = NormUnetModel2d if use_norm_unet else UnetModel2d + modulation_params = ModulationParams( + modulation=conv_modulation, + aux_in_features=aux_in_features, + fc_hidden_features=fc_hidden_features, + fc_groups=fc_groups, + fc_activation=fc_activation, + num_weights=num_weights, + ) + self.image_model = unet_architecture( in_channels=2, out_channels=2, num_filters=kwargs.get("image_unet_num_filters", 8), num_pool_layers=kwargs.get("image_unet_num_pool_layers", 4), dropout_probability=kwargs.get("image_unet_dropout", 0.0), + modulation_params=modulation_params, ) self.kspace_model = unet_architecture( in_channels=2, @@ -73,6 +115,7 @@ def __init__( num_filters=kwargs.get("kspace_unet_num_filters", 8), num_pool_layers=kwargs.get("kspace_unet_num_pool_layers", 4), dropout_probability=kwargs.get("kspace_unet_dropout", 0.0), + modulation_params=modulation_params, ) self.sens_model = unet_architecture( in_channels=2, @@ -80,6 +123,7 @@ def __init__( num_filters=kwargs.get("sens_unet_num_filters", 8), num_pool_layers=kwargs.get("sens_unet_num_pool_layers", 4), dropout_probability=kwargs.get("sens_unet_dropout", 0.0), + modulation_params=modulation_params, ) self.conv_out = nn.Conv2d(in_channels=2, out_channels=2, kernel_size=1) @@ -94,40 +138,65 @@ def __init__( self._complex_dim = -1 self._spatial_dims = (2, 3) - def _image_model(self, image: torch.Tensor) -> torch.Tensor: + def _image_model(self, image: torch.Tensor, auxiliary_data: Optional[torch.Tensor] = None) -> torch.Tensor: image = image.permute(0, 3, 1, 2) + if self.conv_modulation != ModConvType.NONE: + return self.image_model(image, auxiliary_data).permute(0, 2, 3, 1).contiguous() return self.image_model(image).permute(0, 2, 3, 1).contiguous() - def _kspace_model(self, kspace: torch.Tensor) -> torch.Tensor: + def _kspace_model(self, kspace: torch.Tensor, auxiliary_data: Optional[torch.Tensor] = None) -> torch.Tensor: kspace = kspace.permute(0, 3, 1, 2) + if self.conv_modulation != ModConvType.NONE: + return self.kspace_model(kspace, auxiliary_data).permute(0, 2, 3, 1).contiguous() return self.kspace_model(kspace).permute(0, 2, 3, 1).contiguous() - def _sens_model(self, sensitivity_map: torch.Tensor) -> torch.Tensor: + def _sens_model( + self, + sensitivity_map: torch.Tensor, + auxiliary_data: Optional[torch.Tensor] = None, + ) -> torch.Tensor: return ( - self._compute_model_per_coil(self.sens_model, sensitivity_map.permute(0, 1, 4, 2, 3)) + self._compute_model_per_coil(self.sens_model, sensitivity_map.permute(0, 1, 4, 2, 3), auxiliary_data) .permute(0, 1, 3, 4, 2) .contiguous() ) - def _compute_model_per_coil(self, model: nn.Module, data: torch.Tensor) -> torch.Tensor: + def _compute_model_per_coil( + self, + model: nn.Module, + data: torch.Tensor, + auxiliary_data: Optional[torch.Tensor] = None, + ) -> torch.Tensor: output = [] for idx in range(data.size(self._coil_dim)): subselected_data = data.select(self._coil_dim, idx) - output.append(model(subselected_data)) + if self.conv_modulation != ModConvType.NONE: + output.append(model(subselected_data, auxiliary_data)) + else: + output.append(model(subselected_data)) return torch.stack(output, dim=self._coil_dim) def _forward_operator( - self, image: torch.Tensor, sampling_mask: torch.Tensor, sensitivity_map: torch.Tensor + self, + image: torch.Tensor, + sampling_mask: torch.Tensor, + sensitivity_map: torch.Tensor, ) -> torch.Tensor: forward = torch.where( sampling_mask == 0, torch.tensor([0.0], dtype=image.dtype).to(image.device), - self.forward_operator(T.expand_operator(image, sensitivity_map, self._coil_dim), dim=self._spatial_dims), + self.forward_operator( + T.expand_operator(image, sensitivity_map, self._coil_dim), + dim=self._spatial_dims, + ), ) return forward def _backward_operator( - self, kspace: torch.Tensor, sampling_mask: torch.Tensor, sensitivity_map: torch.Tensor + self, + kspace: torch.Tensor, + sampling_mask: torch.Tensor, + sensitivity_map: torch.Tensor, ) -> torch.Tensor: backward = T.reduce_operator( self.backward_operator( @@ -148,6 +217,7 @@ def forward( masked_kspace: torch.Tensor, sampling_mask: torch.Tensor, sensitivity_map: torch.Tensor, + auxiliary_data: Optional[torch.Tensor] = None, ) -> torch.Tensor: """Computes forward pass of :class:`JointICNet`. @@ -159,6 +229,8 @@ def forward( Sampling mask of shape (N, 1, height, width, 1). sensitivity_map: torch.Tensor Sensitivity map of shape (N, coil, height, width, complex=2). + auxiliary_data: torch.Tensor, optional + Auxiliary data for modulation of shape (N, aux_in_features). Returns ------- @@ -190,7 +262,10 @@ def forward( + self.reg_param_C[curr_iter] * ( sensitivity_map - - self._sens_model(self.backward_operator(masked_kspace, dim=self._spatial_dims)) + - self._sens_model( + self.backward_operator(masked_kspace, dim=self._spatial_dims), + auxiliary_data, + ) ) ) ) @@ -209,12 +284,13 @@ def forward( sampling_mask, sensitivity_map, ) - + self.reg_param_I[curr_iter] * (input_image - self._image_model(input_image)) + + self.reg_param_I[curr_iter] * (input_image - self._image_model(input_image, auxiliary_data)) + self.reg_param_F[curr_iter] * ( input_image - self.backward_operator( - self._kspace_model(input_kspace), dim=tuple(d - 1 for d in self._spatial_dims) + self._kspace_model(input_kspace, auxiliary_data), + dim=tuple(d - 1 for d in self._spatial_dims), ) ) ) diff --git a/direct/nn/jointicnet/jointicnet_engine.py b/direct/nn/jointicnet/jointicnet_engine.py index 8b40c611..504f134e 100644 --- a/direct/nn/jointicnet/jointicnet_engine.py +++ b/direct/nn/jointicnet/jointicnet_engine.py @@ -51,6 +51,7 @@ def forward_function(self, data: Dict[str, Any]) -> Tuple[torch.Tensor, None]: masked_kspace=data["masked_kspace"], sampling_mask=data["sampling_mask"], sensitivity_map=data["sensitivity_map"], + auxiliary_data=self.auxiliary_data_from(data), ) # shape (batch, height, width) output_kspace = None diff --git a/direct/nn/kikinet/config.py b/direct/nn/kikinet/config.py index 826c8b7a..888a0604 100644 --- a/direct/nn/kikinet/config.py +++ b/direct/nn/kikinet/config.py @@ -12,8 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. from dataclasses import dataclass +from typing import Optional from direct.config.defaults import ModelConfig +from direct.nn.conv.modulated import ModConvActivation, ModConvType @dataclass @@ -38,3 +40,11 @@ class KIKINetConfig(ModelConfig): kspace_unet_num_pool_layers: int = 4 kspace_unet_dropout_probability: float = 0.0 normalize: bool = False + conv_modulation: ModConvType = ModConvType.NONE + aux_in_features: Optional[int] = None + auxiliary_features: Optional[tuple[str, ...]] = None + log_aux: bool = False + fc_hidden_features: Optional[tuple[int, ...]] = None + fc_groups: int = 1 + fc_activation: ModConvActivation = ModConvActivation.SIGMOID + num_weights: Optional[int] = None diff --git a/direct/nn/kikinet/kikinet.py b/direct/nn/kikinet/kikinet.py index d2b9d563..3a6de51e 100644 --- a/direct/nn/kikinet/kikinet.py +++ b/direct/nn/kikinet/kikinet.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import annotations + from typing import Optional import torch @@ -18,6 +20,7 @@ import direct.data.transforms as T from direct.nn.conv.conv import Conv2d +from direct.nn.conv.modulated import ModConvActivation, ModConvType, ModulationParams from direct.nn.crossdomain.multicoil import MultiCoil from direct.nn.didn.didn import DIDN from direct.nn.mwcnn.mwcnn import MWCNN @@ -28,10 +31,18 @@ class KIKINet(nn.Module): """Based on KIKINet implementation [1]_. Modified to work with multi-coil k-space data. + Supports conditional weight modulation as proposed in [2]_. + References ---------- - .. [1] Eo, Taejoon, et al. “KIKI-Net: Cross-Domain Convolutional Neural Networks for Reconstructing Undersampled Magnetic Resonance Images.” Magnetic Resonance in Medicine, vol. 80, no. 5, Nov. 2018, pp. 2188–201. PubMed, https://doi.org/10.1002/mrm.27201. + .. [1] Eo, Taejoon, et al. "KIKI-Net: Cross-Domain Convolutional Neural Networks for Reconstructing Undersampled + Magnetic Resonance Images." Magnetic Resonance in Medicine, vol. 80, no. 5, Nov. 2018, pp. 2188-201. + https://doi.org/10.1002/mrm.27201. + + .. [2] Moriakov, N., Yiasemis, G., Sonke, J.-J. & Teuwen, J. (2026). Conditional Learned Reconstruction for + Medical Imaging. Proceedings of The 9th International Conference on Medical Imaging with Deep Learning, + PMLR 315:754-780. https://proceedings.mlr.press/v315/moriakov26a.html """ def __init__( @@ -42,6 +53,12 @@ def __init__( kspace_model_architecture: str = "DIDN", num_iter: int = 2, normalize: bool = False, + conv_modulation: ModConvType = ModConvType.NONE, + aux_in_features: Optional[int] = None, + fc_hidden_features: Optional[tuple[int] | int] = None, + fc_groups: int = 1, + fc_activation: ModConvActivation = ModConvActivation.SIGMOID, + num_weights: Optional[int] = None, **kwargs, ): """Inits :class:`KIKINet`. @@ -60,10 +77,32 @@ def __init__( Number of unrolled iterations. normalize: bool If true, input is normalised based on input scaling_factor. + conv_modulation : ModConvType + Modulation type for convolutional layers. Default: ModConvType.NONE. + aux_in_features : int, optional + Number of features in the auxiliary input for modulation. + fc_hidden_features : int or tuple of int, optional + Hidden features in the modulation MLP. + fc_groups : int + Groups for modulation MLP output. Default: 1. + fc_activation : ModConvActivation + Activation after modulation MLP. Default: ModConvActivation.SIGMOID. + num_weights : int, optional + Number of weight bases for ModConvType.SUM. kwargs: dict Keyword arguments for model architectures. """ super().__init__() + + modulation_params = ModulationParams( + modulation=conv_modulation, + aux_in_features=aux_in_features, + fc_hidden_features=fc_hidden_features, + fc_groups=fc_groups, + fc_activation=fc_activation, + num_weights=num_weights, + ) + image_model: nn.Module if image_model_architecture == "MWCNN": image_model = MWCNN( @@ -72,6 +111,7 @@ def __init__( num_scales=kwargs.get("image_mwcnn_num_scales", 4), bias=kwargs.get("image_mwcnn_bias", False), batchnorm=kwargs.get("image_mwcnn_batchnorm", False), + modulation_params=modulation_params, ) elif image_model_architecture in ["UNET", "NORMUNET"]: unet = UnetModel2d if image_model_architecture == "UNET" else NormUnetModel2d @@ -81,11 +121,12 @@ def __init__( num_filters=kwargs.get("image_unet_num_filters", 8), num_pool_layers=kwargs.get("image_unet_num_pool_layers", 4), dropout_probability=kwargs.get("image_unet_dropout_probability", 0.0), + modulation_params=modulation_params, ) else: raise NotImplementedError( - f"XPDNet is currently implemented only with image_model_architecture == 'MWCNN', 'UNET' or 'NORMUNET." - f"Got {image_model_architecture}." + f"KIKINet is currently implemented only with image_model_architecture == 'MWCNN', 'UNET' or 'NORMUNET'." + f" Got {image_model_architecture}." ) kspace_model: nn.Module @@ -96,6 +137,7 @@ def __init__( hidden_channels=kwargs.get("kspace_conv_hidden_channels", 16), n_convs=kwargs.get("kspace_conv_n_convs", 4), batchnorm=kwargs.get("kspace_conv_batchnorm", False), + modulation_params=modulation_params, ) elif kspace_model_architecture == "DIDN": kspace_model = DIDN( @@ -104,6 +146,7 @@ def __init__( hidden_channels=kwargs.get("kspace_didn_hidden_channels", 16), num_dubs=kwargs.get("kspace_didn_num_dubs", 6), num_convs_recon=kwargs.get("kspace_didn_num_convs_recon", 9), + modulation_params=modulation_params, ) elif kspace_model_architecture in ["UNET", "NORMUNET"]: unet = UnetModel2d if kspace_model_architecture == "UNET" else NormUnetModel2d @@ -113,10 +156,11 @@ def __init__( num_filters=kwargs.get("kspace_unet_num_filters", 8), num_pool_layers=kwargs.get("kspace_unet_num_pool_layers", 4), dropout_probability=kwargs.get("kspace_unet_dropout_probability", 0.0), + modulation_params=modulation_params, ) else: raise NotImplementedError( - f"XPDNet is currently implemented for kspace_model_architecture == 'CONV', 'DIDN'," + f"KIKINet is currently implemented for kspace_model_architecture == 'CONV', 'DIDN'," f" 'UNET' or 'NORMUNET'. Got kspace_model_architecture == {kspace_model_architecture}." ) @@ -131,6 +175,7 @@ def __init__( self.backward_operator = backward_operator self.num_iter = num_iter self.normalize = normalize + self.conv_modulation = conv_modulation def forward( self, @@ -138,6 +183,7 @@ def forward( sampling_mask: torch.Tensor, sensitivity_map: torch.Tensor, scaling_factor: Optional[torch.Tensor] = None, + auxiliary_data: Optional[torch.Tensor] = None, ) -> torch.Tensor: """Computes forward pass of :class:`KIKINet`. @@ -151,6 +197,8 @@ def forward( Sensitivity map of shape (N, coil, height, width, complex=2). scaling_factor: Optional[torch.Tensor] Scaling factor of shape (N,). If None, no scaling is applied. Default: None. + auxiliary_data: torch.Tensor, optional + Auxiliary data for modulation of shape (N, aux_in_features). Returns ------- @@ -163,7 +211,11 @@ def forward( kspace = kspace / (scaling_factor**2).view(-1, 1, 1, 1, 1) for idx in range(self.num_iter): - kspace = self.kspace_model_list[idx](kspace.permute(0, 1, 4, 2, 3)).permute(0, 1, 3, 4, 2) + kspace_permuted = kspace.permute(0, 1, 4, 2, 3) + if self.conv_modulation != ModConvType.NONE: + kspace = self.kspace_model_list[idx](kspace_permuted, auxiliary_data).permute(0, 1, 3, 4, 2) + else: + kspace = self.kspace_model_list[idx](kspace_permuted).permute(0, 1, 3, 4, 2) image = T.reduce_operator( self.backward_operator( @@ -178,14 +230,18 @@ def forward( self._coil_dim, ) - image = self.image_model_list[idx](image.permute(0, 3, 1, 2)).permute(0, 2, 3, 1) + if self.conv_modulation != ModConvType.NONE: + image = self.image_model_list[idx](image.permute(0, 3, 1, 2), auxiliary_data).permute(0, 2, 3, 1) + else: + image = self.image_model_list[idx](image.permute(0, 3, 1, 2)).permute(0, 2, 3, 1) if idx < self.num_iter - 1: kspace = torch.where( sampling_mask == 0, torch.tensor([0.0], dtype=image.dtype).to(image.device), self.forward_operator( - T.expand_operator(image, sensitivity_map, self._coil_dim), dim=self._spatial_dims + T.expand_operator(image, sensitivity_map, self._coil_dim), + dim=self._spatial_dims, ), ) diff --git a/direct/nn/kikinet/kikinet_engine.py b/direct/nn/kikinet/kikinet_engine.py index f4159077..3dd3ceac 100644 --- a/direct/nn/kikinet/kikinet_engine.py +++ b/direct/nn/kikinet/kikinet_engine.py @@ -52,6 +52,7 @@ def forward_function(self, data: Dict[str, Any]) -> Tuple[torch.Tensor, None]: sampling_mask=data["sampling_mask"], sensitivity_map=data["sensitivity_map"], scaling_factor=data["scaling_factor"], + auxiliary_data=self.auxiliary_data_from(data), ) # shape (batch, height, width, complex[=2]) output_kspace = None diff --git a/direct/nn/lpd/config.py b/direct/nn/lpd/config.py index 0171c384..bbc3fa80 100644 --- a/direct/nn/lpd/config.py +++ b/direct/nn/lpd/config.py @@ -1,8 +1,21 @@ -# coding=utf-8 -# Copyright (c) DIRECT Contributors +# Copyright 2025 AI for Oncology Research Group. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. from dataclasses import dataclass +from typing import Optional from direct.config.defaults import ModelConfig +from direct.nn.conv.modulated import ModConvActivation, ModConvType @dataclass @@ -28,3 +41,11 @@ class LPDNetConfig(ModelConfig): dual_unet_num_filters: int = 8 dual_unet_num_pool_layers: int = 4 dual_unet_dropout_probability: float = 0.0 + conv_modulation: ModConvType = ModConvType.NONE + aux_in_features: Optional[int] = None + auxiliary_features: Optional[tuple[str, ...]] = None + log_aux: bool = False + fc_hidden_features: Optional[tuple[int, ...]] = None + fc_groups: int = 1 + fc_activation: ModConvActivation = ModConvActivation.SIGMOID + num_weights: Optional[int] = None diff --git a/direct/nn/lpd/lpd.py b/direct/nn/lpd/lpd.py index 197dd298..7d021709 100644 --- a/direct/nn/lpd/lpd.py +++ b/direct/nn/lpd/lpd.py @@ -12,11 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import annotations + +from typing import Optional + import torch import torch.nn as nn import direct.data.transforms as T from direct.nn.conv.conv import Conv2d +from direct.nn.conv.modulated import ModConvActivation, ModConvType, ModulationParams from direct.nn.didn.didn import DIDN from direct.nn.mwcnn.mwcnn import MWCNN from direct.nn.unet.unet_2d import NormUnetModel2d, UnetModel2d @@ -26,16 +31,19 @@ class DualNet(nn.Module): """Dual Network for Learned Primal Dual Network.""" - def __init__(self, num_dual: int, **kwargs): + def __init__(self, num_dual: int, conv_modulation: ModConvType = ModConvType.NONE, **kwargs): """Inits :class:`DualNet`. Parameters ---------- num_dual: int Number of dual for LPD algorithm. + conv_modulation: ModConvType + Modulation type. Default: ModConvType.NONE. kwargs: dict """ super().__init__() + self.conv_modulation = conv_modulation if kwargs.get("dual_architectue") is None: n_hidden = kwargs.get("n_hidden") @@ -54,7 +62,12 @@ def __init__(self, num_dual: int, **kwargs): self.dual_block = kwargs.get("dual_architectue") # type: ignore @staticmethod - def compute_model_per_coil(model: nn.Module, data: torch.Tensor) -> torch.Tensor: + def compute_model_per_coil( + model: nn.Module, + data: torch.Tensor, + conv_modulation: ModConvType = ModConvType.NONE, + auxiliary_data: Optional[torch.Tensor] = None, + ) -> torch.Tensor: """Computes model per coil. Parameters @@ -63,6 +76,10 @@ def compute_model_per_coil(model: nn.Module, data: torch.Tensor) -> torch.Tensor Model to compute. data: torch.Tensor Multi-coil input. + conv_modulation: ModConvType + Modulation type. + auxiliary_data: torch.Tensor, optional + Auxiliary data for modulation. Returns ------- @@ -72,28 +89,42 @@ def compute_model_per_coil(model: nn.Module, data: torch.Tensor) -> torch.Tensor output = [] for idx in range(data.size(1)): subselected_data = data.select(1, idx) - output.append(model(subselected_data)) + if conv_modulation != ModConvType.NONE: + output.append(model(subselected_data, auxiliary_data)) + else: + output.append(model(subselected_data)) return torch.stack(output, dim=1) - def forward(self, h: torch.Tensor, forward_f: torch.Tensor, g: torch.Tensor) -> torch.Tensor: + def forward( + self, + h: torch.Tensor, + forward_f: torch.Tensor, + g: torch.Tensor, + auxiliary_data: Optional[torch.Tensor] = None, + ) -> torch.Tensor: inp = torch.cat([h, forward_f, g], dim=-1).permute(0, 1, 4, 2, 3) assert self.dual_block is not None - return self.compute_model_per_coil(self.dual_block, inp).permute(0, 1, 3, 4, 2) + return self.compute_model_per_coil(self.dual_block, inp, self.conv_modulation, auxiliary_data).permute( + 0, 1, 3, 4, 2 + ) class PrimalNet(nn.Module): """Primal Network for Learned Primal Dual Network.""" - def __init__(self, num_primal: int, **kwargs): + def __init__(self, num_primal: int, conv_modulation: ModConvType = ModConvType.NONE, **kwargs): """Inits :class:`PrimalNet`. Parameters ---------- num_primal: int Number of primal for LPD algorithm. + conv_modulation: ModConvType + Modulation type. Default: ModConvType.NONE. """ super().__init__() + self.conv_modulation = conv_modulation if kwargs.get("primal_architectue") is None: n_hidden = kwargs.get("n_hidden") @@ -111,19 +142,33 @@ def __init__(self, num_primal: int, **kwargs): else: self.primal_block = kwargs.get("primal_architectue") # type: ignore - def forward(self, f: torch.Tensor, backward_h: torch.Tensor) -> torch.Tensor: + def forward( + self, + f: torch.Tensor, + backward_h: torch.Tensor, + auxiliary_data: Optional[torch.Tensor] = None, + ) -> torch.Tensor: inp = torch.cat([f, backward_h], dim=-1).permute(0, 3, 1, 2) assert self.primal_block is not None + if self.conv_modulation != ModConvType.NONE: + return self.primal_block(inp, auxiliary_data).permute(0, 2, 3, 1) return self.primal_block(inp).permute(0, 2, 3, 1) class LPDNet(nn.Module): """Learned Primal Dual network implementation inspired by [1]_. + Supports conditional weight modulation as proposed in [2]_. + References ---------- - .. [1] Adler, Jonas, and Ozan Öktem. “Learned Primal-Dual Reconstruction.” IEEE Transactions on Medical Imaging, vol. 37, no. 6, June 2018, pp. 1322–32. arXiv.org, https://doi.org/10.1109/TMI.2018.2799231. + .. [1] Adler, Jonas, and Ozan Oktem. "Learned Primal-Dual Reconstruction." IEEE Transactions on Medical Imaging, + vol. 37, no. 6, June 2018, pp. 1322-32. https://doi.org/10.1109/TMI.2018.2799231. + + .. [2] Moriakov, N., Yiasemis, G., Sonke, J.-J. & Teuwen, J. (2026). Conditional Learned Reconstruction for + Medical Imaging. Proceedings of The 9th International Conference on Medical Imaging with Deep Learning, + PMLR 315:754-780. https://proceedings.mlr.press/v315/moriakov26a.html """ def __init__( @@ -135,6 +180,12 @@ def __init__( num_dual: int, primal_model_architecture: str = "MWCNN", dual_model_architecture: str = "DIDN", + conv_modulation: ModConvType = ModConvType.NONE, + aux_in_features: Optional[int] = None, + fc_hidden_features: Optional[tuple[int] | int] = None, + fc_groups: int = 1, + fc_activation: ModConvActivation = ModConvActivation.SIGMOID, + num_weights: Optional[int] = None, **kwargs, ): """Inits :class:`LPDNet`. @@ -155,6 +206,18 @@ def __init__( Primal model architecture. Currently only implemented for MWCNN and (NORM)UNET. Default: 'MWCNN'. dual_model_architecture: str Dual model architecture. Currently only implemented for CONV and DIDN and (NORM)UNET. Default: 'DIDN'. + conv_modulation : ModConvType + Modulation type for convolutional layers. Default: ModConvType.NONE. + aux_in_features : int, optional + Number of features in the auxiliary input for modulation. + fc_hidden_features : int or tuple of int, optional + Hidden features in the modulation MLP. + fc_groups : int + Groups for modulation MLP output. Default: 1. + fc_activation : ModConvActivation + Activation after modulation MLP. Default: ModConvActivation.SIGMOID. + num_weights : int, optional + Number of weight bases for ModConvType.SUM. kwargs: dict Keyword arguments for model architectures. """ @@ -165,6 +228,16 @@ def __init__( self.num_iter = num_iter self.num_primal = num_primal self.num_dual = num_dual + self.conv_modulation = conv_modulation + + modulation_params = ModulationParams( + modulation=conv_modulation, + aux_in_features=aux_in_features, + fc_hidden_features=fc_hidden_features, + fc_groups=fc_groups, + fc_activation=fc_activation, + num_weights=num_weights, + ) primal_model: nn.Module if primal_model_architecture == "MWCNN": @@ -176,6 +249,7 @@ def __init__( num_scales=kwargs.get("primal_mwcnn_num_scales", 4), bias=kwargs.get("primal_mwcnn_bias", False), batchnorm=kwargs.get("primal_mwcnn_batchnorm", False), + modulation_params=modulation_params, ), nn.Conv2d(2 * (num_primal + 1), 2 * num_primal, kernel_size=1), ] @@ -188,11 +262,12 @@ def __init__( num_filters=kwargs.get("primal_unet_num_filters", 8), num_pool_layers=kwargs.get("primal_unet_num_pool_layers", 4), dropout_probability=kwargs.get("primal_unet_dropout_probability", 0.0), + modulation_params=modulation_params, ) else: raise NotImplementedError( - f"XPDNet is currently implemented only with primal_model_architecture == 'MWCNN', 'UNET' or 'NORMUNET." - f"Got {primal_model_architecture}." + f"LPDNet is currently implemented only with primal_model_architecture == 'MWCNN', 'UNET' or 'NORMUNET'." + f" Got {primal_model_architecture}." ) dual_model: nn.Module if dual_model_architecture == "CONV": @@ -202,6 +277,7 @@ def __init__( hidden_channels=kwargs.get("dual_conv_hidden_channels", 16), n_convs=kwargs.get("dual_conv_n_convs", 4), batchnorm=kwargs.get("dual_conv_batchnorm", False), + modulation_params=modulation_params, ) elif dual_model_architecture == "DIDN": dual_model = DIDN( @@ -210,6 +286,7 @@ def __init__( hidden_channels=kwargs.get("dual_didn_hidden_channels", 16), num_dubs=kwargs.get("dual_didn_num_dubs", 6), num_convs_recon=kwargs.get("dual_didn_num_convs_recon", 9), + modulation_params=modulation_params, ) elif dual_model_architecture in ["UNET", "NORMUNET"]: unet = UnetModel2d if dual_model_architecture == "UNET" else NormUnetModel2d @@ -219,10 +296,11 @@ def __init__( num_filters=kwargs.get("dual_unet_num_filters", 8), num_pool_layers=kwargs.get("dual_unet_num_pool_layers", 4), dropout_probability=kwargs.get("dual_unet_dropout_probability", 0.0), + modulation_params=modulation_params, ) else: raise NotImplementedError( - f"XPDNet is currently implemented for dual_model_architecture == 'CONV', 'DIDN'," + f"LPDNet is currently implemented for dual_model_architecture == 'CONV', 'DIDN'," f" 'UNET' or 'NORMUNET'. Got dual_model_architecture == {dual_model_architecture}." ) @@ -231,22 +309,47 @@ def __init__( self._spatial_dims = (2, 3) self.primal_net = nn.ModuleList( - [PrimalNet(num_primal, primal_architectue=primal_model) for _ in range(num_iter)] + [ + PrimalNet( + num_primal, + conv_modulation=conv_modulation, + primal_architectue=primal_model, + ) + for _ in range(num_iter) + ] + ) + self.dual_net = nn.ModuleList( + [ + DualNet( + num_dual, + conv_modulation=conv_modulation, + dual_architectue=dual_model, + ) + for _ in range(num_iter) + ] ) - self.dual_net = nn.ModuleList([DualNet(num_dual, dual_architectue=dual_model) for _ in range(num_iter)]) def _forward_operator( - self, image: torch.Tensor, sampling_mask: torch.Tensor, sensitivity_map: torch.Tensor + self, + image: torch.Tensor, + sampling_mask: torch.Tensor, + sensitivity_map: torch.Tensor, ) -> torch.Tensor: forward = torch.where( sampling_mask == 0, torch.tensor([0.0], dtype=image.dtype).to(image.device), - self.forward_operator(T.expand_operator(image, sensitivity_map, self._coil_dim), dim=self._spatial_dims), + self.forward_operator( + T.expand_operator(image, sensitivity_map, self._coil_dim), + dim=self._spatial_dims, + ), ) return forward def _backward_operator( - self, kspace: torch.Tensor, sampling_mask: torch.Tensor, sensitivity_map: torch.Tensor + self, + kspace: torch.Tensor, + sampling_mask: torch.Tensor, + sensitivity_map: torch.Tensor, ) -> torch.Tensor: backward = T.reduce_operator( self.backward_operator( @@ -267,6 +370,7 @@ def forward( masked_kspace: torch.Tensor, sensitivity_map: torch.Tensor, sampling_mask: torch.Tensor, + auxiliary_data: Optional[torch.Tensor] = None, ) -> torch.Tensor: """Computes forward pass of :class:`LPDNet`. @@ -278,6 +382,8 @@ def forward( Sensitivity map of shape (N, coil, height, width, complex=2). sampling_mask: torch.Tensor Sampling mask of shape (N, 1, height, width, 1). + auxiliary_data: torch.Tensor, optional + Auxiliary data for modulation of shape (N, aux_in_features). Returns ------- @@ -292,13 +398,18 @@ def forward( # Dual f_2 = primal_buffer[..., 2:4].clone() dual_buffer = self.dual_net[curr_iter]( - dual_buffer, self._forward_operator(f_2, sampling_mask, sensitivity_map), masked_kspace + dual_buffer, + self._forward_operator(f_2, sampling_mask, sensitivity_map), + masked_kspace, + auxiliary_data, ) # Primal h_1 = dual_buffer[..., 0:2].clone() primal_buffer = self.primal_net[curr_iter]( - primal_buffer, self._backward_operator(h_1, sampling_mask, sensitivity_map) + primal_buffer, + self._backward_operator(h_1, sampling_mask, sensitivity_map), + auxiliary_data, ) output = primal_buffer[..., 0:2] diff --git a/direct/nn/lpd/lpd_engine.py b/direct/nn/lpd/lpd_engine.py index 6c77ab45..a169a156 100644 --- a/direct/nn/lpd/lpd_engine.py +++ b/direct/nn/lpd/lpd_engine.py @@ -52,5 +52,6 @@ def forward_function(self, data: Dict[str, Any]) -> Tuple[torch.Tensor, None]: masked_kspace=data["masked_kspace"], sampling_mask=data["sampling_mask"], sensitivity_map=data["sensitivity_map"], + auxiliary_data=self.auxiliary_data_from(data), ) # shape (batch, height, width) return output_image, None diff --git a/direct/nn/mri_models.py b/direct/nn/mri_models.py index f6806547..d5d9491d 100644 --- a/direct/nn/mri_models.py +++ b/direct/nn/mri_models.py @@ -100,6 +100,17 @@ def forward_function(self, data: Dict[str, Any]) -> Tuple[TensorOrNone, TensorOr """ raise NotImplementedError("Must be implemented by child class.") + @staticmethod + def auxiliary_data_from(data: Dict[str, Any]) -> TensorOrNone: + """Return auxiliary conditioning from a batch dict, if present.""" + return data.get("auxiliary_data") + + def _attach_auxiliary_data(self, data: Dict[str, Any]) -> None: + """Populate ``auxiliary_data`` when modulated convolutions are enabled.""" + from direct.nn.conv.modulated import prepare_auxiliary_data + + data["auxiliary_data"] = prepare_auxiliary_data(data, getattr(self.cfg, "model", None)) + def _do_iteration( self, data: Dict[str, Any], @@ -131,6 +142,7 @@ def _do_iteration( regularizer_fns = {} data = dict_to_device(data, self.device) + self._attach_auxiliary_data(data) output_image: TensorOrNone output_kspace: TensorOrNone @@ -864,7 +876,12 @@ def reconstruct_volumes( # type: ignore # Maybe not needed. del data yield ( - (curr_volume, curr_target, reduce_list_of_dicts(loss_dict_list), filename) + ( + curr_volume, + curr_target, + reduce_list_of_dicts(loss_dict_list), + filename, + ) if add_target else ( curr_volume, @@ -994,7 +1011,11 @@ def compute_loss_on_data( for key, value in loss_dict.items(): if "kspace" in key: if output_kspace is not None: - output, target, reconstruction_size = output_kspace, data["kspace"], None + output, target, reconstruction_size = ( + output_kspace, + data["kspace"], + None, + ) else: continue else: @@ -1021,14 +1042,19 @@ def _forward_operator(self, image, sensitivity_map, sampling_mask): def _backward_operator(self, kspace, sensitivity_map, sampling_mask): return T.reduce_operator( - self.backward_operator(T.apply_mask(kspace, sampling_mask, return_mask=False), dim=self._spatial_dims), + self.backward_operator( + T.apply_mask(kspace, sampling_mask, return_mask=False), + dim=self._spatial_dims, + ), sensitivity_map, dim=self._coil_dim, ) def _crop_volume( - source: torch.Tensor, target: torch.Tensor, resolution: Union[List[int], Tuple[int, ...]] + source: torch.Tensor, + target: torch.Tensor, + resolution: Union[List[int], Tuple[int, ...]], ) -> Tuple[torch.Tensor, torch.Tensor]: """2D source/target cropper. @@ -1118,7 +1144,8 @@ def _process_output( def _compute_resolution( - key: Optional[str], reconstruction_size: Optional[Union[List[int], Tuple[int]]] = None + key: Optional[str], + reconstruction_size: Optional[Union[List[int], Tuple[int]]] = None, ) -> Union[List[int], None]: """Computes resolution. diff --git a/direct/nn/mwcnn/mwcnn.py b/direct/nn/mwcnn/mwcnn.py index 1df89b64..3a737357 100644 --- a/direct/nn/mwcnn/mwcnn.py +++ b/direct/nn/mwcnn/mwcnn.py @@ -11,13 +11,22 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from collections import OrderedDict -from typing import List, Optional, Tuple +from __future__ import annotations + +from typing import Optional, Tuple, cast import torch import torch.nn as nn import torch.nn.functional as F +from direct.nn.conv.modulated import ( + ModConv2dBias, + ModConvActivation, + ModConvType, + ModulationParams, + mod_conv2d, +) + class DWT(nn.Module): """2D Discrete Wavelet Transform as implemented in [1]_. @@ -25,7 +34,7 @@ class DWT(nn.Module): References ---------- - .. [1] Liu, Pengju, et al. “Multi-Level Wavelet-CNN for Image Restoration.” ArXiv:1805.07071 [Cs], May 2018. arXiv.org, http://arxiv.org/abs/1805.07071. + .. [1] Liu, Pengju, et al. "Multi-Level Wavelet-CNN for Image Restoration." ArXiv:1805.07071 [Cs], May 2018. arXiv.org, http://arxiv.org/abs/1805.07071. """ def __init__(self): @@ -66,7 +75,7 @@ class IWT(nn.Module): References ---------- - .. [1] Liu, Pengju, et al. “Multi-Level Wavelet-CNN for Image Restoration.” ArXiv:1805.07071 [Cs], May 2018. arXiv.org, http://arxiv.org/abs/1805.07071. + .. [1] Liu, Pengju, et al. "Multi-Level Wavelet-CNN for Image Restoration." ArXiv:1805.07071 [Cs], May 2018. arXiv.org, http://arxiv.org/abs/1805.07071. """ def __init__(self): @@ -89,7 +98,11 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: IWT of `x`. """ batch, in_channel, in_height, in_width = x.size() - out_channel, out_height, out_width = int(in_channel / (self._r**2)), self._r * in_height, self._r * in_width + out_channel, out_height, out_width = ( + int(in_channel / (self._r**2)), + self._r * in_height, + self._r * in_width, + ) x1 = x[:, 0:out_channel, :, :] / 2 x2 = x[:, out_channel : out_channel * 2, :, :] / 2 @@ -112,7 +125,7 @@ class ConvBlock(nn.Module): References ---------- - .. [1] Liu, Pengju, et al. “Multi-Level Wavelet-CNN for Image Restoration.” ArXiv:1805.07071 [Cs], May 2018. arXiv.org, http://arxiv.org/abs/1805.07071. + .. [1] Liu, Pengju, et al. "Multi-Level Wavelet-CNN for Image Restoration." ArXiv:1805.07071 [Cs], May 2018. arXiv.org, http://arxiv.org/abs/1805.07071. """ def __init__( @@ -124,6 +137,13 @@ def __init__( batchnorm: bool = False, activation: nn.Module = nn.ReLU(True), scale: Optional[float] = 1.0, + modulation: ModConvType = ModConvType.NONE, + modulation_params: Optional[ModulationParams] = None, + aux_in_features: Optional[int] = None, + fc_hidden_features: Optional[tuple[int] | int] = None, + fc_groups: int = 1, + fc_activation: ModConvActivation = ModConvActivation.SIGMOID, + num_weights: Optional[int] = None, ): """Inits :class:`ConvBlock`. @@ -143,50 +163,75 @@ def __init__( Activation function. Default: nn.ReLU(True). scale: float, optional Scale. Default: 1.0. + modulation : ModConvType + Modulation type. Default: ModConvType.NONE. + aux_in_features : int, optional + Auxiliary input features for modulation. + fc_hidden_features : int or tuple of int, optional + Hidden features for modulation MLP. + fc_groups : int + Groups for modulation MLP. Default: 1. + fc_activation : ModConvActivation + Activation for modulation MLP. Default: ModConvActivation.SIGMOID. + num_weights : int, optional + Number of weight bases for ModConvType.SUM. """ super().__init__() - net: List[nn.Module] = [] - net.append( - nn.Conv2d( - in_channels=in_channels, - out_channels=out_channels, - kernel_size=kernel_size, - bias=bias, - padding=kernel_size // 2, + if modulation_params is None: + modulation_params = ModulationParams( + modulation=modulation, + aux_in_features=aux_in_features, + fc_hidden_features=fc_hidden_features, + fc_groups=fc_groups, + fc_activation=fc_activation, + num_weights=num_weights, ) + self.modulation = modulation_params.modulation + self.conv = mod_conv2d( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=kernel_size, + padding=kernel_size // 2, + modulation_params=modulation_params, + bias=ModConv2dBias.PARAM if bias else ModConv2dBias.NONE, ) - if batchnorm: - net.append(nn.BatchNorm2d(num_features=out_channels, eps=1e-4, momentum=0.95)) - net.append(activation) - - self.net = nn.Sequential(*net) + self.batchnorm = nn.BatchNorm2d(num_features=out_channels, eps=1e-4, momentum=0.95) if batchnorm else None + self.activation = activation self.scale = scale - def forward(self, x: torch.Tensor) -> torch.Tensor: + def forward(self, x: torch.Tensor, y: Optional[torch.Tensor] = None) -> torch.Tensor: """Performs forward pass of :class:`ConvBlock`. Parameters ---------- x: torch.Tensor Input with shape (N, C, H, W). + y: torch.Tensor, optional + Auxiliary signal for modulation. Returns ------- output: torch.Tensor Output with shape (N, C', H', W'). """ - output = self.net(x) * self.scale + if self.modulation != ModConvType.NONE: + output = self.conv(x, y) + else: + output = self.conv(x) + if self.batchnorm is not None: + output = self.batchnorm(output) + output = self.activation(output) * self.scale return output class DilatedConvBlock(nn.Module): - """Double dilated Convolution Block fpr :class:`MWCNN` as implemented in [1]_. + """Double dilated Convolution Block for :class:`MWCNN` as implemented in [1]_. References ---------- - .. [1] Liu, Pengju, et al. “Multi-Level Wavelet-CNN for Image Restoration.” ArXiv:1805.07071 [Cs], May 2018. arXiv.org, http://arxiv.org/abs/1805.07071. + .. [1] Liu, Pengju, et al. "Multi-Level Wavelet-CNN for Image Restoration." ArXiv:1805.07071 [Cs], May 2018. arXiv.org, http://arxiv.org/abs/1805.07071. """ def __init__( @@ -199,6 +244,13 @@ def __init__( batchnorm: bool = False, activation: nn.Module = nn.ReLU(True), scale: Optional[float] = 1.0, + modulation: ModConvType = ModConvType.NONE, + modulation_params: Optional[ModulationParams] = None, + aux_in_features: Optional[int] = None, + fc_hidden_features: Optional[tuple[int] | int] = None, + fc_groups: int = 1, + fc_activation: ModConvActivation = ModConvActivation.SIGMOID, + num_weights: Optional[int] = None, ): """Inits :class:`DilatedConvBlock`. @@ -220,55 +272,90 @@ def __init__( Activation function. Default: nn.ReLU(True). scale: float, optional Scale. Default: 1.0. + modulation : ModConvType + Modulation type. Default: ModConvType.NONE. + aux_in_features : int, optional + Auxiliary input features for modulation. + fc_hidden_features : int or tuple of int, optional + Hidden features for modulation MLP. + fc_groups : int + Groups for modulation MLP. Default: 1. + fc_activation : ModConvActivation + Activation for modulation MLP. Default: ModConvActivation.SIGMOID. + num_weights : int, optional + Number of weight bases for ModConvType.SUM. """ super().__init__() - net: List[nn.Module] = [] - net.append( - nn.Conv2d( - in_channels=in_channels, - out_channels=in_channels, - kernel_size=kernel_size, - bias=bias, - dilation=dilations[0], - padding=kernel_size // 2 + dilations[0] - 1, - ) - ) - if batchnorm: - net.append(nn.BatchNorm2d(num_features=in_channels, eps=1e-4, momentum=0.95)) - net.append(activation) if out_channels is None: out_channels = in_channels - net.append( - nn.Conv2d( - in_channels=in_channels, - out_channels=out_channels, - kernel_size=kernel_size, - bias=bias, - dilation=dilations[1], - padding=kernel_size // 2 + dilations[1] - 1, + + if modulation_params is None: + modulation_params = ModulationParams( + modulation=modulation, + aux_in_features=aux_in_features, + fc_hidden_features=fc_hidden_features, + fc_groups=fc_groups, + fc_activation=fc_activation, + num_weights=num_weights, ) + self.modulation = modulation_params.modulation + bias_type = ModConv2dBias.PARAM if bias else ModConv2dBias.NONE + self.conv1 = mod_conv2d( + in_channels=in_channels, + out_channels=in_channels, + kernel_size=kernel_size, + dilation=dilations[0], + padding=kernel_size // 2 + dilations[0] - 1, + modulation_params=modulation_params, + bias=bias_type, ) - if batchnorm: - net.append(nn.BatchNorm2d(num_features=in_channels, eps=1e-4, momentum=0.95)) - net.append(activation) + self.bn1 = nn.BatchNorm2d(num_features=in_channels, eps=1e-4, momentum=0.95) if batchnorm else None + self.act1 = activation + + self.conv2 = mod_conv2d( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=kernel_size, + dilation=dilations[1], + padding=kernel_size // 2 + dilations[1] - 1, + modulation_params=modulation_params, + bias=bias_type, + ) + self.bn2 = nn.BatchNorm2d(num_features=out_channels, eps=1e-4, momentum=0.95) if batchnorm else None + self.act2 = activation - self.net = nn.Sequential(*net) self.scale = scale - def forward(self, x: torch.Tensor) -> torch.Tensor: + def forward(self, x: torch.Tensor, y: Optional[torch.Tensor] = None) -> torch.Tensor: """Performs forward pass of :class:`DilatedConvBlock`. Parameters ---------- x: torch.Tensor Input with shape (N, C, H, W). + y: torch.Tensor, optional + Auxiliary signal for modulation. Returns ------- output: torch.Tensor Output with shape (N, C', H', W'). """ - output = self.net(x) * self.scale + if self.modulation != ModConvType.NONE: + output = self.conv1(x, y) + else: + output = self.conv1(x) + if self.bn1 is not None: + output = self.bn1(output) + output = self.act1(output) + + if self.modulation != ModConvType.NONE: + output = self.conv2(output, y) + else: + output = self.conv2(output) + if self.bn2 is not None: + output = self.bn2(output) + output = self.act2(output) * self.scale return output @@ -278,7 +365,7 @@ class MWCNN(nn.Module): References ---------- - .. [1] Liu, Pengju, et al. “Multi-Level Wavelet-CNN for Image Restoration.” ArXiv:1805.07071 [Cs], May 2018. arXiv.org, http://arxiv.org/abs/1805.07071. + .. [1] Liu, Pengju, et al. "Multi-Level Wavelet-CNN for Image Restoration." ArXiv:1805.07071 [Cs], May 2018. arXiv.org, http://arxiv.org/abs/1805.07071. """ def __init__( @@ -289,6 +376,13 @@ def __init__( bias: bool = True, batchnorm: bool = False, activation: nn.Module = nn.ReLU(True), + modulation: ModConvType = ModConvType.NONE, + modulation_params: Optional[ModulationParams] = None, + aux_in_features: Optional[int] = None, + fc_hidden_features: Optional[tuple[int] | int] = None, + fc_groups: int = 1, + fc_activation: ModConvActivation = ModConvActivation.SIGMOID, + num_weights: Optional[int] = None, ): """Inits :class:`MWCNN`. @@ -306,45 +400,70 @@ def __init__( If True, a batchnorm layer is added after each convolution. Default: False. activation: nn.Module Activation function applied after each convolution. Default: nn.ReLU(). + modulation : ModConvType + Modulation type. Default: ModConvType.NONE. + aux_in_features : int, optional + Auxiliary input features for modulation. + fc_hidden_features : int or tuple of int, optional + Hidden features for modulation MLP. + fc_groups : int + Groups for modulation MLP. Default: 1. + fc_activation : ModConvActivation + Activation for modulation MLP. Default: ModConvActivation.SIGMOID. + num_weights : int, optional + Number of weight bases for ModConvType.SUM. """ super().__init__() self._kernel_size = 3 + if modulation_params is None: + modulation_params = ModulationParams( + modulation=modulation, + aux_in_features=aux_in_features, + fc_hidden_features=fc_hidden_features, + fc_groups=fc_groups, + fc_activation=fc_activation, + num_weights=num_weights, + ) + self.modulation = modulation_params.modulation self.DWT = DWT() self.IWT = IWT() + modulation_params = ModulationParams( + modulation=modulation, + aux_in_features=aux_in_features, + fc_hidden_features=fc_hidden_features, + fc_groups=fc_groups, + fc_activation=fc_activation, + num_weights=num_weights, + ) + self.down = nn.ModuleList() for idx in range(0, num_scales): in_channels = input_channels if idx == 0 else first_conv_hidden_channels * 2 ** (idx + 1) out_channels = first_conv_hidden_channels * 2**idx dilations = (2, 1) if idx != num_scales - 1 else (2, 3) self.down.append( - nn.Sequential( - OrderedDict( - [ - ( - f"convblock{idx}", - ConvBlock( - in_channels=in_channels, - out_channels=out_channels, - kernel_size=self._kernel_size, - bias=bias, - batchnorm=batchnorm, - activation=activation, - ), - ), - ( - f"dilconvblock{idx}", - DilatedConvBlock( - in_channels=out_channels, - dilations=dilations, - kernel_size=self._kernel_size, - bias=bias, - batchnorm=batchnorm, - activation=activation, - ), - ), - ] - ) + nn.ModuleDict( + { + "convblock": ConvBlock( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=self._kernel_size, + bias=bias, + batchnorm=batchnorm, + activation=activation, + modulation_params=modulation_params, + ), + "dilconvblock": DilatedConvBlock( + in_channels=out_channels, + dilations=dilations, + kernel_size=self._kernel_size, + bias=bias, + batchnorm=batchnorm, + activation=activation, + modulation_params=modulation_params, + ), + } ) ) self.up = nn.ModuleList() @@ -353,33 +472,27 @@ def __init__( out_channels = input_channels if idx == 0 else first_conv_hidden_channels * 2 ** (idx + 1) dilations = (2, 1) if idx != num_scales - 1 else (3, 2) self.up.append( - nn.Sequential( - OrderedDict( - [ - ( - f"invdilconvblock{num_scales - 2 - idx}", - DilatedConvBlock( - in_channels=in_channels, - dilations=dilations, - kernel_size=self._kernel_size, - bias=bias, - batchnorm=batchnorm, - activation=activation, - ), - ), - ( - f"invconvblock{num_scales - 2 - idx}", - ConvBlock( - in_channels=in_channels, - out_channels=out_channels, - kernel_size=self._kernel_size, - bias=bias, - batchnorm=batchnorm, - activation=activation, - ), - ), - ] - ) + nn.ModuleDict( + { + "dilconvblock": DilatedConvBlock( + in_channels=in_channels, + dilations=dilations, + kernel_size=self._kernel_size, + bias=bias, + batchnorm=batchnorm, + activation=activation, + modulation_params=modulation_params, + ), + "convblock": ConvBlock( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=self._kernel_size, + bias=bias, + batchnorm=batchnorm, + activation=activation, + modulation_params=modulation_params, + ), + } ) ) self.num_scales = num_scales @@ -387,11 +500,10 @@ def __init__( @staticmethod def pad(x: torch.Tensor) -> torch.Tensor: padding = [0, 0, 0, 0] - if x.shape[-2] % 2 != 0: - padding[3] = 1 # Padding right - width + padding[3] = 1 if x.shape[-1] % 2 != 0: - padding[1] = 1 # Padding bottom - height + padding[1] = 1 if sum(padding) != 0: x = F.pad(x, padding, "reflect") return x @@ -399,20 +511,26 @@ def pad(x: torch.Tensor) -> torch.Tensor: @staticmethod def crop_to_shape(x: torch.Tensor, shape: tuple) -> torch.Tensor: h, w = x.shape[-2:] - if h > shape[0]: x = x[:, :, : shape[0], :] if w > shape[1]: x = x[:, :, :, : shape[1]] return x - def forward(self, input_tensor: torch.Tensor, res: bool = False) -> torch.Tensor: + def forward( + self, + input_tensor: torch.Tensor, + y: Optional[torch.Tensor] = None, + res: bool = False, + ) -> torch.Tensor: """Computes forward pass of :class:`MWCNN`. Parameters ---------- input_tensor: torch.Tensor Input tensor. + y: torch.Tensor, optional + Auxiliary signal for modulation of shape (N, aux_in_features). res: bool If True, residual connection is applied to the output. Default: False. @@ -424,23 +542,34 @@ def forward(self, input_tensor: torch.Tensor, res: bool = False) -> torch.Tensor res_values = [] x = self.pad(input_tensor.clone()) for idx in range(self.num_scales): + down_block = cast(nn.ModuleDict, self.down[idx]) if idx == 0: - x = self.pad(self.down[idx](x)) + x = down_block["convblock"](x, y) + x = down_block["dilconvblock"](x, y) + x = self.pad(x) res_values.append(x) elif idx == self.num_scales - 1: - x = self.down[idx](self.DWT(x)) + x = down_block["convblock"](self.DWT(x), y) + x = down_block["dilconvblock"](x, y) else: - x = self.pad(self.down[idx](self.DWT(x))) + x = down_block["convblock"](self.DWT(x), y) + x = down_block["dilconvblock"](x, y) + x = self.pad(x) res_values.append(x) for idx in range(self.num_scales): + up_block = cast(nn.ModuleDict, self.up[idx]) if idx != self.num_scales - 1: + x = up_block["dilconvblock"](x, y) + x = up_block["convblock"](x, y) x = ( - self.crop_to_shape(self.IWT(self.up[idx](x)), res_values[self.num_scales - 2 - idx].shape[-2:]) + self.crop_to_shape(self.IWT(x), res_values[self.num_scales - 2 - idx].shape[-2:]) + res_values[self.num_scales - 2 - idx] ) else: - x = self.crop_to_shape(self.up[idx](x), input_tensor.shape[-2:]) + x = up_block["dilconvblock"](x, y) + x = up_block["convblock"](x, y) + x = self.crop_to_shape(x, input_tensor.shape[-2:]) if res: x += input_tensor return x diff --git a/direct/nn/recurrent/recurrent.py b/direct/nn/recurrent/recurrent.py index fb33720e..49efdb16 100644 --- a/direct/nn/recurrent/recurrent.py +++ b/direct/nn/recurrent/recurrent.py @@ -142,7 +142,13 @@ def forward( conv_skip: List[torch.Tensor] = [] if previous_state is None: - batch_size, spatial_size = cell_input.size(0), (cell_input.size(2), cell_input.size(3)) + batch_size, spatial_size = ( + cell_input.size(0), + ( + cell_input.size(2), + cell_input.size(3), + ), + ) state_size = [batch_size, self.hidden_channels] + list(spatial_size) + [self.num_layers] previous_state = torch.zeros(*state_size, dtype=cell_input.dtype).to(cell_input.device) diff --git a/direct/nn/ssl/mri_models.py b/direct/nn/ssl/mri_models.py index 35725be0..f4f08d5e 100644 --- a/direct/nn/ssl/mri_models.py +++ b/direct/nn/ssl/mri_models.py @@ -117,7 +117,11 @@ def log_first_training_example_and_model(self, data: dict[str, Any]) -> None: """ storage = get_event_storage() - self.logger.info("First case: slice_no: %s, filename: %s.", data["slice_no"][0], data["filename"][0]) + self.logger.info( + "First case: slice_no: %s, filename: %s.", + data["slice_no"][0], + data["filename"][0], + ) if "input_sampling_mask" in data: first_input_sampling_mask = data["input_sampling_mask"][0][0] @@ -209,6 +213,7 @@ def _do_iteration( regularizer_fns = {} data = dict_to_device(data, self.device) + self._attach_auxiliary_data(data) # Get the k-space and mask which differ during training and inference for SSL kspace = data["input_kspace"] if self.model.training else data["masked_kspace"] @@ -387,6 +392,7 @@ def _do_iteration( regularizer_fns = {} data = dict_to_device(data, self.device) + self._attach_auxiliary_data(data) # Get a boolean indicating if the sample is for SSL training # This will expect the input data to contain the keys "input_kspace" and "input_sampling_mask" if SSL training diff --git a/direct/nn/transformers/uformer.py b/direct/nn/transformers/uformer.py index 4c7e8211..74fe2766 100644 --- a/direct/nn/transformers/uformer.py +++ b/direct/nn/transformers/uformer.py @@ -37,7 +37,12 @@ from direct.nn.transformers.utils import DropoutPath, init_weights, norm, pad_to_square, unnorm, unpad_to_original from direct.types import DirectEnum -__all__ = ["AttentionTokenProjectionType", "LeWinTransformerMLPTokenType", "UFormer", "UFormerModel"] +__all__ = [ + "AttentionTokenProjectionType", + "LeWinTransformerMLPTokenType", + "UFormer", + "UFormerModel", +] class ECALayer1d(nn.Module): @@ -250,13 +255,28 @@ def __init__( self.heads = heads pad = (kernel_size - q_stride) // 2 self.to_q = SepConv2d( - in_channels=dim, out_channels=inner_dim, kernel_size=kernel_size, stride=q_stride, padding=pad, bias=bias + in_channels=dim, + out_channels=inner_dim, + kernel_size=kernel_size, + stride=q_stride, + padding=pad, + bias=bias, ) self.to_k = SepConv2d( - in_channels=dim, out_channels=inner_dim, kernel_size=kernel_size, stride=k_stride, padding=pad, bias=bias + in_channels=dim, + out_channels=inner_dim, + kernel_size=kernel_size, + stride=k_stride, + padding=pad, + bias=bias, ) self.to_v = SepConv2d( - in_channels=dim, out_channels=inner_dim, kernel_size=kernel_size, stride=v_stride, padding=pad, bias=bias + in_channels=dim, + out_channels=inner_dim, + kernel_size=kernel_size, + stride=v_stride, + padding=pad, + bias=bias, ) def forward( @@ -471,7 +491,10 @@ def __init__( self.softmax = nn.Softmax(dim=-1) def forward( - self, x: torch.Tensor, attn_kv: Optional[torch.Tensor] = None, mask: Optional[torch.Tensor] = None + self, + x: torch.Tensor, + attn_kv: Optional[torch.Tensor] = None, + mask: Optional[torch.Tensor] = None, ) -> torch.Tensor: """Performs forward pass of :class:`WindowAttentionModule`. @@ -586,7 +609,10 @@ def __init__( self.softmax = nn.Softmax(dim=-1) def forward( - self, x: torch.Tensor, attn_kv: Optional[torch.Tensor] = None, mask: Optional[torch.Tensor] = None + self, + x: torch.Tensor, + attn_kv: Optional[torch.Tensor] = None, + mask: Optional[torch.Tensor] = None, ) -> torch.Tensor: """Performs the forward pass of :class:`AttentionModule`. @@ -713,7 +739,11 @@ class LeFF(nn.Module): """ def __init__( - self, dim: int = 32, hidden_dim: int = 128, act_layer: type[nn.Module] = nn.GELU, use_eca: bool = False + self, + dim: int = 32, + hidden_dim: int = 128, + act_layer: type[nn.Module] = nn.GELU, + use_eca: bool = False, ) -> None: """Inits :class:`LeFF`. @@ -731,7 +761,15 @@ def __init__( super().__init__() self.linear1 = nn.Sequential(nn.Linear(dim, hidden_dim), act_layer()) self.dwconv = nn.Sequential( - nn.Conv2d(hidden_dim, hidden_dim, groups=hidden_dim, kernel_size=3, stride=1, padding=1), act_layer() + nn.Conv2d( + hidden_dim, + hidden_dim, + groups=hidden_dim, + kernel_size=3, + stride=1, + padding=1, + ), + act_layer(), ) self.linear2 = nn.Sequential(nn.Linear(hidden_dim, dim)) self.dim = dim @@ -792,7 +830,11 @@ def window_partition(x: torch.Tensor, win_size: int, dilation_rate: int = 1) -> if dilation_rate != 1: x = x.permute(0, 3, 1, 2) # B, C, H, W x = F.unfold( - x, kernel_size=win_size, dilation=dilation_rate, padding=4 * (dilation_rate - 1), stride=win_size + x, + kernel_size=win_size, + dilation=dilation_rate, + padding=4 * (dilation_rate - 1), + stride=win_size, ) # B, C*Wh*Ww, H/Wh*W/Ww windows = x.permute(0, 2, 1).contiguous().view(-1, C, win_size, win_size) # B' ,C ,Wh ,Ww windows = windows.permute(0, 2, 3, 1).contiguous() # B' ,Wh ,Ww ,C @@ -830,7 +872,12 @@ def window_reverse(windows: torch.Tensor, win_size: int, H: int, W: int, dilatio if dilation_rate != 1: x = windows.permute(0, 5, 3, 4, 1, 2).contiguous() # B, C*Wh*Ww, H/Wh*W/Ww x = F.fold( - x, (H, W), kernel_size=win_size, dilation=dilation_rate, padding=4 * (dilation_rate - 1), stride=win_size + x, + (H, W), + kernel_size=win_size, + dilation=dilation_rate, + padding=4 * (dilation_rate - 1), + stride=win_size, ) else: x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, H, W, -1) @@ -983,7 +1030,13 @@ def __init__( super().__init__() kernel_size_int = kernel_size if isinstance(kernel_size, int) else kernel_size[0] self.proj = nn.Sequential( - nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=stride, padding=kernel_size_int // 2), + nn.Conv2d( + in_channels, + out_channels, + kernel_size=3, + stride=stride, + padding=kernel_size_int // 2, + ), act_layer(inplace=True), ) if norm_layer is not None: @@ -1058,7 +1111,13 @@ def __init__( super().__init__() kernel_size_int = kernel_size if isinstance(kernel_size, int) else kernel_size[0] self.proj = nn.Sequential( - nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=stride, padding=kernel_size_int // 2), + nn.Conv2d( + in_channels, + out_channels, + kernel_size=3, + stride=stride, + padding=kernel_size_int // 2, + ), ) if act_layer is not None: self.proj.add_module("activation", act_layer(inplace=True)) @@ -1117,7 +1176,7 @@ class LeWinTransformerBlock(nn.Module): Whether to use bias in the query, key, and value projections of the attention mechanism. Default: True. qk_scale : float, optional Scale factor for the query and key projection vectors. - If set to None, will use the default value of :math`1 / \sqrt(dim)`. Default: None. + If set to None, will use the default value of :math:`1 / \\sqrt(dim)`. Default: None. drop : float Dropout rate for the token-level dropout layer. Default: 0.0. attn_drop : float @@ -1180,7 +1239,7 @@ def __init__( Whether to use bias in the query, key, and value projections of the attention mechanism. Default: True. qk_scale : float, optional Scale factor for the query and key projection vectors. - If set to None, will use the default value of :math`1 / \sqrt(dim)`. Default: None. + If set to None, will use the default value of :math:`1 / \\sqrt(dim)`. Default: None. drop : float Dropout rate for the token-level dropout layer. Default: 0.0. attn_drop : float @@ -1251,7 +1310,12 @@ def __init__( self.norm2 = norm_layer(dim) mlp_hidden_dim = int(dim * mlp_ratio) if token_mlp == LeWinTransformerMLPTokenType.MLP: - self.mlp = MLP(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop) + self.mlp = MLP( + in_features=dim, + hidden_features=mlp_hidden_dim, + act_layer=act_layer, + drop=drop, + ) else: self.mlp = LeFF(dim, mlp_hidden_dim, act_layer=act_layer) @@ -1396,7 +1460,7 @@ class BasicUFormerLayer(nn.Module): Whether to use bias in the query, key, and value projections of the attention mechanism. Default: True. qk_scale : float, optional Scale factor for the query and key projection vectors. - If set to None, will use the default value of :math`1 / \sqrt(dim)`. Default: None. + If set to None, will use the default value of :math:`1 / \\sqrt(dim)`. Default: None. drop : float Dropout rate for the token-level dropout layer. Default: 0.0. attn_drop : float @@ -1457,7 +1521,7 @@ def __init__( Whether to use bias in the query, key, and value projections of the attention mechanism. Default: True. qk_scale : float, optional Scale factor for the query and key projection vectors. - If set to None, will use the default value of :math`1 / \sqrt(dim)`. Default: None. + If set to None, will use the default value of :math:`1 / \\sqrt(dim)`. Default: None. drop : float Dropout rate for the token-level dropout layer. Default: 0.0. attn_drop : float @@ -1493,13 +1557,13 @@ def __init__( input_resolution=input_resolution, num_heads=num_heads, win_size=win_size, - shift_size=(0 if (i % 2 == 0) else win_size // 2) if shift_flag else 0, + shift_size=((0 if (i % 2 == 0) else win_size // 2) if shift_flag else 0), mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, qk_scale=qk_scale, drop=drop, attn_drop=attn_drop, - drop_path=drop_path[i] if isinstance(drop_path, list) else drop_path, + drop_path=(drop_path[i] if isinstance(drop_path, list) else drop_path), norm_layer=norm_layer, token_projection=token_projection, token_mlp=token_mlp, @@ -1682,7 +1746,12 @@ def __init__( self.num_enc_layers = len(encoder_num_heads) self.num_dec_layers = len(encoder_num_heads) depths = (*encoder_depths, bottleneck_depth, *encoder_depths[::-1]) - num_heads = (*encoder_num_heads, bottleneck_num_heads, bottleneck_num_heads, *encoder_num_heads[::-1][:-1]) + num_heads = ( + *encoder_num_heads, + bottleneck_num_heads, + bottleneck_num_heads, + *encoder_num_heads[::-1][:-1], + ) self.embedding_dim = embedding_dim self.patch_norm = patch_norm self.mlp_ratio = mlp_ratio @@ -1701,15 +1770,27 @@ def __init__( # Input self.input_proj = InputProjection( - in_channels=in_channels, out_channels=embedding_dim, kernel_size=3, stride=1, act_layer=nn.LeakyReLU + in_channels=in_channels, + out_channels=embedding_dim, + kernel_size=3, + stride=1, + act_layer=nn.LeakyReLU, ) out_channels = out_channels if out_channels else in_channels # Output self.output_proj = OutputProjection( - in_channels=2 * embedding_dim, out_channels=out_channels, kernel_size=3, stride=1 + in_channels=2 * embedding_dim, + out_channels=out_channels, + kernel_size=3, + stride=1, ) if in_channels != out_channels: - self.conv_out = nn.Conv2d(in_channels=in_channels, out_channels=out_channels, kernel_size=1, padding=0) + self.conv_out = nn.Conv2d( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=1, + padding=0, + ) self.in_channels = in_channels self.out_channels = out_channels @@ -1747,7 +1828,10 @@ def __init__( # Bottleneck self.bottleneck = BasicUFormerLayer( dim=embedding_dim * (2**self.num_enc_layers), - input_resolution=(patch_size // (2**self.num_enc_layers), patch_size // (2**self.num_enc_layers)), + input_resolution=( + patch_size // (2**self.num_enc_layers), + patch_size // (2**self.num_enc_layers), + ), depth=depths[self.num_enc_layers], num_heads=num_heads[self.num_enc_layers], win_size=win_size, @@ -1776,7 +1860,10 @@ def __init__( self.upsamples.add_module(upsample_layer_name, upsample_layer) layer_name = f"decoderlayer_{self.num_dec_layers - i}" - layer_input_resolution = (patch_size // (2 ** (i - 1)), patch_size // (2 ** (i - 1))) + layer_input_resolution = ( + patch_size // (2 ** (i - 1)), + patch_size // (2 ** (i - 1)), + ) layer_dim = embedding_dim * (2**i) layer_num = self.num_enc_layers + self.num_dec_layers - i + 1 layer_depth = depths[layer_num] diff --git a/direct/nn/transformers/utils.py b/direct/nn/transformers/utils.py index 173302ce..227c0b5e 100644 --- a/direct/nn/transformers/utils.py +++ b/direct/nn/transformers/utils.py @@ -23,7 +23,15 @@ from torch import nn from torch.nn import init -__all__ = ["init_weights", "norm", "pad_to_divisible", "pad_to_square", "unnorm", "unpad_to_original", "DropoutPath"] +__all__ = [ + "init_weights", + "norm", + "pad_to_divisible", + "pad_to_square", + "unnorm", + "unpad_to_original", + "DropoutPath", +] def pad_to_divisible(x: torch.Tensor, pad_size: tuple[int, ...]) -> tuple[torch.Tensor, tuple[tuple[int, int], ...]]: @@ -85,7 +93,7 @@ def pad_to_square( Parameters ---------- inp : torch.Tensor - The input tensor to pad to square shape. Expected shape is (\*, height, width). + The input tensor to pad to square shape. Expected shape is (\\*, height, width). factor : float The factor to which the input tensor will be padded. diff --git a/direct/nn/unet/config.py b/direct/nn/unet/config.py index d9bc8f23..59f98552 100644 --- a/direct/nn/unet/config.py +++ b/direct/nn/unet/config.py @@ -12,8 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. from dataclasses import dataclass +from typing import Optional from direct.config.defaults import ModelConfig +from direct.nn.conv.modulated import ModConvActivation, ModConvType from direct.nn.types import InitType @@ -24,6 +26,12 @@ class UnetModel2dConfig(ModelConfig): num_filters: int = 16 num_pool_layers: int = 4 dropout_probability: float = 0.0 + modulation: ModConvType = ModConvType.NONE + aux_in_features: Optional[int] = None + fc_hidden_features: Optional[int] = None + fc_groups: int = 1 + fc_activation: ModConvActivation = ModConvActivation.SIGMOID + num_weights: Optional[int] = None class NormUnetModel2dConfig(ModelConfig): @@ -33,6 +41,12 @@ class NormUnetModel2dConfig(ModelConfig): num_pool_layers: int = 4 dropout_probability: float = 0.0 norm_groups: int = 2 + modulation: ModConvType = ModConvType.NONE + aux_in_features: Optional[int] = None + fc_hidden_features: Optional[int] = None + fc_groups: int = 1 + fc_activation: ModConvActivation = ModConvActivation.SIGMOID + num_weights: Optional[int] = None @dataclass @@ -43,6 +57,14 @@ class Unet2dConfig(ModelConfig): skip_connection: bool = False normalized: bool = False image_initialization: InitType = InitType.ZERO_FILLED + conv_modulation: ModConvType = ModConvType.NONE + aux_in_features: Optional[int] = None + auxiliary_features: Optional[tuple[str, ...]] = None + log_aux: bool = False + fc_hidden_features: Optional[tuple[int, ...]] = None + fc_groups: int = 1 + fc_activation: ModConvActivation = ModConvActivation.SIGMOID + num_weights: Optional[int] = None @dataclass @@ -52,3 +74,12 @@ class UnetModel3dConfig(ModelConfig): num_filters: int = 16 num_pool_layers: int = 4 dropout_probability: float = 0.0 + + +class NormUnetModel3dConfig(ModelConfig): + in_channels: int = 2 + out_channels: int = 2 + num_filters: int = 16 + num_pool_layers: int = 4 + dropout_probability: float = 0.0 + norm_groups: int = 2 diff --git a/direct/nn/unet/unet_2d.py b/direct/nn/unet/unet_2d.py index b83acda4..162b487c 100644 --- a/direct/nn/unet/unet_2d.py +++ b/direct/nn/unet/unet_2d.py @@ -2,6 +2,9 @@ # Copyright (c) DIRECT Contributors # Code borrowed / edited from: https://github.com/facebookresearch/fastMRI/blob/ + +from __future__ import annotations + import math from typing import List, Optional, Tuple @@ -10,18 +13,116 @@ from torch.nn import functional as F from direct.data import transforms as T +from direct.nn.adain.adain import AdaIN2d, NormType +from direct.nn.conv.modulated import ( + ModConv2dBias, + ModConvActivation, + ModConvType, + ModulationParams, + mod_conv2d, + mod_conv_transpose2d, +) from direct.nn.types import InitType from direct.types import FFTOperator +class ConvModule(nn.Module): + """Single convolution + norm + activation + dropout module supporting modulated convolutions.""" + + def __init__( + self, + in_channels: int, + out_channels: int, + kernel_size: int, + padding: int, + dropout_probability: float, + modulation: ModConvType = ModConvType.NONE, + modulation_params: Optional[ModulationParams] = None, + bias: ModConv2dBias = ModConv2dBias.PARAM, + aux_in_features: Optional[int] = None, + fc_hidden_features: Optional[tuple[int] | int] = None, + fc_groups: int = 1, + fc_activation: ModConvActivation = ModConvActivation.SIGMOID, + num_weights: Optional[int] = None, + norm_type: NormType = NormType.INSTANCE, + adain_hidden_features: Optional[tuple[int] | int] = None, + ): + super().__init__() + + if modulation_params is None: + modulation_params = ModulationParams( + modulation=modulation, + aux_in_features=aux_in_features, + fc_hidden_features=fc_hidden_features, + fc_groups=fc_groups, + fc_activation=fc_activation, + num_weights=num_weights, + ) + self.modulation = modulation_params.modulation + + self.conv = mod_conv2d( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=kernel_size, + padding=padding, + bias=bias, + modulation_params=modulation_params, + ) + self.norm_type = norm_type + if norm_type == NormType.ADAIN: + if adain_hidden_features is None: + raise ValueError("AdaIN hidden features must be provided if norm_type is NormType.ADAIN.") + if modulation_params.aux_in_features is None: + raise ValueError("aux_in_features must be provided if norm_type is NormType.ADAIN.") + self.instance_norm = AdaIN2d( + num_channels=out_channels, + aux_in_features=modulation_params.aux_in_features, + hidden_features=adain_hidden_features, + ) + else: + self.instance_norm = nn.InstanceNorm2d(out_channels) + self.leaky_relu = nn.LeakyReLU(negative_slope=0.2, inplace=True) + self.dropout = nn.Dropout2d(dropout_probability) + + def forward(self, x: torch.Tensor, y: Optional[torch.Tensor] = None) -> torch.Tensor: + if self.modulation != ModConvType.NONE: + x = self.conv(x, y) + else: + x = self.conv(x) + if self.norm_type == NormType.ADAIN: + if y is None: + raise ValueError("AdaIN requires aux vector y, but got None.") + x = self.instance_norm(x, y) + else: + x = self.instance_norm(x) + x = self.leaky_relu(x) + x = self.dropout(x) + return x + + class ConvBlock(nn.Module): """U-Net convolutional block. It consists of two convolution layers each followed by instance normalization, LeakyReLU activation and dropout. + Supports modulated convolutions and AdaIN normalization. """ - def __init__(self, in_channels: int, out_channels: int, dropout_probability: float): - """Inits ConvBlock. + def __init__( + self, + in_channels: int, + out_channels: int, + dropout_probability: float, + modulation: ModConvType = ModConvType.NONE, + modulation_params: Optional[ModulationParams] = None, + aux_in_features: Optional[int] = None, + fc_hidden_features: Optional[tuple[int] | int] = None, + fc_groups: int = 1, + fc_activation: ModConvActivation = ModConvActivation.SIGMOID, + num_weights: Optional[int] = None, + norm_type: NormType = NormType.INSTANCE, + adain_hidden_features: Optional[tuple[int] | int] = None, + ): + """Inits :class:`ConvBlock`. Parameters ---------- @@ -31,6 +132,22 @@ def __init__(self, in_channels: int, out_channels: int, dropout_probability: flo Number of output channels. dropout_probability: float Dropout probability. + modulation : ModConvType + Modulation type. Default: ModConvType.NONE. + aux_in_features : int, optional + Number of auxiliary input features for modulation/AdaIN. + fc_hidden_features : int or tuple of int, optional + Hidden features for the modulation MLP. + fc_groups : int + Groups for the modulation MLP. Default: 1. + fc_activation : ModConvActivation + Activation for the modulation MLP. Default: ModConvActivation.SIGMOID. + num_weights : int, optional + Number of weight bases for ModConvType.SUM. + norm_type : NormType + Normalization type. Default: NormType.INSTANCE. + adain_hidden_features : int or tuple of int, optional + Hidden features for AdaIN. Required if norm_type is NormType.ADAIN. """ super().__init__() @@ -38,45 +155,84 @@ def __init__(self, in_channels: int, out_channels: int, dropout_probability: flo self.out_channels = out_channels self.dropout_probability = dropout_probability - self.layers = nn.Sequential( - nn.Conv2d(in_channels, out_channels, kernel_size=3, padding=1, bias=False), - nn.InstanceNorm2d(out_channels), - nn.LeakyReLU(negative_slope=0.2, inplace=True), - nn.Dropout2d(dropout_probability), - nn.Conv2d(out_channels, out_channels, kernel_size=3, padding=1, bias=False), - nn.InstanceNorm2d(out_channels), - nn.LeakyReLU(negative_slope=0.2, inplace=True), - nn.Dropout2d(dropout_probability), + if modulation_params is None: + modulation_params = ModulationParams( + modulation=modulation, + aux_in_features=aux_in_features, + fc_hidden_features=fc_hidden_features, + fc_groups=fc_groups, + fc_activation=fc_activation, + num_weights=num_weights, + ) + self.modulation = modulation_params.modulation + self.norm_type = norm_type + + self.layer_1 = ConvModule( + in_channels, + out_channels, + kernel_size=3, + padding=1, + bias=(ModConv2dBias.NONE if modulation_params.modulation == ModConvType.NONE else ModConv2dBias.LEARNED), + dropout_probability=dropout_probability, + modulation_params=modulation_params, + norm_type=norm_type, + adain_hidden_features=adain_hidden_features, + ) + self.layer_2 = ConvModule( + out_channels, + out_channels, + kernel_size=3, + padding=1, + bias=(ModConv2dBias.NONE if modulation_params.modulation == ModConvType.NONE else ModConv2dBias.LEARNED), + dropout_probability=dropout_probability, + modulation_params=modulation_params, + norm_type=norm_type, + adain_hidden_features=adain_hidden_features, ) - def forward(self, input_data: torch.Tensor) -> torch.Tensor: + def forward(self, input_data: torch.Tensor, aux_data: Optional[torch.Tensor] = None) -> torch.Tensor: """Performs the forward pass of :class:`ConvBlock`. Parameters ---------- - input_data: torch.Tensor + input_data : torch.Tensor + aux_data : torch.Tensor, optional Returns ------- torch.Tensor """ - return self.layers(input_data) + if self.modulation != ModConvType.NONE or self.norm_type == NormType.ADAIN: + return self.layer_2(self.layer_1(input_data, aux_data), aux_data) + return self.layer_2(self.layer_1(input_data)) def __repr__(self): - """Representation of :class:`ConvBlock`.""" return ( f"ConvBlock(in_channels={self.in_channels}, out_channels={self.out_channels}, " - f"dropout_probability={self.dropout_probability})" + f"dropout_probability={self.dropout_probability}, modulation={self.modulation})" ) class TransposeConvBlock(nn.Module): - """U-Net Transpose Convolutional Block. + """U-Net Transpose Convolutional Block with optional modulation. - It consists of one convolution transpose layers followed by instance normalization and LeakyReLU activation. + It consists of one convolution transpose layer followed by instance normalization and LeakyReLU activation. """ - def __init__(self, in_channels: int, out_channels: int): + def __init__( + self, + in_channels: int, + out_channels: int, + modulation: ModConvType = ModConvType.NONE, + modulation_params: Optional[ModulationParams] = None, + aux_in_features: Optional[int] = None, + fc_hidden_features: Optional[tuple[int] | int] = None, + fc_groups: int = 1, + fc_activation: ModConvActivation = ModConvActivation.SIGMOID, + num_weights: Optional[int] = None, + norm_type: NormType = NormType.INSTANCE, + adain_hidden_features: Optional[tuple[int] | int] = None, + ): """Inits :class:`TransposeConvBlock`. Parameters @@ -85,43 +241,100 @@ def __init__(self, in_channels: int, out_channels: int): Number of input channels. out_channels: int Number of output channels. + modulation : ModConvType + Modulation type. Default: ModConvType.NONE. + aux_in_features : int, optional + Number of auxiliary input features. + fc_hidden_features : int or tuple of int, optional + Hidden features for the modulation MLP. + fc_groups : int + Groups for the modulation MLP. Default: 1. + fc_activation : ModConvActivation + Activation for the modulation MLP. Default: ModConvActivation.SIGMOID. + num_weights : int, optional + Number of weight bases for ModConvType.SUM. + norm_type : NormType + Normalization type. Default: NormType.INSTANCE. + adain_hidden_features : int or tuple of int, optional + Hidden features for AdaIN. """ super().__init__() self.in_channels = in_channels self.out_channels = out_channels + if modulation_params is None: + modulation_params = ModulationParams( + modulation=modulation, + aux_in_features=aux_in_features, + fc_hidden_features=fc_hidden_features, + fc_groups=fc_groups, + fc_activation=fc_activation, + num_weights=num_weights, + ) + self.modulation = modulation_params.modulation + self.norm_type = norm_type - self.layers = nn.Sequential( - nn.ConvTranspose2d(in_channels, out_channels, kernel_size=2, stride=2, bias=False), - nn.InstanceNorm2d(out_channels), - nn.LeakyReLU(negative_slope=0.2, inplace=True), + self.conv = mod_conv_transpose2d( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=2, + stride=2, + bias=(ModConv2dBias.NONE if modulation_params.modulation == ModConvType.NONE else ModConv2dBias.LEARNED), + modulation_params=modulation_params, ) + if norm_type == NormType.ADAIN: + if adain_hidden_features is None: + raise ValueError("AdaIN hidden features must be provided if norm_type is NormType.ADAIN.") + if modulation_params.aux_in_features is None: + raise ValueError("aux_in_features must be provided if norm_type is NormType.ADAIN.") + self.instance_norm = AdaIN2d( + num_channels=out_channels, + aux_in_features=modulation_params.aux_in_features, + hidden_features=adain_hidden_features, + ) + else: + self.instance_norm = nn.InstanceNorm2d(out_channels) + self.leaky_relu = nn.LeakyReLU(negative_slope=0.2, inplace=True) - def forward(self, input_data: torch.Tensor) -> torch.Tensor: + def forward(self, input_data: torch.Tensor, aux_data: Optional[torch.Tensor] = None) -> torch.Tensor: """Performs forward pass of :class:`TransposeConvBlock`. Parameters ---------- input_data: torch.Tensor + aux_data: torch.Tensor, optional Returns ------- torch.Tensor """ - return self.layers(input_data) + if self.modulation != ModConvType.NONE: + x = self.conv(input_data, aux_data) + else: + x = self.conv(input_data) + if self.norm_type == NormType.ADAIN: + x = self.instance_norm(x, aux_data) + else: + x = self.instance_norm(x) + return self.leaky_relu(x) def __repr__(self): - """Representation of "class:`TransposeConvBlock`.""" - return f"ConvBlock(in_channels={self.in_channels}, out_channels={self.out_channels})" + return f"TransposeConvBlock(in_channels={self.in_channels}, out_channels={self.out_channels})" class UnetModel2d(nn.Module): """PyTorch implementation of a U-Net model based on [1]_. + Supports optional modulated convolutions and AdaIN normalization conditioned + on an auxiliary input signal as proposed in [2]_. + References ---------- + .. [1] Ronneberger, Olaf, et al. "U-Net: Convolutional Networks for Biomedical Image Segmentation." MICCAI 2015. - .. [1] Ronneberger, Olaf, et al. “U-Net: Convolutional Networks for Biomedical Image Segmentation.” Medical Image Computing and Computer-Assisted Intervention – MICCAI 2015, edited by Nassir Navab et al., Springer International Publishing, 2015, pp. 234–41. Springer Link, https://doi.org/10.1007/978-3-319-24574-4_28. + .. [2] Moriakov, N., Yiasemis, G., Sonke, J.-J. & Teuwen, J. (2026). Conditional Learned Reconstruction for + Medical Imaging. Proceedings of The 9th International Conference on Medical Imaging with Deep Learning, + PMLR 315:754-780. https://proceedings.mlr.press/v315/moriakov26a.html """ def __init__( @@ -131,6 +344,16 @@ def __init__( num_filters: int, num_pool_layers: int, dropout_probability: float, + modulation: ModConvType = ModConvType.NONE, + modulation_params: Optional[ModulationParams] = None, + aux_in_features: Optional[int] = None, + fc_hidden_features: Optional[tuple[int] | int] = None, + fc_groups: int = 1, + fc_activation: ModConvActivation = ModConvActivation.SIGMOID, + num_weights: Optional[int] = None, + modulation_at_input: bool = False, + norm_type: NormType = NormType.INSTANCE, + adain_hidden_features: Optional[tuple[int] | int] = None, ): """Inits :class:`UnetModel2d`. @@ -146,6 +369,24 @@ def __init__( Number of down-sampling and up-sampling layers (depth). dropout_probability: float Dropout probability. + modulation : ModConvType + Modulation type. Default: ModConvType.NONE. + aux_in_features : int, optional + Number of auxiliary input features. + fc_hidden_features : int or tuple of int, optional + Hidden features for the modulation MLP. + fc_groups : int + Groups for the modulation MLP. Default: 1. + fc_activation : ModConvActivation + Activation for the modulation MLP. Default: ModConvActivation.SIGMOID. + num_weights : int, optional + Number of weight bases for ModConvType.SUM. + modulation_at_input : bool + If True, only the first conv block uses modulation. Default: False. + norm_type : NormType + Normalization type. Default: NormType.INSTANCE. + adain_hidden_features : int or tuple of int, optional + Hidden features for AdaIN. """ super().__init__() @@ -154,35 +395,127 @@ def __init__( self.num_filters = num_filters self.num_pool_layers = num_pool_layers self.dropout_probability = dropout_probability - - self.down_sample_layers = nn.ModuleList([ConvBlock(in_channels, num_filters, dropout_probability)]) + if modulation_params is None: + modulation_params = ModulationParams( + modulation=modulation, + aux_in_features=aux_in_features, + fc_hidden_features=fc_hidden_features, + fc_groups=fc_groups, + fc_activation=fc_activation, + num_weights=num_weights, + ) + self.modulation = modulation_params.modulation + self.norm_type = norm_type + + self.down_sample_layers = nn.ModuleList( + [ + ConvBlock( + in_channels, + num_filters, + dropout_probability, + modulation_params=modulation_params, + norm_type=norm_type, + adain_hidden_features=adain_hidden_features, + ) + ] + ) ch = num_filters + + block_modulation_params = modulation_params + if modulation_params.modulation != ModConvType.NONE and modulation_at_input: + block_modulation_params = ModulationParams( + modulation=ModConvType.NONE, + aux_in_features=modulation_params.aux_in_features, + fc_hidden_features=modulation_params.fc_hidden_features, + fc_groups=modulation_params.fc_groups, + fc_activation=modulation_params.fc_activation, + num_weights=modulation_params.num_weights, + fc_bias=modulation_params.fc_bias, + ) + for _ in range(num_pool_layers - 1): - self.down_sample_layers += [ConvBlock(ch, ch * 2, dropout_probability)] + self.down_sample_layers += [ + ConvBlock( + ch, + ch * 2, + dropout_probability, + modulation_params=block_modulation_params, + norm_type=norm_type, + adain_hidden_features=adain_hidden_features, + ) + ] ch *= 2 - self.conv = ConvBlock(ch, ch * 2, dropout_probability) + self.conv = ConvBlock( + ch, + ch * 2, + dropout_probability, + modulation_params=block_modulation_params, + norm_type=norm_type, + adain_hidden_features=adain_hidden_features, + ) self.up_conv = nn.ModuleList() self.up_transpose_conv = nn.ModuleList() for _ in range(num_pool_layers - 1): - self.up_transpose_conv += [TransposeConvBlock(ch * 2, ch)] - self.up_conv += [ConvBlock(ch * 2, ch, dropout_probability)] + self.up_transpose_conv += [ + TransposeConvBlock( + ch * 2, + ch, + modulation_params=block_modulation_params, + norm_type=norm_type, + adain_hidden_features=adain_hidden_features, + ) + ] + self.up_conv += [ + ConvBlock( + ch * 2, + ch, + dropout_probability, + modulation_params=block_modulation_params, + norm_type=norm_type, + adain_hidden_features=adain_hidden_features, + ) + ] ch //= 2 - self.up_transpose_conv += [TransposeConvBlock(ch * 2, ch)] + self.up_transpose_conv += [ + TransposeConvBlock( + ch * 2, + ch, + modulation_params=block_modulation_params, + norm_type=norm_type, + adain_hidden_features=adain_hidden_features, + ) + ] self.up_conv += [ - nn.Sequential( - ConvBlock(ch * 2, ch, dropout_probability), - nn.Conv2d(ch, self.out_channels, kernel_size=1, stride=1), + ConvBlock( + ch * 2, + ch, + dropout_probability, + modulation_params=block_modulation_params, + norm_type=norm_type, + adain_hidden_features=adain_hidden_features, ) ] + self.conv_out = mod_conv2d( + ch, + self.out_channels, + kernel_size=1, + stride=1, + bias=( + ModConv2dBias.NONE if block_modulation_params.modulation == ModConvType.NONE else ModConv2dBias.LEARNED + ), + modulation_params=block_modulation_params, + ) - def forward(self, input_data: torch.Tensor) -> torch.Tensor: + def forward(self, input_data: torch.Tensor, aux_data: Optional[torch.Tensor] = None) -> torch.Tensor: """Performs forward pass of :class:`UnetModel2d`. Parameters ---------- input_data: torch.Tensor + aux_data: torch.Tensor, optional + Auxiliary data for modulation/AdaIN. Returns ------- @@ -191,36 +524,50 @@ def forward(self, input_data: torch.Tensor) -> torch.Tensor: stack = [] output = input_data - # Apply down-sampling layers - for _, layer in enumerate(self.down_sample_layers): - output = layer(output) + for layer in self.down_sample_layers: + if self.modulation != ModConvType.NONE or self.norm_type == NormType.ADAIN: + output = layer(output, aux_data) + else: + output = layer(output) stack.append(output) output = F.avg_pool2d(output, kernel_size=2, stride=2, padding=0) - output = self.conv(output) + if self.modulation != ModConvType.NONE or self.norm_type == NormType.ADAIN: + output = self.conv(output, aux_data) + else: + output = self.conv(output) - # Apply up-sampling layers for transpose_conv, conv in zip(self.up_transpose_conv, self.up_conv): downsample_layer = stack.pop() - output = transpose_conv(output) + if self.modulation != ModConvType.NONE or self.norm_type == NormType.ADAIN: + output = transpose_conv(output, aux_data) + else: + output = transpose_conv(output) - # Reflect pad on the right/bottom if needed to handle odd input dimensions. padding = [0, 0, 0, 0] if output.shape[-1] != downsample_layer.shape[-1]: - padding[1] = 1 # Padding right + padding[1] = 1 if output.shape[-2] != downsample_layer.shape[-2]: - padding[3] = 1 # Padding bottom + padding[3] = 1 if sum(padding) != 0: output = F.pad(output, padding, "reflect") output = torch.cat([output, downsample_layer], dim=1) - output = conv(output) + if self.modulation != ModConvType.NONE or self.norm_type == NormType.ADAIN: + output = conv(output, aux_data) + else: + output = conv(output) + + if self.modulation != ModConvType.NONE or self.norm_type == NormType.ADAIN: + output = self.conv_out(output, aux_data) + else: + output = self.conv_out(output) return output class NormUnetModel2d(nn.Module): - """Implementation of a Normalized U-Net model.""" + """Implementation of a Normalized U-Net model with optional modulation support.""" def __init__( self, @@ -230,6 +577,16 @@ def __init__( num_pool_layers: int, dropout_probability: float, norm_groups: int = 2, + modulation: ModConvType = ModConvType.NONE, + modulation_params: Optional[ModulationParams] = None, + aux_in_features: Optional[int] = None, + fc_hidden_features: Optional[tuple[int] | int] = None, + fc_groups: int = 1, + fc_activation: ModConvActivation = ModConvActivation.SIGMOID, + num_weights: Optional[int] = None, + modulation_at_input: bool = False, + norm_type: NormType = NormType.INSTANCE, + adain_hidden_features: Optional[tuple[int] | int] = None, ): """Inits :class:`NormUnetModel2d`. @@ -245,8 +602,26 @@ def __init__( Number of down-sampling and up-sampling layers (depth). dropout_probability: float Dropout probability. - norm_groups: int, + norm_groups: int Number of normalization groups. + modulation : ModConvType + Modulation type. Default: ModConvType.NONE. + aux_in_features : int, optional + Number of auxiliary input features. + fc_hidden_features : int or tuple of int, optional + Hidden features for the modulation MLP. + fc_groups : int + Groups for the modulation MLP. Default: 1. + fc_activation : ModConvActivation + Activation for the modulation MLP. Default: ModConvActivation.SIGMOID. + num_weights : int, optional + Number of weight bases for ModConvType.SUM. + modulation_at_input : bool + If True, only the first conv block uses modulation. Default: False. + norm_type : NormType + Normalization type. Default: NormType.INSTANCE. + adain_hidden_features : int or tuple of int, optional + Hidden features for AdaIN. """ super().__init__() @@ -256,14 +631,23 @@ def __init__( num_filters=num_filters, num_pool_layers=num_pool_layers, dropout_probability=dropout_probability, + modulation_params=modulation_params, + modulation=modulation, + aux_in_features=aux_in_features, + fc_hidden_features=fc_hidden_features, + fc_groups=fc_groups, + fc_activation=fc_activation, + modulation_at_input=modulation_at_input, + norm_type=norm_type, + adain_hidden_features=adain_hidden_features, + num_weights=num_weights, ) - + self.modulation = self.unet2d.modulation self.norm_groups = norm_groups @staticmethod def norm(input_data: torch.Tensor, groups: int) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: """Performs group normalization.""" - # group norm b, c, h, w = input_data.shape input_data = input_data.reshape(b, groups, -1) @@ -282,7 +666,9 @@ def unnorm(input_data: torch.Tensor, mean: torch.Tensor, std: torch.Tensor, grou return (input_data * std + mean).reshape(b, c, h, w) @staticmethod - def pad(input_data: torch.Tensor) -> Tuple[torch.Tensor, Tuple[List[int], List[int], int, int]]: + def pad( + input_data: torch.Tensor, + ) -> Tuple[torch.Tensor, Tuple[List[int], List[int], int, int]]: _, _, h, w = input_data.shape w_mult = ((w - 1) | 15) + 1 h_mult = ((h - 1) | 15) + 1 @@ -302,21 +688,22 @@ def unpad( ) -> torch.Tensor: return input_data[..., h_pad[0] : h_mult - h_pad[1], w_pad[0] : w_mult - w_pad[1]] - def forward(self, input_data: torch.Tensor) -> torch.Tensor: + def forward(self, input_data: torch.Tensor, aux_data: Optional[torch.Tensor] = None) -> torch.Tensor: """Performs forward pass of :class:`NormUnetModel2d`. Parameters ---------- - input_data: torch.Tensor + input_data : torch.Tensor + aux_data : torch.Tensor, optional Returns ------- torch.Tensor """ - output, mean, std = self.norm(input_data, self.norm_groups) output, pad_sizes = self.pad(output) - output = self.unet2d(output) + + output = self.unet2d(output, aux_data) output = self.unpad(output, *pad_sizes) output = self.unnorm(output, mean, std, self.norm_groups) @@ -337,15 +724,22 @@ def __init__( skip_connection: bool = False, normalized: bool = False, image_initialization: InitType = InitType.ZERO_FILLED, + conv_modulation: ModConvType = ModConvType.NONE, + modulation_params: Optional[ModulationParams] = None, + aux_in_features: Optional[int] = None, + fc_hidden_features: Optional[tuple[int] | int] = None, + fc_groups: int = 1, + fc_activation: ModConvActivation = ModConvActivation.SIGMOID, + num_weights: Optional[int] = None, **kwargs, ): """Inits :class:`Unet2d`. Parameters ---------- - forward_operator: Callable + forward_operator: FFTOperator Forward Operator. - backward_operator: Callable + backward_operator: FFTOperator Backward Operator. num_filters: int Number of first layer filters. @@ -362,11 +756,22 @@ def __init__( kwargs: dict """ super().__init__() + modulation_params = ModulationParams( + modulation=conv_modulation, + aux_in_features=aux_in_features, + fc_hidden_features=fc_hidden_features, + fc_groups=fc_groups, + fc_activation=fc_activation, + num_weights=num_weights, + ) extra_keys = kwargs.keys() for extra_key in extra_keys: if extra_key not in [ "sensitivity_map_model", "model_name", + "auxiliary_features", + "log_aux", + "conv_modulation", ]: raise ValueError(f"{type(self).__name__} got key `{extra_key}` which is not supported.") self.unet: nn.Module @@ -377,6 +782,7 @@ def __init__( num_filters=num_filters, num_pool_layers=num_pool_layers, dropout_probability=dropout_probability, + modulation_params=modulation_params, ) else: self.unet = UnetModel2d( @@ -385,7 +791,10 @@ def __init__( num_filters=num_filters, num_pool_layers=num_pool_layers, dropout_probability=dropout_probability, + modulation_params=modulation_params, ) + self.conv_modulation = conv_modulation + self.modulation = conv_modulation self.forward_operator = forward_operator self.backward_operator = backward_operator self.skip_connection = skip_connection @@ -424,6 +833,7 @@ def forward( self, masked_kspace: torch.Tensor, sensitivity_map: Optional[torch.Tensor] = None, + auxiliary_data: Optional[torch.Tensor] = None, ) -> torch.Tensor: """Computes forward pass of Unet2d. @@ -454,7 +864,10 @@ def forward( f"Got {self.image_initialization}." ) - output = self.unet(input_image.permute(0, 3, 1, 2)).permute(0, 2, 3, 1) + if self.modulation != ModConvType.NONE: + output = self.unet(input_image.permute(0, 3, 1, 2), auxiliary_data).permute(0, 2, 3, 1) + else: + output = self.unet(input_image.permute(0, 3, 1, 2)).permute(0, 2, 3, 1) if self.skip_connection: output += input_image return output diff --git a/direct/nn/unet/unet_3d.py b/direct/nn/unet/unet_3d.py index 019161ba..2c4d497b 100644 --- a/direct/nn/unet/unet_3d.py +++ b/direct/nn/unet/unet_3d.py @@ -17,16 +17,103 @@ from __future__ import annotations import math +from typing import Optional import torch from torch import nn from torch.nn import functional as F +from direct.nn.adain.adain import AdaIN3d, NormType +from direct.nn.conv.modulated import ModConv2dBias, ModConv3d, ModConvActivation, ModConvTranspose3d, ModConvType + + +class ConvModule3D(nn.Module): + """Single 3D convolution + norm + activation + dropout module supporting modulated convolutions.""" + + def __init__( + self, + in_channels: int, + out_channels: int, + kernel_size: int, + padding: int, + dropout_probability: float, + modulation: ModConvType = ModConvType.NONE, + bias: ModConv2dBias = ModConv2dBias.PARAM, + aux_in_features: Optional[int] = None, + fc_hidden_features: Optional[tuple[int] | int] = None, + fc_groups: int = 1, + fc_activation: ModConvActivation = ModConvActivation.SIGMOID, + num_weights: Optional[int] = None, + norm_type: NormType = NormType.INSTANCE, + adain_hidden_features: Optional[tuple[int] | int] = None, + ): + super().__init__() + + self.modulation = modulation + self.norm_type = norm_type + + self.conv = ModConv3d( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=kernel_size, + padding=padding, + bias=bias, + modulation=modulation, + aux_in_features=aux_in_features, + fc_hidden_features=fc_hidden_features, + fc_groups=fc_groups, + fc_activation=fc_activation, + num_weights=num_weights, + ) + if norm_type == NormType.ADAIN: + if adain_hidden_features is None: + raise ValueError("AdaIN hidden features must be provided if norm_type is NormType.ADAIN.") + if aux_in_features is None: + raise ValueError("aux_in_features must be provided if norm_type is NormType.ADAIN.") + self.instance_norm = AdaIN3d( + num_channels=out_channels, + aux_in_features=aux_in_features, + hidden_features=adain_hidden_features, + ) + else: + self.instance_norm = nn.InstanceNorm3d(out_channels) + self.leaky_relu = nn.LeakyReLU(negative_slope=0.2, inplace=True) + self.dropout = nn.Dropout3d(dropout_probability) + + def forward(self, x: torch.Tensor, y: Optional[torch.Tensor] = None) -> torch.Tensor: + if self.modulation != ModConvType.NONE: + x = self.conv(x, y) + else: + x = self.conv(x) + + if self.norm_type == NormType.ADAIN: + if y is None: + raise ValueError("AdaIN requires aux vector y, but got None.") + x = self.instance_norm(x, y) + else: + x = self.instance_norm(x) + x = self.leaky_relu(x) + x = self.dropout(x) + return x + class ConvBlock3D(nn.Module): - """3D U-Net convolutional block.""" + """3D U-Net convolutional block with optional modulation and AdaIN support.""" - def __init__(self, in_channels: int, out_channels: int, dropout_probability: float) -> None: + def __init__( + self, + in_channels: int, + out_channels: int, + dropout_probability: float, + modulation: ModConvType = ModConvType.NONE, + aux_in_features: Optional[int] = None, + fc_hidden_features: Optional[tuple[int] | int] = None, + fc_groups: int = 1, + fc_activation: ModConvActivation = ModConvActivation.SIGMOID, + num_weights: Optional[int] = None, + norm_type: NormType = NormType.INSTANCE, + adain_hidden_features: Optional[tuple[int] | int] = None, + ) -> None: """Inits :class:`ConvBlock3D`. Parameters @@ -37,43 +124,97 @@ def __init__(self, in_channels: int, out_channels: int, dropout_probability: flo Number of channels produced by the convolutional layers. dropout_probability : float Dropout probability applied after convolutional layers. + modulation : ModConvType + Modulation type. Default: ModConvType.NONE. + aux_in_features : int, optional + Number of auxiliary input features. + fc_hidden_features : int or tuple of int, optional + Hidden features for the modulation MLP. + fc_groups : int + Groups for the modulation MLP. Default: 1. + fc_activation : ModConvActivation + Activation for the modulation MLP. Default: ModConvActivation.SIGMOID. + num_weights : int, optional + Number of weight bases for ModConvType.SUM. + norm_type : NormType + Normalization type. Default: NormType.INSTANCE. + adain_hidden_features : int or tuple of int, optional + Hidden features for AdaIN. """ super().__init__() self.in_channels = in_channels self.out_channels = out_channels self.dropout_probability = dropout_probability + self.modulation = modulation + self.norm_type = norm_type - self.layers = nn.Sequential( - nn.Conv3d(in_channels, out_channels, kernel_size=3, padding=1, bias=False), - nn.InstanceNorm3d(out_channels), - nn.LeakyReLU(negative_slope=0.2, inplace=True), - nn.Dropout3d(dropout_probability), - nn.Conv3d(out_channels, out_channels, kernel_size=3, padding=1, bias=False), - nn.InstanceNorm3d(out_channels), - nn.LeakyReLU(negative_slope=0.2, inplace=True), - nn.Dropout3d(dropout_probability), + self.layer_1 = ConvModule3D( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=3, + padding=1, + dropout_probability=dropout_probability, + bias=(ModConv2dBias.NONE if modulation == ModConvType.NONE else ModConv2dBias.LEARNED), + modulation=modulation, + aux_in_features=aux_in_features, + fc_hidden_features=fc_hidden_features, + fc_groups=fc_groups, + fc_activation=fc_activation, + num_weights=num_weights, + norm_type=norm_type, + adain_hidden_features=adain_hidden_features, + ) + self.layer_2 = ConvModule3D( + in_channels=out_channels, + out_channels=out_channels, + kernel_size=3, + padding=1, + dropout_probability=dropout_probability, + bias=(ModConv2dBias.NONE if modulation == ModConvType.NONE else ModConv2dBias.LEARNED), + modulation=modulation, + aux_in_features=aux_in_features, + fc_hidden_features=fc_hidden_features, + fc_groups=fc_groups, + fc_activation=fc_activation, + num_weights=num_weights, + norm_type=norm_type, + adain_hidden_features=adain_hidden_features, ) - def forward(self, input_data: torch.Tensor) -> torch.Tensor: - """Performs the forward pass of :class:`ConvBlock3D`.. + def forward(self, input_data: torch.Tensor, aux_data: Optional[torch.Tensor] = None) -> torch.Tensor: + """Performs the forward pass of :class:`ConvBlock3D`. Parameters ---------- input_data : torch.Tensor - Input data. + aux_data : torch.Tensor, optional Returns ------- torch.Tensor """ - return self.layers(input_data) + if self.modulation != ModConvType.NONE or self.norm_type == NormType.ADAIN: + return self.layer_2(self.layer_1(input_data, aux_data), aux_data) + return self.layer_2(self.layer_1(input_data)) class TransposeConvBlock3D(nn.Module): - """3D U-Net Transpose Convolutional Block.""" + """3D U-Net Transpose Convolutional Block with optional modulation.""" - def __init__(self, in_channels: int, out_channels: int) -> None: + def __init__( + self, + in_channels: int, + out_channels: int, + modulation: ModConvType = ModConvType.NONE, + aux_in_features: Optional[int] = None, + fc_hidden_features: Optional[tuple[int] | int] = None, + fc_groups: int = 1, + fc_activation: ModConvActivation = ModConvActivation.SIGMOID, + num_weights: Optional[int] = None, + norm_type: NormType = NormType.INSTANCE, + adain_hidden_features: Optional[tuple[int] | int] = None, + ) -> None: """Inits :class:`TransposeConvBlock3D`. Parameters @@ -82,38 +223,93 @@ def __init__(self, in_channels: int, out_channels: int) -> None: Number of channels in the input tensor. out_channels : int Number of channels produced by the convolutional layers. + modulation : ModConvType + Modulation type. Default: ModConvType.NONE. + aux_in_features : int, optional + Number of auxiliary input features. + fc_hidden_features : int or tuple of int, optional + Hidden features for the modulation MLP. + fc_groups : int + Groups for the modulation MLP. Default: 1. + fc_activation : ModConvActivation + Activation for the modulation MLP. Default: ModConvActivation.SIGMOID. + num_weights : int, optional + Number of weight bases for ModConvType.SUM. + norm_type : NormType + Normalization type. Default: NormType.INSTANCE. + adain_hidden_features : int or tuple of int, optional + Hidden features for AdaIN. """ super().__init__() self.in_channels = in_channels self.out_channels = out_channels + self.modulation = modulation + self.norm_type = norm_type - self.layers = nn.Sequential( - nn.ConvTranspose3d(in_channels, out_channels, kernel_size=2, stride=2, bias=False), - nn.InstanceNorm3d(out_channels), - nn.LeakyReLU(negative_slope=0.2, inplace=True), + self.conv = ModConvTranspose3d( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=2, + stride=2, + modulation=modulation, + bias=(ModConv2dBias.NONE if modulation == ModConvType.NONE else ModConv2dBias.LEARNED), + aux_in_features=aux_in_features, + fc_hidden_features=fc_hidden_features, + fc_groups=fc_groups, + fc_activation=fc_activation, + num_weights=num_weights, ) + if norm_type == NormType.ADAIN: + if adain_hidden_features is None: + raise ValueError("AdaIN hidden features must be provided if norm_type is NormType.ADAIN.") + if aux_in_features is None: + raise ValueError("aux_in_features must be provided if norm_type is NormType.ADAIN.") + self.instance_norm = AdaIN3d( + num_channels=out_channels, + aux_in_features=aux_in_features, + hidden_features=adain_hidden_features, + ) + else: + self.instance_norm = nn.InstanceNorm3d(out_channels) + self.leaky_relu = nn.LeakyReLU(negative_slope=0.2, inplace=True) - def forward(self, input_data: torch.Tensor) -> torch.Tensor: + def forward(self, input_data: torch.Tensor, aux_data: Optional[torch.Tensor] = None) -> torch.Tensor: """Performs the forward pass of :class:`TransposeConvBlock3D`. Parameters ---------- input_data : torch.Tensor - Input data. + aux_data : torch.Tensor, optional Returns ------- torch.Tensor """ - return self.layers(input_data) + if self.modulation != ModConvType.NONE: + x = self.conv(input_data, aux_data) + else: + x = self.conv(input_data) + if self.norm_type == NormType.ADAIN: + x = self.instance_norm(x, aux_data) + else: + x = self.instance_norm(x) + return self.leaky_relu(x) class UnetModel3d(nn.Module): - """PyTorch implementation of a 3D U-Net model. + """PyTorch implementation of a 3D U-Net model with optional modulated convolutions. This class defines a 3D U-Net architecture consisting of down-sampling and up-sampling layers with 3D convolutional blocks. This is an extension to 3D volumes of :class:`direct.nn.unet.unet_2d.UnetModel2d`. + Modulated convolutions are based on [1]_. + + References + ---------- + + .. [1] Moriakov, N., Yiasemis, G., Sonke, J.-J. & Teuwen, J. (2026). Conditional Learned Reconstruction for + Medical Imaging. Proceedings of The 9th International Conference on Medical Imaging with Deep Learning, + PMLR 315:754-780. https://proceedings.mlr.press/v315/moriakov26a.html """ def __init__( @@ -123,6 +319,15 @@ def __init__( num_filters: int, num_pool_layers: int, dropout_probability: float, + modulation: ModConvType = ModConvType.NONE, + aux_in_features: Optional[int] = None, + fc_hidden_features: Optional[tuple[int] | int] = None, + fc_groups: int = 1, + fc_activation: ModConvActivation = ModConvActivation.SIGMOID, + num_weights: Optional[int] = None, + modulation_at_input: bool = False, + norm_type: NormType = NormType.INSTANCE, + adain_hidden_features: Optional[tuple[int] | int] = None, ) -> None: """Inits :class:`UnetModel3d`. @@ -138,6 +343,24 @@ def __init__( Number of down-sampling and up-sampling layers (depth). dropout_probability : float Dropout probability. + modulation : ModConvType + Modulation type. Default: ModConvType.NONE. + aux_in_features : int, optional + Number of auxiliary input features. + fc_hidden_features : int or tuple of int, optional + Hidden features for the modulation MLP. + fc_groups : int + Groups for the modulation MLP. Default: 1. + fc_activation : ModConvActivation + Activation for the modulation MLP. Default: ModConvActivation.SIGMOID. + num_weights : int, optional + Number of weight bases for ModConvType.SUM. + modulation_at_input : bool + If True, only the first conv block uses modulation. Default: False. + norm_type : NormType + Normalization type. Default: NormType.INSTANCE. + adain_hidden_features : int or tuple of int, optional + Hidden features for AdaIN. """ super().__init__() @@ -146,36 +369,140 @@ def __init__( self.num_filters = num_filters self.num_pool_layers = num_pool_layers self.dropout_probability = dropout_probability - - self.down_sample_layers = nn.ModuleList([ConvBlock3D(in_channels, num_filters, dropout_probability)]) + self.modulation = modulation + self.norm_type = norm_type + + self.down_sample_layers = nn.ModuleList( + [ + ConvBlock3D( + in_channels, + num_filters, + dropout_probability, + modulation, + aux_in_features, + fc_hidden_features, + fc_groups, + fc_activation, + num_weights, + norm_type, + adain_hidden_features, + ) + ] + ) ch = num_filters + + if modulation != ModConvType.NONE and modulation_at_input: + modulation = ModConvType.NONE + for _ in range(num_pool_layers - 1): - self.down_sample_layers += [ConvBlock3D(ch, ch * 2, dropout_probability)] + self.down_sample_layers += [ + ConvBlock3D( + ch, + ch * 2, + dropout_probability, + modulation, + aux_in_features, + fc_hidden_features, + fc_groups, + fc_activation, + num_weights, + norm_type, + adain_hidden_features, + ) + ] ch *= 2 - self.conv = ConvBlock3D(ch, ch * 2, dropout_probability) + self.conv = ConvBlock3D( + ch, + ch * 2, + dropout_probability, + modulation, + aux_in_features, + fc_hidden_features, + fc_groups, + fc_activation, + num_weights, + norm_type, + adain_hidden_features, + ) self.up_conv = nn.ModuleList() self.up_transpose_conv = nn.ModuleList() for _ in range(num_pool_layers - 1): - self.up_transpose_conv += [TransposeConvBlock3D(ch * 2, ch)] - self.up_conv += [ConvBlock3D(ch * 2, ch, dropout_probability)] + self.up_transpose_conv += [ + TransposeConvBlock3D( + ch * 2, + ch, + modulation, + aux_in_features, + fc_hidden_features, + fc_groups, + fc_activation, + num_weights, + norm_type, + adain_hidden_features, + ) + ] + self.up_conv += [ + ConvBlock3D( + ch * 2, + ch, + dropout_probability, + modulation, + aux_in_features, + fc_hidden_features, + fc_groups, + fc_activation, + num_weights, + norm_type, + adain_hidden_features, + ) + ] ch //= 2 - self.up_transpose_conv += [TransposeConvBlock3D(ch * 2, ch)] + if modulation != ModConvType.NONE and modulation_at_input: + modulation = ModConvType.NONE + self.up_transpose_conv += [ + TransposeConvBlock3D( + ch * 2, + ch, + modulation, + aux_in_features, + fc_hidden_features, + fc_groups, + fc_activation, + num_weights, + norm_type, + adain_hidden_features, + ) + ] self.up_conv += [ nn.Sequential( - ConvBlock3D(ch * 2, ch, dropout_probability), + ConvBlock3D( + ch * 2, + ch, + dropout_probability, + modulation, + aux_in_features, + fc_hidden_features, + fc_groups, + fc_activation, + num_weights, + norm_type, + adain_hidden_features, + ), nn.Conv3d(ch, out_channels, kernel_size=1, stride=1), ) ] - def forward(self, input_data: torch.Tensor) -> torch.Tensor: + def forward(self, input_data: torch.Tensor, aux_data: Optional[torch.Tensor] = None) -> torch.Tensor: """Performs forward pass of :class:`UnetModel3d`. Parameters ---------- input_data : torch.Tensor Input tensor of shape (N, in_channels, slice/time, height, width). + aux_data : torch.Tensor, optional + Auxiliary data for modulation/AdaIN. Returns ------- @@ -185,18 +512,25 @@ def forward(self, input_data: torch.Tensor) -> torch.Tensor: stack = [] output, inp_pad = pad_to_pow_of_2(input_data, self.num_pool_layers) - # Apply down-sampling layers - for _, layer in enumerate(self.down_sample_layers): - output = layer(output) + for layer in self.down_sample_layers: + if self.modulation != ModConvType.NONE or self.norm_type == NormType.ADAIN: + output = layer(output, aux_data) + else: + output = layer(output) stack.append(output) output = F.avg_pool3d(output, kernel_size=2, stride=2, padding=0) - output = self.conv(output) + if self.modulation != ModConvType.NONE or self.norm_type == NormType.ADAIN: + output = self.conv(output, aux_data) + else: + output = self.conv(output) - # Apply up-sampling layers for transpose_conv, conv in zip(self.up_transpose_conv, self.up_conv): downsample_layer = stack.pop() - output = transpose_conv(output) + if self.modulation != ModConvType.NONE or self.norm_type == NormType.ADAIN: + output = transpose_conv(output, aux_data) + else: + output = transpose_conv(output) padding = [0, 0, 0, 0, 0, 0] if output.shape[-1] != downsample_layer.shape[-1]: @@ -209,7 +543,14 @@ def forward(self, input_data: torch.Tensor) -> torch.Tensor: output = F.pad(output, padding, "reflect") output = torch.cat([output, downsample_layer], dim=1) - output = conv(output) + if self.modulation != ModConvType.NONE or self.norm_type == NormType.ADAIN: + if isinstance(conv, nn.Sequential): + output = conv[0](output, aux_data) + output = conv[1](output) + else: + output = conv(output, aux_data) + else: + output = conv(output) if sum(inp_pad) != 0: output = output[ @@ -224,7 +565,7 @@ def forward(self, input_data: torch.Tensor) -> torch.Tensor: class NormUnetModel3d(nn.Module): - """Implementation of a Normalized U-Net model for 3D data. + """Implementation of a Normalized U-Net model for 3D data with optional modulation. This is an extension to 3D volumes of :class:`direct.nn.unet.unet_2d.NormUnetModel2d`. """ @@ -237,8 +578,17 @@ def __init__( num_pool_layers: int, dropout_probability: float, norm_groups: int = 2, + modulation: ModConvType = ModConvType.NONE, + aux_in_features: Optional[int] = None, + fc_hidden_features: Optional[tuple[int] | int] = None, + fc_groups: int = 1, + fc_activation: ModConvActivation = ModConvActivation.SIGMOID, + num_weights: Optional[int] = None, + modulation_at_input: bool = False, + norm_type: NormType = NormType.INSTANCE, + adain_hidden_features: Optional[tuple[int] | int] = None, ) -> None: - """Inits :class:`NormUnetModel3D`. + """Inits :class:`NormUnetModel3d`. Parameters ---------- @@ -252,8 +602,26 @@ def __init__( Number of down-sampling and up-sampling layers (depth). dropout_probability : float Dropout probability. - norm_groups: int, + norm_groups: int Number of normalization groups. + modulation : ModConvType + Modulation type. Default: ModConvType.NONE. + aux_in_features : int, optional + Number of auxiliary input features. + fc_hidden_features : int or tuple of int, optional + Hidden features for the modulation MLP. + fc_groups : int + Groups for the modulation MLP. Default: 1. + fc_activation : ModConvActivation + Activation for the modulation MLP. Default: ModConvActivation.SIGMOID. + num_weights : int, optional + Number of weight bases for ModConvType.SUM. + modulation_at_input : bool + If True, only the first conv block uses modulation. Default: False. + norm_type : NormType + Normalization type. Default: NormType.INSTANCE. + adain_hidden_features : int or tuple of int, optional + Hidden features for AdaIN. """ super().__init__() @@ -263,27 +631,24 @@ def __init__( num_filters=num_filters, num_pool_layers=num_pool_layers, dropout_probability=dropout_probability, + modulation=modulation, + aux_in_features=aux_in_features, + fc_hidden_features=fc_hidden_features, + fc_groups=fc_groups, + fc_activation=fc_activation, + num_weights=num_weights, + modulation_at_input=modulation_at_input, + norm_type=norm_type, + adain_hidden_features=adain_hidden_features, ) self.norm_groups = norm_groups + self.modulation = modulation + self.norm_type = norm_type @staticmethod def norm(input_data: torch.Tensor, groups: int) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """Applies group normalization for 3D data. - - Parameters - ---------- - input_data : torch.Tensor - The input tensor to normalize. - groups : int - The number of groups to divide the tensor into for normalization. - - Returns - ------- - tuple[torch.Tensor, torch.Tensor, torch.Tensor] - A tuple containing the normalized tensor, the mean, and the standard deviation used for normalization. - """ - # Group norm + """Performs group normalization.""" b, c, z, h, w = input_data.shape input_data = input_data.reshape(b, groups, -1) @@ -296,30 +661,7 @@ def norm(input_data: torch.Tensor, groups: int) -> tuple[torch.Tensor, torch.Ten return output, mean, std @staticmethod - def unnorm( - input_data: torch.Tensor, - mean: torch.Tensor, - std: torch.Tensor, - groups: int, - ) -> torch.Tensor: - """Reverts the normalization applied to the 3D tensor. - - Parameters - ---------- - input_data : torch.Tensor - The normalized tensor to revert normalization on. - mean : torch.Tensor - The mean used during normalization. - std : torch.Tensor - The standard deviation used during normalization. - groups : int - The number of groups the tensor was divided into during normalization. - - Returns - ------- - torch.Tensor - The tensor after reverting the normalization. - """ + def unnorm(input_data: torch.Tensor, mean: torch.Tensor, std: torch.Tensor, groups: int) -> torch.Tensor: b, c, z, h, w = input_data.shape input_data = input_data.reshape(b, groups, -1) return (input_data * std + mean).reshape(b, c, z, h, w) @@ -328,19 +670,6 @@ def unnorm( def pad( input_data: torch.Tensor, ) -> tuple[torch.Tensor, tuple[list[int], list[int], list[int], int, int, int]]: - """Applies padding to the input 3D tensor to ensure its dimensions are multiples of 16. - - Parameters - ---------- - input_data : torch.Tensor - The input tensor to pad. - - Returns - ------- - tuple[torch.Tensor, tuple[list[int], list[int], int, int, list[int], list[int]]] - A tuple containing the padded tensor and a tuple with the padding applied to each dimension - (height, width, depth) and the target dimensions after padding. - """ _, _, z, h, w = input_data.shape w_mult = ((w - 1) | 15) + 1 h_mult = ((h - 1) | 15) + 1 @@ -362,50 +691,32 @@ def unpad( w_mult: int, z_mult: int, ) -> torch.Tensor: - """Removes padding from the 3D input tensor, reverting it to its original dimensions before padding was applied. - - This method is typically used after the model has processed the padded input. - - Parameters - ---------- - input_data : torch.Tensor - The tensor from which padding will be removed. - h_pad : list[int] - Padding applied to the height, specified as [top, bottom]. - w_pad : list[int] - Padding applied to the width, specified as [left, right]. - z_pad : list[int] - Padding applied to the depth, specified as [front, back]. - h_mult : int - The height as computed in the `pad` method. - w_mult : int - The width as computed in the `pad` method. - z_mult : int - The depth as computed in the `pad` method. - - Returns - ------- - torch.Tensor - The tensor with padding removed, restored to its original dimensions. - """ - return input_data[..., z_pad[0] : z_mult - z_pad[1], h_pad[0] : h_mult - h_pad[1], w_pad[0] : w_mult - w_pad[1]] + return input_data[ + ..., + z_pad[0] : z_mult - z_pad[1], + h_pad[0] : h_mult - h_pad[1], + w_pad[0] : w_mult - w_pad[1], + ] - def forward(self, input_data: torch.Tensor) -> torch.Tensor: - """Performs the forward pass of :class:`NormUnetModel3D`. + def forward(self, input_data: torch.Tensor, aux_data: Optional[torch.Tensor] = None) -> torch.Tensor: + """Performs the forward pass of :class:`NormUnetModel3d`. Parameters ---------- input_data : torch.Tensor - Input tensor of shape (N, in_channels, slice/time, height, width). + aux_data : torch.Tensor, optional Returns ------- torch.Tensor - Output of shape (N, out_channels, slice/time, height, width). """ output, mean, std = self.norm(input_data, self.norm_groups) output, pad_sizes = self.pad(output) - output = self.unet3d(output) + + if self.modulation != ModConvType.NONE or self.norm_type == NormType.ADAIN: + output = self.unet3d(output, aux_data) + else: + output = self.unet3d(output) h_pad, w_pad, z_pad, h_mult, w_mult, z_mult = pad_sizes output = self.unpad(output, h_pad, w_pad, z_pad, h_mult, w_mult, z_mult) @@ -415,30 +726,19 @@ def forward(self, input_data: torch.Tensor) -> torch.Tensor: def pad_to_pow_of_2(inp: torch.Tensor, k: int) -> tuple[torch.Tensor, list[int]]: - """Pads the input tensor along the spatial dimensions (depth, height, width) to the nearest power of 2. - - This is necessary for certain operations in the 3D U-Net architecture to maintain dimensionality. + """Pads the input tensor along the spatial dimensions to the nearest power of 2. Parameters ---------- inp : torch.Tensor The input tensor to be padded. k : int - The exponent to which the base of 2 is raised to determine the padding. Used to calculate - the target dimension size as a power of 2. + The exponent to which 2 is raised to determine target dimension size. Returns ------- tuple[torch.Tensor, list[int]] - A tuple containing the padded tensor and a list of padding applied to each spatial dimension - in the format [depth_front, depth_back, height_top, height_bottom, width_left, width_right]. - - Examples - -------- - >>> inp = torch.rand(1, 1, 15, 15, 15) # A random tensor with shape [1, 1, 15, 15, 15] - >>> padded_inp, padding = pad_to_pow_of_2(inp, 4) - >>> print(padded_inp.shape, padding) - torch.Size([...]), [1, 1, 1, 1, 1, 1] + A tuple containing the padded tensor and the padding list. """ diffs = [_ - 2**k for _ in inp.shape[2:]] padding = [0, 0, 0, 0, 0, 0] diff --git a/direct/nn/unet/unet_engine.py b/direct/nn/unet/unet_engine.py index beeedd09..a3ef4b97 100644 --- a/direct/nn/unet/unet_engine.py +++ b/direct/nn/unet/unet_engine.py @@ -110,7 +110,11 @@ def forward_function(self, data: dict[str, Any]) -> tuple[torch.Tensor, None]: data["sensitivity_map"] if self.cfg.model.image_initialization == "sense" else None # type: ignore ) - output_image = self.model(masked_kspace=data["masked_kspace"], sensitivity_map=sensitity_map) + output_image = self.model( + masked_kspace=data["masked_kspace"], + sensitivity_map=sensitity_map, + auxiliary_data=self.auxiliary_data_from(data), + ) output_image = T.modulus(output_image) output_kspace = None @@ -207,7 +211,11 @@ def forward_function(self, data: dict[str, Any]) -> tuple[torch.Tensor, None]: data["sensitivity_map"] if self.cfg.model.image_initialization == "sense" else None # type: ignore ) - output_image = self.model(masked_kspace=kspace, sensitivity_map=sensitity_map) + output_image = self.model( + masked_kspace=kspace, + sensitivity_map=sensitity_map, + auxiliary_data=self.auxiliary_data_from(data), + ) output_kspace = None return output_image, output_kspace @@ -308,7 +316,11 @@ def forward_function(self, data: dict[str, Any]) -> tuple[torch.Tensor, None]: data["sensitivity_map"] if self.cfg.model.image_initialization == "sense" else None # type: ignore ) - output_image = self.model(masked_kspace=kspace, sensitivity_map=sensitity_map) + output_image = self.model( + masked_kspace=kspace, + sensitivity_map=sensitity_map, + auxiliary_data=self.auxiliary_data_from(data), + ) output_kspace = None return output_image, output_kspace diff --git a/direct/nn/varnet/config.py b/direct/nn/varnet/config.py index af49108c..9c8e5f57 100644 --- a/direct/nn/varnet/config.py +++ b/direct/nn/varnet/config.py @@ -12,8 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. from dataclasses import dataclass +from typing import Optional from direct.config.defaults import ModelConfig +from direct.nn.conv.modulated import ModConvActivation, ModConvType @dataclass @@ -22,3 +24,11 @@ class EndToEndVarNetConfig(ModelConfig): regularizer_num_filters: int = 18 regularizer_num_pull_layers: int = 4 regularizer_dropout: float = 0.0 + conv_modulation: ModConvType = ModConvType.NONE + aux_in_features: Optional[int] = None + auxiliary_features: Optional[tuple[str, ...]] = None + log_aux: bool = False + fc_hidden_features: Optional[tuple[int, ...]] = None + fc_groups: int = 1 + fc_activation: ModConvActivation = ModConvActivation.SIGMOID + num_weights: Optional[int] = None diff --git a/direct/nn/varnet/varnet.py b/direct/nn/varnet/varnet.py index 56b19d0a..59657ce0 100644 --- a/direct/nn/varnet/varnet.py +++ b/direct/nn/varnet/varnet.py @@ -12,10 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import annotations + +from typing import Optional + import torch import torch.nn as nn from direct.data.transforms import expand_operator, reduce_operator +from direct.nn.conv.modulated import ModConvActivation, ModConvType from direct.nn.unet import UnetModel2d from direct.types import FFTOperator @@ -23,11 +28,17 @@ class EndToEndVarNet(nn.Module): """End-to-End Variational Network based on [1]_. + Supports conditional weight modulation as proposed in [2]_. + References ---------- - .. [1] Sriram, Anuroop, et al. “End-to-End Variational Networks for Accelerated MRI Reconstruction.” + .. [1] Sriram, Anuroop, et al. "End-to-End Variational Networks for Accelerated MRI Reconstruction." ArXiv:2004.06688 [Cs, Eess], Apr. 2020. arXiv.org, http://arxiv.org/abs/2004.06688. + + .. [2] Moriakov, N., Yiasemis, G., Sonke, J.-J. & Teuwen, J. (2026). Conditional Learned Reconstruction for + Medical Imaging. Proceedings of The 9th International Conference on Medical Imaging with Deep Learning, + PMLR 315:754-780. https://proceedings.mlr.press/v315/moriakov26a.html """ def __init__( @@ -39,6 +50,12 @@ def __init__( regularizer_num_pull_layers: int = 4, regularizer_dropout: float = 0.0, in_channels: int = 2, + conv_modulation: ModConvType = ModConvType.NONE, + aux_in_features: Optional[int] = None, + fc_hidden_features: Optional[tuple[int] | int] = None, + fc_groups: int = 1, + fc_activation: ModConvActivation = ModConvActivation.SIGMOID, + num_weights: Optional[int] = None, **kwargs, ): """Inits :class:`EndToEndVarNet`. @@ -57,15 +74,32 @@ def __init__( Regularizer model number of pulling layers. regularizer_dropout: float Regularizer model dropout probability. + in_channels: int + Number of input channels. Default: 2. + conv_modulation : ModConvType + Modulation type for convolutional layers. Default: ModConvType.NONE. + aux_in_features : int, optional + Number of features in the auxiliary input for modulation. + fc_hidden_features : int or tuple of int, optional + Hidden features in the modulation MLP. + fc_groups : int + Groups for modulation MLP output. Default: 1. + fc_activation : ModConvActivation + Activation after modulation MLP. Default: ModConvActivation.SIGMOID. + num_weights : int, optional + Number of weight bases for ModConvType.SUM. """ super().__init__() extra_keys = kwargs.keys() for extra_key in extra_keys: - if extra_key not in [ + if extra_key not in ( "model_name", - ]: + "log_aux", + "auxiliary_features", + ): raise ValueError(f"{type(self).__name__} got key `{extra_key}` which is not supported.") + self.conv_modulation = conv_modulation self.layers_list = nn.ModuleList() for _ in range(num_layers): @@ -79,12 +113,23 @@ def __init__( num_filters=regularizer_num_filters, num_pool_layers=regularizer_num_pull_layers, dropout_probability=regularizer_dropout, + modulation=conv_modulation, + aux_in_features=aux_in_features, + fc_hidden_features=fc_hidden_features, + fc_groups=fc_groups, + fc_activation=fc_activation, + num_weights=num_weights, ), + conv_modulation=conv_modulation, ) ) def forward( - self, masked_kspace: torch.Tensor, sampling_mask: torch.Tensor, sensitivity_map: torch.Tensor + self, + masked_kspace: torch.Tensor, + sampling_mask: torch.Tensor, + sensitivity_map: torch.Tensor, + auxiliary_data: Optional[torch.Tensor] = None, ) -> torch.Tensor: """Performs the forward pass of :class:`EndToEndVarNet`. @@ -96,6 +141,8 @@ def forward( Sampling mask of shape (N, 1, height, width, 1). sensitivity_map: torch.Tensor Sensitivity map of shape (N, coil, height, width, complex=2). + auxiliary_data: torch.Tensor, optional + Auxiliary data for modulation of shape (N, aux_in_features). Returns ------- @@ -105,7 +152,13 @@ def forward( kspace_prediction = masked_kspace.clone() for layer in self.layers_list: - kspace_prediction = layer(kspace_prediction, masked_kspace, sampling_mask, sensitivity_map) + kspace_prediction = layer( + kspace_prediction, + masked_kspace, + sampling_mask, + sensitivity_map, + auxiliary_data, + ) return kspace_prediction @@ -117,6 +170,7 @@ def __init__( forward_operator: FFTOperator, backward_operator: FFTOperator, regularizer_model: nn.Module, + conv_modulation: ModConvType = ModConvType.NONE, ): """Inits :class:`EndToEndVarNetBlock`. @@ -128,12 +182,15 @@ def __init__( Backward Operator. regularizer_model: nn.Module Regularizer model. + conv_modulation: ModConvType + Modulation type. Default: ModConvType.NONE. """ super().__init__() self.regularizer_model = regularizer_model self.forward_operator = forward_operator self.backward_operator = backward_operator self.learning_rate = nn.Parameter(torch.tensor([1.0])) + self.conv_modulation = conv_modulation self._coil_dim = 1 self._complex_dim = -1 self._spatial_dims = (2, 3) @@ -144,6 +201,7 @@ def forward( masked_kspace: torch.Tensor, sampling_mask: torch.Tensor, sensitivity_map: torch.Tensor, + auxiliary_data: Optional[torch.Tensor] = None, ) -> torch.Tensor: """Performs the forward pass of :class:`EndToEndVarNetBlock`. @@ -157,6 +215,8 @@ def forward( Sampling mask of shape (N, 1, height, width, 1). sensitivity_map: torch.Tensor Sensitivity map of shape (N, coil, height, width, complex=2). + auxiliary_data: torch.Tensor, optional + Auxiliary data for modulation of shape (N, aux_in_features). Returns ------- @@ -171,17 +231,25 @@ def forward( regularization_term = torch.cat( [ reduce_operator( - self.backward_operator(kspace, dim=self._spatial_dims), sensitivity_map, dim=self._coil_dim + self.backward_operator(kspace, dim=self._spatial_dims), + sensitivity_map, + dim=self._coil_dim, ) for kspace in torch.split(current_kspace, 2, self._complex_dim) ], dim=self._complex_dim, ).permute(0, 3, 1, 2) - regularization_term = self.regularizer_model(regularization_term).permute(0, 2, 3, 1) + + if self.conv_modulation != ModConvType.NONE: + regularization_term = self.regularizer_model(regularization_term, auxiliary_data).permute(0, 2, 3, 1) + else: + regularization_term = self.regularizer_model(regularization_term).permute(0, 2, 3, 1) + regularization_term = torch.cat( [ self.forward_operator( - expand_operator(image, sensitivity_map, dim=self._coil_dim), dim=self._spatial_dims + expand_operator(image, sensitivity_map, dim=self._coil_dim), + dim=self._spatial_dims, ) for image in torch.split(regularization_term, 2, self._complex_dim) ], diff --git a/direct/nn/varnet/varnet_engine.py b/direct/nn/varnet/varnet_engine.py index 22fbd4b3..47df7b3d 100644 --- a/direct/nn/varnet/varnet_engine.py +++ b/direct/nn/varnet/varnet_engine.py @@ -92,10 +92,13 @@ def __init__( ) def forward_function(self, data: dict[str, Any]) -> tuple[torch.Tensor, torch.Tensor]: + auxiliary_data = self.auxiliary_data_from(data) + output_kspace = self.model( masked_kspace=data["masked_kspace"], sampling_mask=data["sampling_mask"], sensitivity_map=data["sensitivity_map"], + auxiliary_data=auxiliary_data, ) output_image = T.root_sum_of_squares( self.backward_operator(output_kspace, dim=self._spatial_dims), @@ -197,6 +200,7 @@ def forward_function(self, data: dict[str, Any]) -> tuple[None, torch.Tensor]: masked_kspace=kspace, sampling_mask=mask, sensitivity_map=data["sensitivity_map"], + auxiliary_data=self.auxiliary_data_from(data), ) output_image = None @@ -298,6 +302,7 @@ def forward_function(self, data: dict[str, Any]) -> tuple[None, torch.Tensor]: masked_kspace=kspace, sampling_mask=mask, sensitivity_map=data["sensitivity_map"], + auxiliary_data=self.auxiliary_data_from(data), ) output_image = None diff --git a/direct/nn/vsharp/config.py b/direct/nn/vsharp/config.py index 0a39e942..9d6eda32 100644 --- a/direct/nn/vsharp/config.py +++ b/direct/nn/vsharp/config.py @@ -15,8 +15,11 @@ from __future__ import annotations from dataclasses import dataclass +from typing import Optional from direct.config.defaults import ModelConfig +from direct.nn.adain.adain import NormType +from direct.nn.conv.modulated import ModConvActivation, ModConvType from direct.nn.types import ActivationType, InitType, ModelName @@ -32,6 +35,15 @@ class VSharpNetConfig(ModelConfig): initializer_dilations: tuple[int, ...] = (1, 1, 2, 4) initializer_multiscale: int = 1 initializer_activation: ActivationType = ActivationType.PRELU + conv_modulation: ModConvType = ModConvType.NONE + aux_in_features: int = 2 + auxiliary_features: Optional[tuple[str, ...]] = None + log_aux: bool = False + fc_hidden_features: Optional[tuple[int]] = None + fc_groups: int = 1 + fc_activation: ModConvActivation = ModConvActivation.SIGMOID + num_weights: Optional[int] = None + modulation_at_input: bool = False image_resnet_hidden_channels: int = 128 image_resnet_num_blocks: int = 15 image_resnet_batchnorm: bool = True @@ -39,6 +51,8 @@ class VSharpNetConfig(ModelConfig): image_unet_num_filters: int = 32 image_unet_num_pool_layers: int = 4 image_unet_dropout: float = 0.0 + image_unet_norm_type: NormType = NormType.INSTANCE + image_unet_adain_hidden_features: Optional[tuple[int]] = None image_didn_hidden_channels: int = 16 image_didn_num_dubs: int = 6 image_didn_num_convs_recon: int = 9 @@ -59,7 +73,18 @@ class VSharpNet3DConfig(ModelConfig): initializer_dilations: tuple[int, ...] = (1, 1, 2, 4) initializer_multiscale: int = 1 initializer_activation: ActivationType = ActivationType.PRELU + conv_modulation: ModConvType = ModConvType.NONE + aux_in_features: int = 2 + auxiliary_features: Optional[tuple[str, ...]] = None + fc_hidden_features: Optional[tuple[int]] = None + fc_groups: int = 1 + fc_activation: ModConvActivation = ModConvActivation.SIGMOID + num_weights: Optional[int] = None + modulation_at_input: bool = False + log_aux: bool = True unet_num_filters: int = 32 unet_num_pool_layers: int = 4 unet_dropout: float = 0.0 unet_norm: bool = False + unet_norm_type: NormType = NormType.INSTANCE + unet_adain_hidden_features: Optional[tuple[int]] = None diff --git a/direct/nn/vsharp/vsharp.py b/direct/nn/vsharp/vsharp.py index e9c1b568..c25352c6 100644 --- a/direct/nn/vsharp/vsharp.py +++ b/direct/nn/vsharp/vsharp.py @@ -27,6 +27,8 @@ from __future__ import annotations +from typing import Optional, cast + import numpy as np import torch import torch.nn.functional as F @@ -34,6 +36,15 @@ from direct.constants import COMPLEX_SIZE from direct.data.transforms import apply_mask, expand_operator, reduce_operator +from direct.nn.adain.adain import NormType +from direct.nn.conv.modulated import ( + ModConv2dBias, + ModConv3d, + ModConvActivation, + ModConvType, + ModulationParams, + mod_conv2d, +) from direct.nn.get_nn_model_config import ModelName, _get_model_config, _get_relu_activation from direct.nn.types import ActivationType, InitType from direct.nn.unet.unet_3d import NormUnetModel3d, UnetModel3d @@ -41,13 +52,13 @@ class LagrangeMultipliersInitializer(nn.Module): - """A convolutional neural network model that initializers the Lagrange multiplier of the :class:`vSHARPNet` [1]_. + """A convolutional neural network model that initializes the Lagrange multiplier of the :class:`VSharpNet` [1]_. More specifically, it produces an initial value for the Lagrange Multiplier based on the zero-filled image: .. math:: - u^0 = \mathcal{G}_{\psi}(x^0). + u^0 = \\mathcal{G}_{\\psi}(x^0). References ---------- @@ -63,6 +74,13 @@ def __init__( dilations: tuple[int, ...], multiscale_depth: int = 1, activation: ActivationType = ActivationType.PRELU, + conv_modulation: ModConvType = ModConvType.NONE, + aux_in_features: Optional[int] = None, + fc_hidden_features: Optional[tuple[int] | int] = None, + fc_groups: int = 1, + fc_activation: ModConvActivation = ModConvActivation.SIGMOID, + num_weights: Optional[int] = None, + modulation_at_input: bool = False, ) -> None: """Inits :class:`LagrangeMultipliersInitializer`. @@ -73,103 +91,149 @@ def __init__( out_channels : int Number of output channels. channels : tuple of ints - Tuple of integers specifying the number of output channels for each convolutional layer in the network. + Number of output channels for each convolutional layer. dilations : tuple of ints - Tuple of integers specifying the dilation factor for each convolutional layer in the network. + Dilation factor for each convolutional layer. multiscale_depth : int Number of multiscale features to include in the output. Default: 1. activation : ActivationType - Activation function to use on the output. Default: ActivationType.PRELU. + Activation function. Default: ActivationType.PRELU. + conv_modulation : ModConvType + Modulation type. Default: ModConvType.NONE. + aux_in_features : int, optional + Number of auxiliary input features. + fc_hidden_features : int or tuple of int, optional + Hidden features for the modulation MLP. + fc_groups : int + Groups for the modulation MLP. Default: 1. + fc_activation : ModConvActivation + Activation for the modulation MLP. Default: ModConvActivation.SIGMOID. + num_weights : int, optional + Number of weight bases for ModConvType.SUM. + modulation_at_input : bool + If True, modulation is only applied at the first layers. Default: False. """ super().__init__() - # Define convolutional blocks + modulation_params = ModulationParams( + modulation=conv_modulation, + aux_in_features=aux_in_features, + fc_hidden_features=fc_hidden_features, + fc_groups=fc_groups, + fc_activation=fc_activation, + num_weights=num_weights, + ) + self.conv_blocks = nn.ModuleList() tch = in_channels - for curr_channels, curr_dilations in zip(channels, dilations): - block = nn.Sequential( - nn.ReplicationPad2d(curr_dilations), - nn.Conv2d(tch, curr_channels, 3, padding=0, dilation=curr_dilations), + for i, (curr_channels, curr_dilations) in enumerate(zip(channels, dilations)): + block_modulation_params = modulation_params + if conv_modulation != ModConvType.NONE: + if modulation_at_input and i > 1: + block_modulation_params = ModulationParams( + modulation=ModConvType.NONE, + aux_in_features=aux_in_features, + fc_hidden_features=fc_hidden_features, + fc_groups=fc_groups, + fc_activation=fc_activation, + num_weights=num_weights, + ) + + block = nn.ModuleList( + [ + nn.ReplicationPad2d(curr_dilations), + mod_conv2d( + tch, + curr_channels, + kernel_size=3, + padding=0, + dilation=curr_dilations, + bias=( + ModConv2dBias.NONE + if block_modulation_params.modulation == ModConvType.NONE + else ModConv2dBias.LEARNED + ), + modulation_params=block_modulation_params, + ), + ] ) tch = curr_channels self.conv_blocks.append(block) - # Define output block - tch = np.sum(channels[-multiscale_depth:]) - block = nn.Conv2d(tch, out_channels, 1, padding=0) - self.out_block = nn.Sequential(block) + out_modulation_params = modulation_params + if (conv_modulation != ModConvType.NONE) and modulation_at_input: + out_modulation_params = ModulationParams( + modulation=ModConvType.NONE, + aux_in_features=aux_in_features, + fc_hidden_features=fc_hidden_features, + fc_groups=fc_groups, + fc_activation=fc_activation, + num_weights=num_weights, + ) + tch = np.sum(channels[-multiscale_depth:]).item() + self.out_block = mod_conv2d( + tch, + out_channels, + kernel_size=1, + padding=0, + bias=( + ModConv2dBias.NONE if out_modulation_params.modulation == ModConvType.NONE else ModConv2dBias.LEARNED + ), + modulation_params=out_modulation_params, + ) self.multiscale_depth = multiscale_depth - self.activation = _get_relu_activation(activation) + self.conv_modulation = conv_modulation - def forward(self, x: torch.Tensor) -> torch.Tensor: + def forward(self, x: torch.Tensor, y: Optional[torch.Tensor] = None) -> torch.Tensor: """Forward pass of :class:`LagrangeMultipliersInitializer`. Parameters ---------- x : torch.Tensor Input tensor of shape (batch_size, in_channels, height, width). + y : torch.Tensor, optional + Auxiliary tensor of shape (batch_size, aux_in_features). Default: None. Returns ------- torch.Tensor Output tensor of shape (batch_size, out_channels, height, width). """ - features = [] for block in self.conv_blocks: - x = F.relu(block(x), inplace=True) + block_modules = cast(nn.ModuleList, block) + x = block_modules[0](x) + if self.conv_modulation != ModConvType.NONE: + x = F.relu(block_modules[1](x, y), inplace=True) + else: + x = F.relu(block_modules[1](x), inplace=True) if self.multiscale_depth > 1: features.append(x) if self.multiscale_depth > 1: x = torch.cat(features[-self.multiscale_depth :], dim=1) + if self.conv_modulation != ModConvType.NONE: + return self.activation(self.out_block(x, y)) return self.activation(self.out_block(x)) class VSharpNet(nn.Module): - """ - Variable Splitting Half-quadratic ADMM algorithm for Reconstruction of Parallel MRI [1]_. - - Variable Splitting Half Quadratic VSharpNet is a deep learning model that solves - the augmented Lagrangian derivation of the variable half quadratic splitting problem - using ADMM (Alternating Direction Method of Multipliers). It is specifically designed - for solving inverse problems in magnetic resonance imaging (MRI). - - The VSharpNet model incorporates an iterative optimization algorithm that consists of - three steps: z-step, x-step, and u-step. These steps are detailed mathematically as follows: - - .. math:: - - z^{t+1} = \mathrm{argmin}_{z} \\lambda \mathcal{G}(z) + \\frac{\\rho}{2} || x^{t} - z + - \\frac{u^t}{\\rho} ||_2^2 \\quad \mathrm{[z-step]} - - .. math:: - - x^{t+1} = \mathrm{argmin}_{x} \\frac{1}{2} || \mathcal{A}_{\mathbf{U},\mathbf{S}}(x) - \\tilde{y} ||_2^2 + - \\frac{\\rho}{2} || x - z^{t+1} + \\frac{u^t}{\\rho} ||_2^2 \\quad \mathrm{[x-step]} - - .. math:: - - u^{t+1} = u^t + \\rho (x^{t+1} - z^{t+1}) \\quad \mathrm{[u-step]} - - During the z-step, the model minimizes the augmented Lagrangian function with respect to z, utilizing - DL-based denoisers. In the x-step, it optimizes x by minimizing the data consistency term through - unrolling a gradient descent scheme (DC-GD). The u-step involves updating the Lagrange multiplier u. - These steps are iterated for a specified number of cycles. - - The model includes an initializer for Lagrange multipliers. - - It also allows for outputting auxiliary steps. + """Variable Splitting Half-quadratic ADMM algorithm for Reconstruction of Parallel MRI [1]_. - :class:`VSharpNet` is tailored for 2D MRI data reconstruction. + This model incorporates an iterative optimization algorithm (z-step, x-step, u-step) and + supports optional modulated convolutions conditioned on auxiliary data as proposed in [2]_. References ---------- .. [1] George Yiasemis et al., "VSHARP: Variable Splitting Half-quadratic ADMM Algorithm for Reconstruction of Inverse Problems" (2023). https://arxiv.org/abs/2309.09954. + + .. [2] Moriakov, N., Yiasemis, G., Sonke, J.-J. & Teuwen, J. (2026). Conditional Learned Reconstruction for + Medical Imaging. Proceedings of The 9th International Conference on Medical Imaging with Deep Learning, + PMLR 315:754-780. https://proceedings.mlr.press/v315/moriakov26a.html """ def __init__( @@ -186,58 +250,80 @@ def __init__( initializer_multiscale: int = 1, initializer_activation: ActivationType = ActivationType.PRELU, auxiliary_steps: int = 0, + conv_modulation: ModConvType = ModConvType.NONE, + aux_in_features: Optional[int] = None, + fc_hidden_features: Optional[tuple[int] | int] = None, + fc_groups: int = 1, + fc_activation: ModConvActivation = ModConvActivation.SIGMOID, + num_weights: Optional[int] = None, + modulation_at_input: bool = False, **kwargs, ) -> None: """Inits :class:`VSharpNet`. Parameters ---------- - forward_operator : Callable[[tuple[Any, ...]], torch.Tensor] + forward_operator : FFTOperator Forward operator function. - backward_operator : Callable[[tuple[Any, ...]], torch.Tensor] + backward_operator : FFTOperator Backward operator function. num_steps : int Number of steps in the ADMM algorithm. num_steps_dc_gd : int - Number of steps in the Data Consistency using Gradient Descent step of ADMM. - image_init : str - Image initialization method. Default: 'sense'. + Number of gradient descent steps for data consistency. + image_init : InitType + Image initialization method. Default: InitType.SENSE. no_parameter_sharing : bool - Flag indicating whether parameter sharing is enabled in the denoiser blocks. + If True, each ADMM step has its own denoiser. Default: True. image_model_architecture : ModelName - Image model architecture. Default: ModelName.UNET. + Denoiser model architecture. Default: ModelName.UNET. initializer_channels : tuple[int, ...] - Tuple of integers specifying the number of output channels for each convolutional layer in the - Lagrange multiplier initializer. Default: (32, 32, 64, 64). + Output channels for the Lagrange initializer layers. Default: (32, 32, 64, 64). initializer_dilations : tuple[int, ...] - Tuple of integers specifying the dilation factor for each convolutional layer in the Lagrange multiplier - initializer. Default: (1, 1, 2, 4). + Dilations for the Lagrange initializer layers. Default: (1, 1, 2, 4). initializer_multiscale : int - Number of multiscale features to include in the Lagrange multiplier initializer output. Default: 1. + Multiscale depth for the initializer. Default: 1. initializer_activation : ActivationType - Activation type for the Lagrange multiplier initializer. Default: ActivationType.PRELU. + Activation for the initializer output. Default: ActivationType.PRELU. auxiliary_steps : int - Number of auxiliary steps to output. Can be -1 or a positive integer lower or equal to `num_steps`. - If -1, it uses all steps. If I, the last I steps will be used. - **kwargs: Additional keyword arguments. - Can be `model_name` or `image_model_` where `` represent parameters of the selected - image model architecture beyond the standard parameters. - Depending on the `image_model_architecture` chosen, different kwargs will be applicable. + Number of auxiliary output steps. -1 uses all steps. Default: 0. + conv_modulation : ModConvType + Modulation type for convolutions. Default: ModConvType.NONE. + aux_in_features : int, optional + Number of auxiliary input features. + fc_hidden_features : int or tuple of int, optional + Hidden features for the modulation MLP. + fc_groups : int + Groups for the modulation MLP. Default: 1. + fc_activation : ModConvActivation + Activation for the modulation MLP. Default: ModConvActivation.SIGMOID. + num_weights : int, optional + Number of weight bases for ModConvType.SUM. + modulation_at_input : bool + If True, only the first layers use modulation. Default: False. + **kwargs + Additional keyword arguments for the image model. """ - # pylint: disable=too-many-locals super().__init__() for extra_key in kwargs: - if extra_key != "model_name" and not extra_key.startswith("image_"): + if extra_key not in ("model_name", "log_aux", "auxiliary_features") and not extra_key.startswith("image_"): raise ValueError(f"{type(self).__name__} got key `{extra_key}` which is not supported.") self.num_steps = num_steps self.num_steps_dc_gd = num_steps_dc_gd - self.no_parameter_sharing = no_parameter_sharing + self.conv_modulation = conv_modulation image_model, image_model_kwargs = _get_model_config( image_model_architecture, in_channels=COMPLEX_SIZE * 3, out_channels=COMPLEX_SIZE, + modulation=conv_modulation, + aux_in_features=aux_in_features, + fc_hidden_features=fc_hidden_features, + fc_groups=fc_groups, + fc_activation=fc_activation, + num_weights=num_weights, + modulation_at_input=modulation_at_input, **{k.replace("image_", ""): v for (k, v) in kwargs.items() if "image_" in k}, ) @@ -252,6 +338,13 @@ def __init__( dilations=initializer_dilations, multiscale_depth=initializer_multiscale, activation=initializer_activation, + conv_modulation=conv_modulation, + aux_in_features=aux_in_features, + fc_hidden_features=fc_hidden_features, + fc_groups=fc_groups, + fc_activation=fc_activation, + num_weights=num_weights, + modulation_at_input=modulation_at_input, ) self.learning_rate_eta = nn.Parameter(torch.ones(num_steps_dc_gd, requires_grad=True)) @@ -265,9 +358,8 @@ def __init__( if image_init not in [InitType.SENSE, InitType.ZERO_FILLED]: raise ValueError( - f"Unknown image_initialization. Expected `InitType.SENSE` or `InitType.ZERO_FILLED`. Got {image_init}." + f"Unknown image_initialization. Expected InitType.SENSE or InitType.ZERO_FILLED. Got {image_init}." ) - self.image_init = image_init if not ((auxiliary_steps == -1) or (0 < auxiliary_steps <= num_steps)): @@ -289,6 +381,7 @@ def forward( masked_kspace: torch.Tensor, sensitivity_map: torch.Tensor, sampling_mask: torch.Tensor, + auxiliary_data: Optional[torch.Tensor] = None, ) -> list[torch.Tensor]: """Computes forward pass of :class:`VSharpNet`. @@ -300,6 +393,8 @@ def forward( Sensitivity map of shape (N, coil, height, width, complex=2). sampling_mask: torch.Tensor Sampling mask of shape (N, 1, height, width, 1). + auxiliary_data: torch.Tensor, optional + Auxiliary tensor of shape (N, aux_in_features). Default: None. Returns ------- @@ -318,19 +413,35 @@ def forward( z = x.clone() - u = self.initializer(x.permute(0, 3, 1, 2)).permute(0, 2, 3, 1) + if self.conv_modulation != ModConvType.NONE: + u = self.initializer(x.permute(0, 3, 1, 2), auxiliary_data).permute(0, 2, 3, 1) + else: + u = self.initializer(x.permute(0, 3, 1, 2)).permute(0, 2, 3, 1) for admm_step in range(self.num_steps): - z = self.denoiser_blocks[admm_step if self.no_parameter_sharing else 0]( + denoiser_input = [ torch.cat( [z, x, u / self.rho[admm_step]], dim=self._complex_dim, ).permute(0, 3, 1, 2) - ).permute(0, 2, 3, 1) + ] + if auxiliary_data is not None and ( + self.conv_modulation != ModConvType.NONE + or ( + hasattr(self.denoiser_blocks[0], "norm_type") + and self.denoiser_blocks[0].norm_type == NormType.ADAIN + ) + ): + denoiser_input.append(auxiliary_data) + + z = self.denoiser_blocks[admm_step if self.no_parameter_sharing else 0](*denoiser_input).permute(0, 2, 3, 1) for dc_gd_step in range(self.num_steps_dc_gd): dc = apply_mask( - self.forward_operator(expand_operator(x, sensitivity_map, self._coil_dim), dim=self._spatial_dims) + self.forward_operator( + expand_operator(x, sensitivity_map, self._coil_dim), + dim=self._spatial_dims, + ) - masked_kspace, sampling_mask, return_mask=False, @@ -362,8 +473,15 @@ def __init__( dilations: tuple[int, ...], multiscale_depth: int = 1, activation: ActivationType = ActivationType.PRELU, + conv_modulation: ModConvType = ModConvType.NONE, + aux_in_features: Optional[int] = None, + fc_hidden_features: Optional[tuple[int] | int] = None, + fc_groups: int = 1, + fc_activation: ModConvActivation = ModConvActivation.SIGMOID, + num_weights: Optional[int] = None, + modulation_at_input: bool = False, ): - """Initializes LagrangeMultipliersInitializer3D. + """Initializes :class:`LagrangeMultipliersInitializer3D`. Parameters ---------- @@ -372,70 +490,130 @@ def __init__( out_channels : int Number of output channels. channels : tuple of ints - Tuple of integers specifying the number of output channels for each convolutional layer in the network. + Number of output channels for each convolutional layer. dilations : tuple of ints - Tuple of integers specifying the dilation factor for each convolutional layer in the network. + Dilation factor for each convolutional layer. multiscale_depth : int Number of multiscale features to include in the output. Default: 1. activation : ActivationType - Activation function to use on the output. Default: ActivationType.PRELU. + Activation function. Default: ActivationType.PRELU. + conv_modulation : ModConvType + Modulation type. Default: ModConvType.NONE. + aux_in_features : int, optional + Number of auxiliary input features. + fc_hidden_features : int or tuple of int, optional + Hidden features for the modulation MLP. + fc_groups : int + Groups for the modulation MLP. Default: 1. + fc_activation : ModConvActivation + Activation for the modulation MLP. Default: ModConvActivation.SIGMOID. + num_weights : int, optional + Number of weight bases for ModConvType.SUM. + modulation_at_input : bool + If True, modulation is only applied at the first layers. Default: False. """ super().__init__() - # Define convolutional blocks self.conv_blocks = nn.ModuleList() tch = in_channels - for curr_channels, curr_dilations in zip(channels, dilations): - block = nn.Sequential( - nn.ReplicationPad3d(curr_dilations), - nn.Conv3d(tch, curr_channels, 3, padding=0, dilation=curr_dilations), + for i, (curr_channels, curr_dilations) in enumerate(zip(channels, dilations)): + modulation = conv_modulation + if conv_modulation != ModConvType.NONE: + if modulation_at_input and i > 1: + modulation = ModConvType.NONE + + block = nn.ModuleList( + [ + nn.ReplicationPad3d(curr_dilations), + ModConv3d( + tch, + curr_channels, + kernel_size=3, + padding=0, + dilation=curr_dilations, + modulation=modulation, + bias=(ModConv2dBias.NONE if modulation == ModConvType.NONE else ModConv2dBias.LEARNED), + aux_in_features=aux_in_features, + fc_hidden_features=fc_hidden_features, + fc_groups=fc_groups, + fc_activation=fc_activation, + num_weights=num_weights, + ), + ] ) tch = curr_channels self.conv_blocks.append(block) - # Define output block - tch = np.sum(channels[-multiscale_depth:]) - block = nn.Conv3d(tch, out_channels, 1, padding=0) - self.out_block = nn.Sequential(block) + modulation = ( + ModConvType.NONE if ((conv_modulation != ModConvType.NONE) and modulation_at_input) else conv_modulation + ) + tch = np.sum(channels[-multiscale_depth:]).item() + self.out_block = ModConv3d( + tch, + out_channels, + kernel_size=1, + padding=0, + modulation=modulation, + bias=(ModConv2dBias.NONE if modulation == ModConvType.NONE else ModConv2dBias.LEARNED), + aux_in_features=aux_in_features, + fc_hidden_features=fc_hidden_features, + fc_groups=fc_groups, + fc_activation=fc_activation, + num_weights=num_weights, + ) self.multiscale_depth = multiscale_depth self.activation = _get_relu_activation(activation) + self.conv_modulation = conv_modulation - def forward(self, x: torch.Tensor) -> torch.Tensor: + def forward(self, x: torch.Tensor, y: Optional[torch.Tensor] = None) -> torch.Tensor: """Forward pass of :class:`LagrangeMultipliersInitializer3D`. Parameters ---------- x : torch.Tensor Input tensor of shape (batch_size, in_channels, z, x, y). + y : torch.Tensor, optional + Auxiliary tensor of shape (batch_size, aux_in_features). Default: None. Returns ------- torch.Tensor Output tensor of shape (batch_size, out_channels, z, x, y). """ - features = [] for block in self.conv_blocks: - x = F.relu(block(x), inplace=True) + block_modules = cast(nn.ModuleList, block) + x = block_modules[0](x) + if self.conv_modulation != ModConvType.NONE: + x = F.relu(block_modules[1](x, y), inplace=True) + else: + x = F.relu(block_modules[1](x), inplace=True) if self.multiscale_depth > 1: features.append(x) if self.multiscale_depth > 1: x = torch.cat(features[-self.multiscale_depth :], dim=1) + if self.conv_modulation != ModConvType.NONE: + return self.activation(self.out_block(x, y)) return self.activation(self.out_block(x)) class VSharpNet3D(nn.Module): - """VharpNet 3D version using 3D U-Nets as denoisers. + """VSharpNet 3D version using 3D U-Nets as denoisers. This is an extension to 3D of :class:`VSharpNet`. For the original paper refer to [1]_. + Supports conditional weight modulation as proposed in [2]_. References ---------- .. [1] George Yiasemis et al., "VSHARP: Variable Splitting Half-quadratic ADMM Algorithm for Reconstruction of Inverse Problems" (2023). https://arxiv.org/abs/2309.09954. + + .. [2] Moriakov, N., Yiasemis, G., Sonke, J.-J. & Teuwen, J. (2026). Conditional Learned Reconstruction for + Medical Imaging. Proceedings of The 9th International Conference on Medical Imaging with Deep Learning, + PMLR 315:754-780. https://proceedings.mlr.press/v315/moriakov26a.html """ def __init__( @@ -451,71 +629,106 @@ def __init__( initializer_multiscale: int = 1, initializer_activation: ActivationType = ActivationType.PRELU, auxiliary_steps: int = -1, + conv_modulation: ModConvType = ModConvType.NONE, + aux_in_features: Optional[int] = None, + fc_hidden_features: Optional[tuple[int] | int] = None, + fc_groups: int = 1, + fc_activation: ModConvActivation = ModConvActivation.SIGMOID, + num_weights: Optional[int] = None, + modulation_at_input: bool = False, unet_num_filters: int = 32, unet_num_pool_layers: int = 4, unet_dropout: float = 0.0, unet_norm: bool = False, + unet_norm_type: NormType = NormType.INSTANCE, + unet_adain_hidden_features: Optional[tuple[int]] = None, **kwargs, ): """Inits :class:`VSharpNet3D`. Parameters ---------- - forward_operator : Callable[[tuple[Any, ...]], torch.Tensor] + forward_operator : FFTOperator Forward operator function. - backward_operator : Callable[[tuple[Any, ...]], torch.Tensor] + backward_operator : FFTOperator Backward operator function. num_steps : int Number of steps in the ADMM algorithm. num_steps_dc_gd : int - Number of steps in the Data Consistency using Gradient Descent step of ADMM. - image_init : str - Image initialization method. Default: 'sense'. + Number of gradient descent steps for data consistency. + image_init : InitType + Image initialization method. Default: InitType.SENSE. no_parameter_sharing : bool - Flag indicating whether parameter sharing is enabled in the denoiser blocks. + If True, each ADMM step has its own denoiser. Default: True. initializer_channels : tuple[int, ...] - Tuple of integers specifying the number of output channels for each convolutional layer in the - Lagrange multiplier initializer. Default: (32, 32, 64, 64). + Output channels for the Lagrange initializer layers. Default: (32, 32, 64, 64). initializer_dilations : tuple[int, ...] - Tuple of integers specifying the dilation factor for each convolutional layer in the Lagrange multiplier - initializer. Default: (1, 1, 2, 4). + Dilations for the Lagrange initializer layers. Default: (1, 1, 2, 4). initializer_multiscale : int - Number of multiscale features to include in the Lagrange multiplier initializer output. Default: 1. + Multiscale depth for the initializer. Default: 1. initializer_activation : ActivationType - Activation type for the Lagrange multiplier initializer. Default: ActivationType.PReLU. + Activation for the initializer output. Default: ActivationType.PRELU. auxiliary_steps : int - Number of auxiliary steps to output. Can be -1 or a positive integer lower or equal to `num_steps`. - If -1, it uses all steps. If I, the last I steps will be used. + Number of auxiliary output steps. -1 uses all steps. Default: -1. + conv_modulation : ModConvType + Modulation type for convolutions. Default: ModConvType.NONE. + aux_in_features : int, optional + Number of auxiliary input features. + fc_hidden_features : int or tuple of int, optional + Hidden features for the modulation MLP. + fc_groups : int + Groups for the modulation MLP. Default: 1. + fc_activation : ModConvActivation + Activation for the modulation MLP. Default: ModConvActivation.SIGMOID. + num_weights : int, optional + Number of weight bases for ModConvType.SUM. + modulation_at_input : bool + If True, only the first layers use modulation. Default: False. unet_num_filters : int - U-Net denoisers number of output channels of the first convolutional layer. Default: 32. + U-Net first layer filter count. Default: 32. unet_num_pool_layers : int - U-Net denoisers number of down-sampling and up-sampling layers (depth). Default: 4. + U-Net depth. Default: 4. unet_dropout : float - U-Net denoisers dropout probability. Default: 0.0 + U-Net dropout. Default: 0.0. unet_norm : bool - Whether to use normalized U-Net as denoiser or not. Default: False. - **kwargs: Additional keyword arguments. - Can be `model_name`. + Whether to use normalized U-Net. Default: False. + unet_norm_type : NormType + Normalization type for U-Net. Default: NormType.INSTANCE. + unet_adain_hidden_features : tuple[int], optional + Hidden features for AdaIN in U-Net. + **kwargs + Additional keyword arguments (e.g., model_name, log_aux). """ - # pylint: disable=too-many-locals super().__init__() for extra_key in kwargs: - if extra_key != "model_name": + if extra_key not in ("model_name", "log_aux", "auxiliary_features"): raise ValueError(f"{type(self).__name__} got key `{extra_key}` which is not supported.") self.num_steps = num_steps self.num_steps_dc_gd = num_steps_dc_gd - self.no_parameter_sharing = no_parameter_sharing + self.conv_modulation = conv_modulation + self.unet_norm_type = unet_norm_type + + unet = UnetModel3d if not unet_norm else NormUnetModel3d self.denoiser_blocks = nn.ModuleList() for _ in range(num_steps if self.no_parameter_sharing else 1): self.denoiser_blocks.append( - (UnetModel3d if not unet_norm else NormUnetModel3d)( + unet( in_channels=COMPLEX_SIZE * 3, out_channels=COMPLEX_SIZE, num_filters=unet_num_filters, num_pool_layers=unet_num_pool_layers, dropout_probability=unet_dropout, + modulation=conv_modulation, + aux_in_features=aux_in_features, + fc_hidden_features=fc_hidden_features, + fc_groups=fc_groups, + fc_activation=fc_activation, + num_weights=num_weights, + modulation_at_input=modulation_at_input, + norm_type=unet_norm_type, + adain_hidden_features=unet_adain_hidden_features, ) ) @@ -526,6 +739,13 @@ def __init__( dilations=initializer_dilations, multiscale_depth=initializer_multiscale, activation=initializer_activation, + conv_modulation=conv_modulation, + aux_in_features=aux_in_features, + fc_hidden_features=fc_hidden_features, + fc_groups=fc_groups, + fc_activation=fc_activation, + num_weights=num_weights, + modulation_at_input=modulation_at_input, ) self.learning_rate_eta = nn.Parameter(torch.ones(num_steps_dc_gd, requires_grad=True)) @@ -537,9 +757,10 @@ def __init__( self.forward_operator = forward_operator self.backward_operator = backward_operator - if image_init not in ["sense", "zero_filled"]: - raise ValueError(f"Unknown image_initialization. Expected 'sense' or 'zero_filled'. Got {image_init}.") - + if image_init not in [InitType.SENSE, InitType.ZERO_FILLED]: + raise ValueError( + f"Unknown image_initialization. Expected InitType.SENSE or InitType.ZERO_FILLED. Got {image_init}." + ) self.image_init = image_init if not (auxiliary_steps == -1 or 0 < auxiliary_steps <= num_steps): @@ -561,22 +782,25 @@ def forward( masked_kspace: torch.Tensor, sensitivity_map: torch.Tensor, sampling_mask: torch.Tensor, + auxiliary_data: Optional[torch.Tensor] = None, ) -> list[torch.Tensor]: """Computes forward pass of :class:`VSharpNet3D`. Parameters ---------- - masked_kspace : torch.Tensor + masked_kspace: torch.Tensor Masked k-space of shape (N, coil, slice, height, width, complex=2). - sensitivity_map : torch.Tensor + sensitivity_map: torch.Tensor Sensitivity map of shape (N, coil, slice, height, width, complex=2). - sampling_mask : torch.Tensor + sampling_mask: torch.Tensor Sampling mask of shape (N, 1, 1 or slice, height, width, 1). + auxiliary_data: torch.Tensor, optional + Auxiliary tensor of shape (N, aux_in_features). Default: None. Returns ------- out : list of torch.Tensors - List of output images each of shape (N, slice, height, width, complex=2). + List of output images of shape (N, slice, height, width, complex=2). """ out = [] if self.image_init == InitType.SENSE: @@ -590,19 +814,33 @@ def forward( z = x.clone() - u = self.initializer(x.permute(0, 4, 1, 2, 3)).permute(0, 2, 3, 4, 1) + if self.conv_modulation != ModConvType.NONE: + u = self.initializer(x.permute(0, 4, 1, 2, 3), auxiliary_data).permute(0, 2, 3, 4, 1) + else: + u = self.initializer(x.permute(0, 4, 1, 2, 3)).permute(0, 2, 3, 4, 1) for admm_step in range(self.num_steps): - z = self.denoiser_blocks[admm_step if self.no_parameter_sharing else 0]( + denoiser_input = [ torch.cat( [z, x, u / self.rho[admm_step]], dim=self._complex_dim, ).permute(0, 4, 1, 2, 3) - ).permute(0, 2, 3, 4, 1) + ] + if auxiliary_data is not None and ( + self.conv_modulation != ModConvType.NONE or self.unet_norm_type == NormType.ADAIN + ): + denoiser_input.append(auxiliary_data) + + z = self.denoiser_blocks[admm_step if self.no_parameter_sharing else 0](*denoiser_input).permute( + 0, 2, 3, 4, 1 + ) for dc_gd_step in range(self.num_steps_dc_gd): dc = apply_mask( - self.forward_operator(expand_operator(x, sensitivity_map, self._coil_dim), dim=self._spatial_dims) + self.forward_operator( + expand_operator(x, sensitivity_map, self._coil_dim), + dim=self._spatial_dims, + ) - masked_kspace, sampling_mask, return_mask=False, diff --git a/direct/nn/vsharp/vsharp_engine.py b/direct/nn/vsharp/vsharp_engine.py index 76a9ff87..27421041 100644 --- a/direct/nn/vsharp/vsharp_engine.py +++ b/direct/nn/vsharp/vsharp_engine.py @@ -119,6 +119,8 @@ def _do_iteration( output_kspace: TensorOrNone with autocast("cuda", enabled=self.mixed_precision): + self._attach_auxiliary_data(data) + output_images, output_kspace = self.forward_function(data) output_images = [T.modulus_if_complex(_, complex_axis=self._complex_dim) for _ in output_images] loss_dict = {k: torch.tensor([0.0], dtype=data["target"].dtype).to(self.device) for k in loss_fns.keys()} @@ -126,7 +128,12 @@ def _do_iteration( auxiliary_loss_weights = torch.logspace(-1, 0, steps=len(output_images)).to(output_images[0]) for i, output_image in enumerate(output_images): loss_dict = self.compute_loss_on_data( - loss_dict, loss_fns, data, output_image, None, auxiliary_loss_weights[i] + loss_dict, + loss_fns, + data, + output_image, + None, + auxiliary_loss_weights[i], ) # Compute loss on k-space loss_dict = self.compute_loss_on_data(loss_dict, loss_fns, data, None, output_kspace) @@ -152,6 +159,7 @@ def forward_function(self, data: dict[str, Any]) -> tuple[torch.Tensor, None]: masked_kspace=data["masked_kspace"], sampling_mask=data["sampling_mask"], sensitivity_map=data["sensitivity_map"], + auxiliary_data=self.auxiliary_data_from(data), ) # shape (batch, height, width, complex[=2]) output_image = output_images[-1] @@ -245,6 +253,8 @@ def _do_iteration( output_kspace: TensorOrNone with autocast("cuda", enabled=self.mixed_precision): + self._attach_auxiliary_data(data) + output_images, output_kspace = self.forward_function(data) output_images = [T.modulus_if_complex(_, complex_axis=self._complex_dim) for _ in output_images] loss_dict = {k: torch.tensor([0.0], dtype=data["target"].dtype).to(self.device) for k in loss_fns.keys()} @@ -252,7 +262,12 @@ def _do_iteration( auxiliary_loss_weights = torch.logspace(-1, 0, steps=len(output_images)).to(output_images[0]) for i, output_image in enumerate(output_images): loss_dict = self.compute_loss_on_data( - loss_dict, loss_fns, data, output_image, None, auxiliary_loss_weights[i] + loss_dict, + loss_fns, + data, + output_image, + None, + auxiliary_loss_weights[i], ) # Compute loss on k-space loss_dict = self.compute_loss_on_data(loss_dict, loss_fns, data, None, output_kspace) @@ -278,6 +293,7 @@ def forward_function(self, data: dict[str, Any]) -> tuple[torch.Tensor, torch.Te masked_kspace=data["masked_kspace"], sampling_mask=data["sampling_mask"], sensitivity_map=data["sensitivity_map"], + auxiliary_data=self.auxiliary_data_from(data), ) # shape (batch, height, width, complex[=2]) output_image = output_images[-1] @@ -417,6 +433,7 @@ def _do_iteration( # Move data to device data = dict_to_device(data, self.device) + self._attach_auxiliary_data(data) # Get the k-space and mask which differ during training and inference for SSL if self.model.training: @@ -437,6 +454,7 @@ def _do_iteration( masked_kspace=kspace, sampling_mask=mask, sensitivity_map=data["sensitivity_map"], + auxiliary_data=self.auxiliary_data_from(data), ) if self.model.training: @@ -458,10 +476,20 @@ def _do_iteration( # Compute k-space loss per auxiliary step loss_dict = self.compute_loss_on_data( - loss_dict, loss_fns, data, None, output_kspace, auxiliary_loss_weights[i] + loss_dict, + loss_fns, + data, + None, + output_kspace, + auxiliary_loss_weights[i], ) regularizer_dict = self.compute_loss_on_data( - regularizer_dict, regularizer_fns, data, None, output_kspace, auxiliary_loss_weights[i] + regularizer_dict, + regularizer_fns, + data, + None, + output_kspace, + auxiliary_loss_weights[i], ) # SENSE reconstruction @@ -475,10 +503,20 @@ def _do_iteration( # Compute image loss per auxiliary step loss_dict = self.compute_loss_on_data( - loss_dict, loss_fns, data, output_images[i], None, auxiliary_loss_weights[i] + loss_dict, + loss_fns, + data, + output_images[i], + None, + auxiliary_loss_weights[i], ) regularizer_dict = self.compute_loss_on_data( - regularizer_dict, regularizer_fns, data, output_images[i], None, auxiliary_loss_weights[i] + regularizer_dict, + regularizer_fns, + data, + output_images[i], + None, + auxiliary_loss_weights[i], ) loss = sum(loss_dict.values()) + sum(regularizer_dict.values()) # type: ignore @@ -631,6 +669,7 @@ def _do_iteration( # Move data to device data = dict_to_device(data, self.device) + self._attach_auxiliary_data(data) # Get a boolean indicating if the sample is for SSL training # This will expect the input data to contain the keys "input_kspace" and "input_sampling_mask" if SSL training @@ -656,6 +695,7 @@ def _do_iteration( masked_kspace=kspace, sampling_mask=mask, sensitivity_map=data["sensitivity_map"], + auxiliary_data=self.auxiliary_data_from(data), ) if self.model.training: @@ -674,14 +714,28 @@ def _do_iteration( ) if is_ssl: # Project predicted k-space onto target k-space if SSL - output_kspace = T.apply_mask(output_kspace, data["target_sampling_mask"], return_mask=False) + output_kspace = T.apply_mask( + output_kspace, + data["target_sampling_mask"], + return_mask=False, + ) # Compute k-space loss per auxiliary step loss_dict = self.compute_loss_on_data( - loss_dict, loss_fns, data, None, output_kspace, auxiliary_loss_weights[i] + loss_dict, + loss_fns, + data, + None, + output_kspace, + auxiliary_loss_weights[i], ) regularizer_dict = self.compute_loss_on_data( - regularizer_dict, regularizer_fns, data, None, output_kspace, auxiliary_loss_weights[i] + regularizer_dict, + regularizer_fns, + data, + None, + output_kspace, + auxiliary_loss_weights[i], ) # SENSE reconstruction if SSL else modulus if supervised @@ -697,10 +751,20 @@ def _do_iteration( # Compute image loss per auxiliary step loss_dict = self.compute_loss_on_data( - loss_dict, loss_fns, data, output_images[i], None, auxiliary_loss_weights[i] + loss_dict, + loss_fns, + data, + output_images[i], + None, + auxiliary_loss_weights[i], ) regularizer_dict = self.compute_loss_on_data( - regularizer_dict, regularizer_fns, data, output_images[i], None, auxiliary_loss_weights[i] + regularizer_dict, + regularizer_fns, + data, + output_images[i], + None, + auxiliary_loss_weights[i], ) loss = sum(loss_dict.values()) + sum(regularizer_dict.values()) # type: ignore diff --git a/direct/nn/xpdnet/config.py b/direct/nn/xpdnet/config.py index b6cc7d6a..5971a6ce 100644 --- a/direct/nn/xpdnet/config.py +++ b/direct/nn/xpdnet/config.py @@ -12,8 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. from dataclasses import dataclass +from typing import Optional from direct.config.defaults import ModelConfig +from direct.nn.conv.modulated import ModConvActivation, ModConvType @dataclass @@ -34,3 +36,11 @@ class XPDNetConfig(ModelConfig): mwcnn_bias: bool = True mwcnn_batchnorm: bool = False normalize: bool = False + conv_modulation: ModConvType = ModConvType.NONE + aux_in_features: Optional[int] = None + auxiliary_features: Optional[tuple[str, ...]] = None + log_aux: bool = False + fc_hidden_features: Optional[tuple[int, ...]] = None + fc_groups: int = 1 + fc_activation: ModConvActivation = ModConvActivation.SIGMOID + num_weights: Optional[int] = None diff --git a/direct/nn/xpdnet/xpdnet.py b/direct/nn/xpdnet/xpdnet.py index 61fd06f4..93d16b4e 100644 --- a/direct/nn/xpdnet/xpdnet.py +++ b/direct/nn/xpdnet/xpdnet.py @@ -11,11 +11,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import annotations + from typing import Optional +import torch import torch.nn as nn from direct.nn.conv.conv import Conv2d +from direct.nn.conv.modulated import ModConvActivation, ModConvType, ModulationParams from direct.nn.crossdomain.crossdomain import CrossDomainNetwork from direct.nn.crossdomain.multicoil import MultiCoil from direct.nn.didn.didn import DIDN @@ -23,6 +27,28 @@ from direct.types import FFTOperator +class XPDNetPrimalBlock(nn.Module): + """Primal image block: MWCNN feature extractor followed by a channel projection.""" + + def __init__( + self, + mwcnn: MWCNN, + out_conv: nn.Conv2d, + conv_modulation: ModConvType = ModConvType.NONE, + ) -> None: + super().__init__() + self.mwcnn = mwcnn + self.out_conv = out_conv + self.conv_modulation = conv_modulation + + def forward(self, x: torch.Tensor, y: Optional[torch.Tensor] = None) -> torch.Tensor: + if self.conv_modulation != ModConvType.NONE: + x = self.mwcnn(x, y) + else: + x = self.mwcnn(x) + return self.out_conv(x) + + class XPDNet(CrossDomainNetwork): """XPDNet as implemented in [1]_. @@ -43,6 +69,12 @@ def __init__( image_model_architecture: str = "MWCNN", kspace_model_architecture: Optional[str] = None, normalize: bool = False, + conv_modulation: ModConvType = ModConvType.NONE, + aux_in_features: Optional[int] = None, + fc_hidden_features: Optional[tuple[int] | int] = None, + fc_groups: int = 1, + fc_activation: ModConvActivation = ModConvActivation.SIGMOID, + num_weights: Optional[int] = None, **kwargs, ): """Inits :class:`XPDNet`. @@ -67,9 +99,30 @@ def __init__( Dual-kspace model architecture. Currently only implemented for CONV and DIDN. normalize: bool Normalize input. Default: False. + conv_modulation : ModConvType + Modulation type for convolutional sub-networks. + aux_in_features : int, optional + Number of auxiliary conditioning features. + fc_hidden_features : int or tuple of int, optional + Hidden features in the modulation MLP. + fc_groups : int + Modulation MLP groups. Default: 1. + fc_activation : ModConvActivation + Modulation MLP activation. Default: SIGMOID. + num_weights : int, optional + Number of weight bases for SUM modulation. kwargs: dict Keyword arguments for model architectures. """ + modulation_params = ModulationParams( + modulation=conv_modulation, + aux_in_features=aux_in_features, + fc_hidden_features=fc_hidden_features, + fc_groups=fc_groups, + fc_activation=fc_activation, + num_weights=num_weights, + ) + if use_primal_only: kspace_model_list = None num_dual = 1 @@ -83,6 +136,7 @@ def __init__( kwargs.get("dual_conv_hidden_channels", 16), kwargs.get("dual_conv_n_convs", 4), batchnorm=kwargs.get("dual_conv_batchnorm", False), + modulation_params=modulation_params, ) ) for _ in range(num_iter) @@ -98,6 +152,7 @@ def __init__( hidden_channels=kwargs.get("dual_didn_hidden_channels", 16), num_dubs=kwargs.get("dual_didn_num_dubs", 6), num_convs_recon=kwargs.get("dual_didn_num_convs_recon", 9), + modulation_params=modulation_params, ) ) for _ in range(num_iter) @@ -112,15 +167,22 @@ def __init__( if image_model_architecture == "MWCNN": image_model_list = nn.ModuleList( [ - nn.Sequential( + XPDNetPrimalBlock( MWCNN( input_channels=2 * (num_primal + num_dual), first_conv_hidden_channels=kwargs.get("mwcnn_hidden_channels", 32), num_scales=kwargs.get("mwcnn_num_scales", 4), bias=kwargs.get("mwcnn_bias", False), batchnorm=kwargs.get("mwcnn_batchnorm", False), + modulation_params=modulation_params, + ), + nn.Conv2d( + 2 * (num_primal + num_dual), + 2 * num_primal, + kernel_size=3, + padding=1, ), - nn.Conv2d(2 * (num_primal + num_dual), 2 * num_primal, kernel_size=3, padding=1), + conv_modulation=conv_modulation, ) for _ in range(num_iter) ] @@ -139,4 +201,5 @@ def __init__( image_buffer_size=num_primal, kspace_buffer_size=num_dual, normalize_image=normalize, + conv_modulation=conv_modulation, ) diff --git a/direct/nn/xpdnet/xpdnet_engine.py b/direct/nn/xpdnet/xpdnet_engine.py index 97a9917a..289efc8a 100644 --- a/direct/nn/xpdnet/xpdnet_engine.py +++ b/direct/nn/xpdnet/xpdnet_engine.py @@ -52,6 +52,7 @@ def forward_function(self, data: Dict[str, Any]) -> Tuple[torch.Tensor, None]: sampling_mask=data["sampling_mask"], sensitivity_map=data["sensitivity_map"], scaling_factor=data["scaling_factor"], + auxiliary_data=self.auxiliary_data_from(data), ) # shape (batch, height, width, complex[=2]) output_kspace = None diff --git a/direct/types.py b/direct/types.py index 304e78df..788e0d7e 100644 --- a/direct/types.py +++ b/direct/types.py @@ -28,6 +28,7 @@ DictOrDictConfig = Union[dict, DictConfig] Number = Union[float, int] +IntOrTuple = Union[int, tuple] PathOrString = Union[pathlib.Path, str] FileOrUrl = PathOrString HasStateDict = Union[nn.Module, torch.optim.Optimizer, torch.optim.lr_scheduler._LRScheduler, GradScaler] @@ -101,6 +102,12 @@ class MaskFuncMode(DirectEnum): MULTISLICE = "multislice" +class RangeMode(DirectEnum): + DISCRETE = "discrete" + UNIFORM = "uniform" + LINEAR = "linear" + + class IntegerListOrTupleStringMeta(type): """Metaclass for the :class:`IntegerListOrTupleString` class. diff --git a/direct/utils/dataset.py b/direct/utils/dataset.py index 1c18fc00..f9353559 100644 --- a/direct/utils/dataset.py +++ b/direct/utils/dataset.py @@ -11,13 +11,76 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import annotations + import pathlib +import re import urllib.parse -from typing import List +from typing import Any, List, Optional, Union + +import numpy as np from direct.types import PathOrString from direct.utils.io import check_is_valid_url, read_list +# Matches tokens such as ``15T``, ``30T``, ``055T``, ``70T`` in filenames +# (e.g. CMRxRecon2025: ``..._UIH_15T_umr670_...``). +_FIELD_STRENGTH_TOKEN = re.compile(r"(?:^|[^0-9A-Za-z])(\d+)T(?:[^0-9A-Za-z]|$)", re.IGNORECASE) + + +def parse_field_strength_tesla(filename: Union[str, pathlib.Path]) -> Optional[float]: + """Parse magnetic field strength in Tesla from an ``XT`` token in ``filename``. + + Digits before ``T`` are interpreted with a decimal point before the last digit when + there is more than one digit: + + * ``7T`` / ``70T`` → 7.0 T + * ``15T`` → 1.5 T + * ``30T`` → 3.0 T + * ``055T`` → 0.55 T + + Parameters + ---------- + filename : str or pathlib.Path + Path or basename that may contain a field-strength token. + + Returns + ------- + float or None + Field strength in Tesla, or ``None`` if no ``XT`` token is present. + """ + name = pathlib.Path(filename).name + matches = _FIELD_STRENGTH_TOKEN.findall(name) + if not matches: + return None + + digits = matches[-1] + if not digits or int(digits) == 0: + return None + + return float(int(digits)) / float(10 ** max(len(digits) - 1, 0)) + + +def maybe_attach_field_strength(sample: dict[str, Any]) -> dict[str, Any]: + """Attach ``field_strength`` to ``sample`` when the filename encodes an ``XT`` token. + + If no token is found, ``sample`` is left unchanged (key is not added). + + Parameters + ---------- + sample : dict[str, Any] + Dataset sample; expects optional ``filename`` key. + + Returns + ------- + dict[str, Any] + The same sample dict, possibly with ``field_strength`` as a length-1 ``np.ndarray``. + """ + value = parse_field_strength_tesla(sample.get("filename", "")) + if value is not None: + sample["field_strength"] = np.array([value], dtype=np.float64) + return sample + def get_filenames_for_datasets_from_config(cfg, files_root: PathOrString, data_root: PathOrString): """Given a configuration object it returns a list of filenames. diff --git a/direct/utils/distributions.py b/direct/utils/distributions.py new file mode 100644 index 00000000..34912df1 --- /dev/null +++ b/direct/utils/distributions.py @@ -0,0 +1,56 @@ +# Copyright 2026 AI for Oncology Research Group. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Probability distributions for sampling.""" + +from __future__ import annotations + +import numpy as np + + +def triangular_distribution( + a: float, + b: float, + n: int, + rng: np.random.RandomState | None = None, +) -> np.ndarray: + """Sample from a triangular distribution on ``[a, b]`` with mode at ``b``. + + The probability density is proportional to ``x`` on ``[a, b]``, so values near + the upper endpoint are sampled more often than values near the lower endpoint. + + Parameters + ---------- + a : float + Left endpoint of the distribution. + b : float + Right endpoint of the distribution. + n : int + Number of samples to draw. + rng : np.random.RandomState, optional + Random number generator. Default: None. + + Returns + ------- + np.ndarray + Array of ``n`` samples. + """ + + def inverse_cdf(u: np.ndarray) -> np.ndarray: + return np.sqrt(u * (b**2 - a**2) + a**2) + + if rng is None: + rng = np.random.RandomState() + + uniform_samples = rng.uniform(0, 1, n) + return inverse_cdf(uniform_samples) diff --git a/projects/modulated_convolution/README.rst b/projects/modulated_convolution/README.rst new file mode 100644 index 00000000..da9de0f4 --- /dev/null +++ b/projects/modulated_convolution/README.rst @@ -0,0 +1,370 @@ +================================================================================= +Modulated Convolution for Conditional MRI Reconstruction +================================================================================= + +This folder contains configuration files and a short tutorial for **modulated +convolutions** in DIRECT, as introduced in: + +`Conditional Learned Reconstruction for Medical Imaging +`__ +(Moriakov et al., MIDL 2026, PMLR 315:754–780). + +`OpenReview submission (PDF) `__ + +Modulated convolutions let a reconstruction network adapt its convolutional +filters to **acquisition metadata** (for example acceleration factor and ACS +fraction) via a small MLP. The same backbone can therefore be trained once and +conditioned at inference time on the actual undersampling pattern. + +Paper overview +============== + +The paper proposes **conditional learned iterative schemes**: convolutional +weights in unrolled reconstruction networks are modulated by learned functions of +acquisition parameters (Section 3, `OpenReview PDF +`__). This addresses variability in +protocol-dependent settings—MRI acceleration and ACS fraction, CT tube current +and projection count—that standard learned iterative models typically do not +model explicitly (Introduction, Section 1). + +.. figure:: figures/modconv_architecture.png + :alt: Modulated convolution schematic (paper Figure 1) + :width: 90% + + **Figure 1** (`paper `__): standard + convolution vs. modulated convolution. An auxiliary vector + :math:`\mathbf{z}` (acquisition characteristics) drives MLPs + :math:`f_\theta, g_\psi` that produce modulating weights and bias; the + modulated kernel is applied to the input feature maps (Section 3.1, Eq. 6). + +.. figure:: figures/modulation_mlp.png + :alt: Modulator MLP architecture (paper Figure 2) + :width: 85% + + **Figure 2** (`paper `__): + modulator architecture. The auxiliary variable + :math:`\mathbf{z} \in \mathbb{R}^N` passes through linear layers with PReLU + activations; a final Softplus yields modulating weights + :math:`\mathbf{W} \in \mathbb{R}^{M}` (Section 3.1). + +For accelerated MRI, the paper uses (Section 4.3.3, Eq. 7): + +.. math:: + + \mathbf{z} = \log([R,\; 100 \cdot r_{\mathrm{acs}}]) \in \mathbb{R}^2 + +where :math:`R` is the acceleration factor and :math:`r_{\mathrm{acs}}` the ACS +fraction. Training samples :math:`R \in [4, 16]` with triangular sampling +toward higher acceleration (Section 4.3.2); equispaced masks are used in the +MRI experiments (Section 4.3.2). Modulated convolutions consistently outperform +non-modulated baselines on fastMRI prostate and knee (Section 4.3.5, Table 1). + +.. figure:: figures/knee_reconstruction.png + :alt: Knee reconstruction example at 16x acceleration (paper Figure 3) + :width: 95% + + **Figure 3** (`paper `__): + qualitative knee example at :math:`R = 16`, :math:`r_{\mathrm{acs}} = 0.02`. + Modulated models recover sharper detail than the non-modulated baseline + (Section 4.3.5). + +DIRECT maps the paper notation to config fields as follows: + ++---------------------------+-----------------------------------------------+ +| Paper | DIRECT config / batch | ++===========================+===============================================+ +| :math:`\mathbf{z}` | ``auxiliary_data`` from ``prepare_auxiliary_ | +| | data()`` | ++---------------------------+-----------------------------------------------+ +| :math:`R` | ``acceleration`` (batch key) | ++---------------------------+-----------------------------------------------+ +| :math:`r_{\mathrm{acs}}` | ``center_fraction`` (batch key) | ++---------------------------+-----------------------------------------------+ +| ``log_aux: true`` | applies :math:`\log` as in Eq. 7 | ++---------------------------+-----------------------------------------------+ +| MOD S / M / L MLP sizes | ``fc_hidden_features: [32, 8]`` etc. | +| (Section 4.1) | | ++---------------------------+-----------------------------------------------+ +| ``FEATURES`` modulation | element-wise weight modulation (Section 3.1, | +| | Appendix B.1) | ++---------------------------+-----------------------------------------------+ + +Overview +======== + +Standard convolutions use fixed weights. A modulated convolution keeps a base +weight tensor and multiplies it element-wise by an MLP output derived from an +auxiliary vector ``y``: + +.. code-block:: text + + x ──► ModConv2d(·, y) ──► output + ▲ + │ + y = [log(acceleration), log(100 * center_fraction), ...] + +In DIRECT this is implemented as a drop-in replacement for ``torch.nn.Conv2d`` / +``Conv3d`` inside U-Nets, VarNets, vSHARP denoisers, and other unrolled models. + +Code layout +=========== + +All modulated-convolution code lives under ``direct/nn/conv/modulated/``: + +``modulated_conv.py`` + Core layers: ``ModConv2d``, ``ModConv3d``, transposed variants, and enums + ``ModConvType``, ``ModConvActivation``, ``ModConv2dBias``. + +``auxiliary_data.py`` + Registry-based auxiliary feature pipeline: + + * ``prepare_auxiliary_data(data, cfg)`` — builds ``(batch, aux_in_features)`` + from the batch dict. + * ``register_auxiliary_feature()`` — add custom conditioning channels. + * Default features (in order): ``acceleration``, ``center_fraction``, + ``field_strength`` (when present in the batch; typically parsed from an + ``XT`` token in the filename, e.g. CMRxRecon2025 ``15T`` / ``30T``). + +``__init__.py`` + Public re-exports used throughout the codebase. + +Related integration points: + +* **UNet backbone** — ``direct/nn/unet/unet_2d.py`` swaps ``Conv2d`` blocks for + ``ModConv2d`` when ``conv_modulation != NONE``. +* **vSHARP** — ``direct/nn/vsharp/vsharp.py`` passes ``auxiliary_data`` to the + initializer and image denoiser U-Net (Section 3.3). +* **VarNet** — ``direct/nn/varnet/varnet.py`` modulates the regularizer U-Net. +* **Other unrolled models** — KIKINet, JointICNet, IterDualNet, LPD, Conv2d, + DIDN, MWCNN (see their ``config.py`` files for ``conv_modulation``). +* **Engines** — ``MRIModelEngine._attach_auxiliary_data()`` runs in every supervised, + SSL, and JSSL iteration. Conv-based engines pass ``auxiliary_data`` into their model + ``forward``. +* **Data pipeline** — ``direct/data/mri_transforms.py`` enables + ``return_acceleration`` on mask functions; sampled values land in the batch as + ``acceleration`` and ``center_fraction``. +* **Triangular acceleration sampling** — set ``range_mode: LINEAR`` in masking + config to match Section 4.3.2 (see ``direct/common/subsample.py``). + +Modulation types +================ + +Set ``conv_modulation`` on model configs to one of: + +``NONE`` + Standard convolution (default). Auxiliary inputs are ignored. + +``FEATURES`` + MLP output has the same shape as the convolution weight tensor; element-wise + product with the base weights. Most experiments in this folder use this mode + (Section 3.1; see also Appendix B.1 for other variants). + +``FULL`` + MLP output modulates the full weight tensor (one scalar factor per weight). + +``PARTIAL_IN`` / ``PARTIAL_OUT`` + Modulate along input or output channel dimension respectively. + +``SUM`` + Learn ``num_weights`` weight bases; MLP produces mixture coefficients. + +Key config fields +================= + +Model section (example from vSHARP): + +.. code-block:: yaml + + conv_modulation: FEATURES # ModConvType + aux_in_features: 2 # length of y (paper Eq. 7: R + r_acs) + auxiliary_features: # optional; default: first N registry keys + - acceleration + - center_fraction + log_aux: true # apply log() as in Eq. 7 + fc_hidden_features: [32, 8] # MLP hidden layers (MOD S in Section 4.1) + fc_activation: SOFTPLUS # SIGMOID or SOFTPLUS (paper: Softplus output) + fc_groups: 1 # optional grouped low-rank modulation + num_weights: null # only for SUM modulation + +Masking section (training with variable acceleration): + +.. code-block:: yaml + + masking: + name: FastMRIEquispaced + accelerations: [4, 16] + center_fractions: [0.08, 0.02] + range_mode: LINEAR # triangular sampling toward higher R (Sec. 4.3.2) + +When ``log_aux: true``, ``center_fraction`` is multiplied by 100 before logging +(see ``AuxiliaryFeature.log_scale`` in the registry), matching Eq. 7. +``acceleration`` uses scale 1. + +End-to-end data flow +==================== + +1. **Dataset transform** — ``CreateSamplingMask`` calls the mask function with + ``return_acceleration=True`` and stores ``acceleration`` / ``center_fraction`` + in the sample dict. +2. **Collate / device** — tensors move to the training device with the batch. +3. **Engine** — ``MRIModelEngine._attach_auxiliary_data()`` calls + ``prepare_auxiliary_data()`` when modulation is enabled. +4. **Model forward** — each ``ModConv2d`` layer receives ``(x, auxiliary_data)``. + +To add a custom auxiliary channel: + +.. code-block:: python + + from direct.nn.conv.modulated import AuxiliaryFeature, register_auxiliary_feature + + register_auxiliary_feature(AuxiliaryFeature("my_feature")) + # then set auxiliary_features: [acceleration, my_feature] in the yaml + +Configuration files +=================== + +Paper experiment configs are in ``projects/modulated_convolution/configs/vsharp/``: + +**fastMRI knee** (``configs/vsharp/knee/``, 80k iterations unless noted): + ++-------------------------------+-----------------------------------------------+ +| Config file | Description | ++===============================+===============================================+ +| ``vsharp_triang.yaml`` | Baseline vSHARP, no modulation | ++-------------------------------+-----------------------------------------------+ +| ``vsharp_modconv_features_ | FEATURES modulation, MLP ``[32, 32]`` (MOD L) | +| triang.yaml`` | | ++-------------------------------+-----------------------------------------------+ +| ``vsharp_modconv_features_ | FEATURES modulation, MLP ``[32, 8]`` (MOD S) | +| triang_32_8.yaml`` | | ++-------------------------------+-----------------------------------------------+ +| ``vsharp_modconv_features_ | FEATURES modulation, MLP ``[32, 16]`` (MOD M) | +| triang_32_16.yaml`` | | ++-------------------------------+-----------------------------------------------+ +| ``vsharp_modconv_features_ | MOD M with ``modulation_at_input: true``, | +| triang_32_16_mod_inp.yaml`` | 150k iterations | ++-------------------------------+-----------------------------------------------+ +| ``vsharp_modconv_partial_in_ | PARTIAL_IN modulation, MLP ``[32, 32]`` | +| triang.yaml`` | | ++-------------------------------+-----------------------------------------------+ +| ``vsharp_adain_triang_32_16. | AdaIN baseline, MLP ``[32, 16]``, 150k iters | +| yaml`` | | ++-------------------------------+-----------------------------------------------+ + +**fastMRI prostate** (``configs/vsharp/prostate/``, 150k iterations): + +Same variants as knee except AdaIN and modulation-at-input configs (knee-only ablations). + +Other example configs: + +``varnet_prostate_modconv_accel_16_16.yaml`` + End-to-end VarNet on prostate. Acceleration-only conditioning + (``aux_in_features: 1``), MLP ``[16, 16]``. + +Training +======== + +Create an experiment directory, copy or symlink a config as ``config.yaml`` inside it +(or pass ``--cfg`` explicitly). Then: + +.. code-block:: bash + + direct train /path/to/experiments/my_run \ + --cfg projects/modulated_convolution/configs/vsharp/knee/vsharp_modconv_features_triang_32_8.yaml \ + --training-root /path/to/fastmri/knee/ \ + --validation-root /path/to/fastmri/knee/val/ \ + --device mps + +Prostate (training and validation roots are the same directory): + +.. code-block:: bash + + direct train /path/to/experiments/my_run \ + --cfg projects/modulated_convolution/configs/vsharp/prostate/vsharp_modconv_features_triang_32_16.yaml \ + --training-root /path/to/fastmri/prostate/ \ + --validation-root /path/to/fastmri/prostate/ \ + --device mps + +Working knee example (matches the Snellius vSHARP mod-conv checkpoint layout): + +.. code-block:: bash + + direct train ./experiments/base_vsharp_modconv_softplus_features_double_MLP_triang_32_8 \ + --cfg projects/modulated_convolution/configs/vsharp/knee/vsharp_modconv_features_triang_32_8.yaml \ + --training-root /path/to/fastmri/knee/ \ + --validation-root /path/to/fastmri/knee/val/ \ + --device mps + +Resume from a checkpoint: + +.. code-block:: bash + + direct train ./experiments/base_vsharp_modconv_softplus_features_double_MLP_triang_32_8 \ + --cfg ./experiments/base_vsharp_modconv_softplus_features_double_MLP_triang_32_8/config.yaml \ + --training-root /path/to/fastmri/knee/ \ + --validation-root /path/to/fastmri/knee/val/ \ + --device mps \ + --resume + +Inference +========= + +.. code-block:: bash + + direct predict ./output \ + --cfg projects/modulated_convolution/configs/vsharp/knee/vsharp_modconv_features_triang_32_8.yaml \ + --checkpoint /path/to/model_80000.pt \ + --data-root /path/to/fastmri/knee/val \ + --device cuda:0 + +The ``inference`` block in each yaml fixes the validation acceleration (default +4×). Edit ``inference.dataset.transforms.masking`` to evaluate other rates. + +Tests +===== + +Modulated convolution unit tests: + +.. code-block:: bash + + pytest tests/tests_nn/modulated_conv_test.py tests/tests_nn/auxiliary_data_test.py + +Subsampling / triangular range tests: + +.. code-block:: bash + + pytest tests/tests_common/subsample_test.py -k "linear_range or equispaced" + +Citing this work +================ + +.. code-block:: BibTeX + + @inproceedings{moriakov2026conditional, + title = {Conditional Learned Reconstruction for Medical Imaging}, + author = {Moriakov, Nikita and Yiasemis, George and Sonke, Jan-Jakob and Teuwen, Jonas}, + booktitle = {Proceedings of The 9th International Conference on Medical Imaging with Deep Learning}, + pages = {754--780}, + year = {2026}, + volume = {315}, + series = {Proceedings of Machine Learning Research}, + publisher = {PMLR}, + url = {https://proceedings.mlr.press/v315/moriakov26a.html} + } + + @article{DIRECTTOOLKIT, + doi = {10.21105/joss.04278}, + url = {https://doi.org/10.21105/joss.04278}, + year = {2022}, + publisher = {The Open Journal}, + volume = {7}, + number = {73}, + pages = {4278}, + author = {George Yiasemis and Nikita Moriakov and Dimitrios Karkalousos and Matthan Caan and Jonas Teuwen}, + title = {DIRECT: Deep Image REConstruction Toolkit}, + journal = {Journal of Open Source Software} + } + +Figures in ``figures/`` are adapted from the paper (`OpenReview PDF +`__); source diagrams: +``modconv.pdf`` and ``modulation.pdf``. diff --git a/projects/modulated_convolution/configs/varnet_prostate_modconv_accel_16_16.yaml b/projects/modulated_convolution/configs/varnet_prostate_modconv_accel_16_16.yaml new file mode 100644 index 00000000..84f012dd --- /dev/null +++ b/projects/modulated_convolution/configs/varnet_prostate_modconv_accel_16_16.yaml @@ -0,0 +1,164 @@ +model: + model_name: varnet.varnet.EndToEndVarNet + engine_name: null + num_layers: 12 + regularizer_num_filters: 64 + regularizer_num_pull_layers: 4 + regularizer_dropout: 0.0 + conv_modulation: FEATURES + aux_in_features: 1 + auxiliary_features: + - acceleration + log_aux: true + fc_hidden_features: + - 16 + - 16 + fc_groups: 1 + fc_activation: SOFTPLUS + num_weights: null +additional_models: + sensitivity_model: + model_name: unet.unet_2d.UnetModel2d + engine_name: null + in_channels: 2 + out_channels: 2 + num_filters: 16 + num_pool_layers: 4 + dropout_probability: 0.0 + modulation: NONE + aux_in_features: null + fc_hidden_features: null + fc_groups: 1 + fc_activation: SIGMOID + num_weights: null +physics: + forward_operator: fft2 + backward_operator: ifft2 + use_noise_matrix: false + noise_matrix_scaling: 1.0 +training: + datasets: + - name: FastMRI + transforms: + crop: reconstruction_size + estimate_sensitivity_maps: true + scaling_key: masked_kspace + image_center_crop: false + masking: + name: FastMRIEquispaced + accelerations: + - 4 + - 16 + center_fractions: + - 0.08 + - 0.02 + scale_percentile: 0.995 + use_seed: false + delete_kspace: false + model_checkpoint: null + optimizer: Adam + lr: 0.002 + weight_decay: 0.0 + batch_size: 1 + lr_step_size: 30000 + lr_gamma: 0.8 + lr_warmup_iter: 1000 + swa_start_iter: null + num_iterations: 150000 + validation_steps: 4000 + gradient_steps: 1 + gradient_clipping: 0.0 + gradient_debug: false + loss: + crop: header + losses: + - function: l1_loss + multiplier: 1.0 + - function: ssim_loss + multiplier: 1.0 + - function: hfen_l2_norm_loss + multiplier: 1.0 + - function: hfen_l1_norm_loss + multiplier: 1.0 + - function: kspace_nmae_loss + multiplier: 1.0 + - function: kspace_nmse_loss + multiplier: 1.0 + checkpointer: + checkpoint_steps: 4000 + metrics: [] + regularizers: [] +validation: + datasets: + - name: FastMRI + transforms: + estimate_sensitivity_maps: true + scaling_key: masked_kspace + masking: + name: FastMRIEquispaced + accelerations: + - 4 + center_fractions: + - 0.08 + scale_percentile: 0.995 + use_seed: true + text_description: 4x + - name: FastMRI + transforms: + estimate_sensitivity_maps: true + scaling_key: masked_kspace + masking: + name: FastMRIEquispaced + accelerations: + - 8 + center_fractions: + - 0.04 + scale_percentile: 0.995 + use_seed: true + text_description: 8x + - name: FastMRI + transforms: + estimate_sensitivity_maps: true + scaling_key: masked_kspace + masking: + name: FastMRIEquispaced + accelerations: + - 16 + center_fractions: + - 0.02 + scale_percentile: 0.995 + use_seed: true + text_description: 16x + batch_size: 20 + metrics: + - fastmri_psnr + - fastmri_ssim + - fastmri_nmse + regularizers: [] + crop: header +inference: + dataset: + name: FastMRI + transforms: + masking: + name: FastMRIEquispaced + accelerations: + - 4.0 + center_fractions: + - 0.08 + mode: STATIC + cropping: + crop: null + sensitivity_map_estimation: + estimate_sensitivity_maps: true + normalization: + scaling_key: masked_kspace + scale_percentile: 0.995 + use_seed: true + text_description: inference-4x + batch_size: 5 + crop: header +logging: + log_as_image: null + tensorboard: + num_images: 4 diff --git a/projects/modulated_convolution/configs/vsharp/knee/vsharp_adain_triang_32_16.yaml b/projects/modulated_convolution/configs/vsharp/knee/vsharp_adain_triang_32_16.yaml new file mode 100644 index 00000000..ccf182d0 --- /dev/null +++ b/projects/modulated_convolution/configs/vsharp/knee/vsharp_adain_triang_32_16.yaml @@ -0,0 +1,196 @@ +model: + model_name: vsharp.vsharp.VSharpNet + num_steps: 12 + num_steps_dc_gd: 10 + image_init: SENSE + no_parameter_sharing: true + auxiliary_steps: -1 + image_model_architecture: UNET + initializer_channels: + - 32 + - 32 + - 64 + - 64 + initializer_dilations: + - 1 + - 1 + - 2 + - 4 + initializer_multiscale: 1 + initializer_activation: PRELU + conv_modulation: NONE + aux_in_features: 2 + log_aux: true + fc_hidden_features: + - 32 + - 32 + fc_groups: 1 + fc_activation: SOFTPLUS + num_weights: null + modulation_at_input: false + image_resnet_hidden_channels: 128 + image_resnet_num_blocks: 15 + image_resnet_batchnorm: true + image_resnet_scale: 0.1 + image_unet_num_filters: 32 + image_unet_num_pool_layers: 4 + image_unet_dropout: 0.0 + image_unet_norm_type: ADAIN + image_unet_adain_hidden_features: + - 32 + - 16 + image_didn_hidden_channels: 16 + image_didn_num_dubs: 6 + image_didn_num_convs_recon: 9 + image_conv_hidden_channels: 64 + image_conv_n_convs: 15 + image_conv_activation: ActivationType.RELU + image_conv_batchnorm: false +additional_models: + sensitivity_model: + model_name: unet.unet_2d.UnetModel2d + in_channels: 2 + out_channels: 2 + num_filters: 16 + num_pool_layers: 4 + dropout_probability: 0.0 + modulation: NONE + aux_in_features: null + fc_hidden_features: null + fc_groups: 1 + fc_activation: SIGMOID + num_weights: null +physics: + forward_operator: fft2 + backward_operator: ifft2 + use_noise_matrix: false + noise_matrix_scaling: 1.0 +training: + datasets: + - name: FastMRI + transforms: + crop: reconstruction_size + estimate_sensitivity_maps: true + scaling_key: masked_kspace + image_center_crop: false + masking: + name: FastMRIEquispaced + accelerations: + - 4 + - 16 + center_fractions: + - 0.08 + - 0.02 + range_mode: LINEAR + scale_percentile: 0.995 + use_seed: false + delete_kspace: false + random_flip_probability: 0.5 + random_rotation_probability: 0.5 + model_checkpoint: null + optimizer: Adam + lr: 0.002 + weight_decay: 0.0 + batch_size: 1 + lr_step_size: 15000 + lr_gamma: 0.8 + lr_warmup_iter: 1000 + swa_start_iter: null + num_iterations: 150001 + validation_steps: 4000 + gradient_steps: 1 + gradient_clipping: 0.0 + gradient_debug: false + loss: + crop: header + losses: + - function: l1_loss + multiplier: 1.0 + - function: ssim_loss + multiplier: 1.0 + - function: hfen_l2_norm_loss + multiplier: 1.0 + - function: hfen_l1_norm_loss + multiplier: 1.0 + - function: kspace_nmae_loss + multiplier: 1.0 + - function: kspace_nmse_loss + multiplier: 1.0 + checkpointer: + checkpoint_steps: 4000 + metrics: [] + regularizers: [] +validation: + datasets: + - name: FastMRI + transforms: + estimate_sensitivity_maps: true + scaling_key: masked_kspace + masking: + name: FastMRIEquispaced + accelerations: + - 4 + center_fractions: + - 0.08 + scale_percentile: 0.995 + use_seed: true + text_description: 4x + - name: FastMRI + transforms: + estimate_sensitivity_maps: true + scaling_key: masked_kspace + masking: + name: FastMRIEquispaced + accelerations: + - 8 + center_fractions: + - 0.04 + scale_percentile: 0.995 + use_seed: true + text_description: 8x + - name: FastMRI + transforms: + estimate_sensitivity_maps: true + scaling_key: masked_kspace + masking: + name: FastMRIEquispaced + accelerations: + - 16 + center_fractions: + - 0.02 + scale_percentile: 0.995 + use_seed: true + text_description: 16x + batch_size: 20 + metrics: + - fastmri_psnr + - fastmri_ssim + - fastmri_nmse + regularizers: [] + crop: header +inference: + dataset: + name: FastMRI + transforms: + masking: + name: FastMRIEquispaced + accelerations: + - 4.0 + center_fractions: + - 0.08 + mode: STATIC + cropping: + crop: null + sensitivity_map_estimation: + estimate_sensitivity_maps: true + normalization: + scaling_key: masked_kspace + scale_percentile: 0.995 + use_seed: true + text_description: inference-4x + batch_size: 1 + crop: header +logging: + log_as_image: null + tensorboard: + num_images: 4 diff --git a/projects/modulated_convolution/configs/vsharp/knee/vsharp_modconv_features_triang.yaml b/projects/modulated_convolution/configs/vsharp/knee/vsharp_modconv_features_triang.yaml new file mode 100644 index 00000000..c00e01e5 --- /dev/null +++ b/projects/modulated_convolution/configs/vsharp/knee/vsharp_modconv_features_triang.yaml @@ -0,0 +1,194 @@ +model: + model_name: vsharp.vsharp.VSharpNet + num_steps: 12 + num_steps_dc_gd: 10 + image_init: SENSE + no_parameter_sharing: true + auxiliary_steps: -1 + image_model_architecture: UNET + initializer_channels: + - 32 + - 32 + - 64 + - 64 + initializer_dilations: + - 1 + - 1 + - 2 + - 4 + initializer_multiscale: 1 + initializer_activation: PRELU + conv_modulation: FEATURES + aux_in_features: 2 + log_aux: true + fc_hidden_features: + - 32 + - 32 + fc_groups: 1 + fc_activation: SOFTPLUS + num_weights: null + modulation_at_input: false + image_resnet_hidden_channels: 128 + image_resnet_num_blocks: 15 + image_resnet_batchnorm: true + image_resnet_scale: 0.1 + image_unet_num_filters: 32 + image_unet_num_pool_layers: 4 + image_unet_dropout: 0.0 + image_unet_norm_type: INSTANCE + image_unet_adain_hidden_features: null + image_didn_hidden_channels: 16 + image_didn_num_dubs: 6 + image_didn_num_convs_recon: 9 + image_conv_hidden_channels: 64 + image_conv_n_convs: 15 + image_conv_activation: ActivationType.RELU + image_conv_batchnorm: false +additional_models: + sensitivity_model: + model_name: unet.unet_2d.UnetModel2d + in_channels: 2 + out_channels: 2 + num_filters: 16 + num_pool_layers: 4 + dropout_probability: 0.0 + modulation: NONE + aux_in_features: null + fc_hidden_features: null + fc_groups: 1 + fc_activation: SIGMOID + num_weights: null +physics: + forward_operator: fft2 + backward_operator: ifft2 + use_noise_matrix: false + noise_matrix_scaling: 1.0 +training: + datasets: + - name: FastMRI + transforms: + crop: reconstruction_size + estimate_sensitivity_maps: true + scaling_key: masked_kspace + image_center_crop: false + masking: + name: FastMRIEquispaced + accelerations: + - 4 + - 16 + center_fractions: + - 0.08 + - 0.02 + range_mode: LINEAR + scale_percentile: 0.995 + use_seed: false + delete_kspace: false + random_flip_probability: 0.5 + random_rotation_probability: 0.5 + model_checkpoint: null + optimizer: Adam + lr: 0.002 + weight_decay: 0.0 + batch_size: 1 + lr_step_size: 15000 + lr_gamma: 0.8 + lr_warmup_iter: 1000 + swa_start_iter: null + num_iterations: 80001 + validation_steps: 4000 + gradient_steps: 1 + gradient_clipping: 0.0 + gradient_debug: false + loss: + crop: header + losses: + - function: l1_loss + multiplier: 1.0 + - function: ssim_loss + multiplier: 1.0 + - function: hfen_l2_norm_loss + multiplier: 1.0 + - function: hfen_l1_norm_loss + multiplier: 1.0 + - function: kspace_nmae_loss + multiplier: 1.0 + - function: kspace_nmse_loss + multiplier: 1.0 + checkpointer: + checkpoint_steps: 4000 + metrics: [] + regularizers: [] +validation: + datasets: + - name: FastMRI + transforms: + estimate_sensitivity_maps: true + scaling_key: masked_kspace + masking: + name: FastMRIEquispaced + accelerations: + - 4 + center_fractions: + - 0.08 + scale_percentile: 0.995 + use_seed: true + text_description: 4x + - name: FastMRI + transforms: + estimate_sensitivity_maps: true + scaling_key: masked_kspace + masking: + name: FastMRIEquispaced + accelerations: + - 8 + center_fractions: + - 0.04 + scale_percentile: 0.995 + use_seed: true + text_description: 8x + - name: FastMRI + transforms: + estimate_sensitivity_maps: true + scaling_key: masked_kspace + masking: + name: FastMRIEquispaced + accelerations: + - 16 + center_fractions: + - 0.02 + scale_percentile: 0.995 + use_seed: true + text_description: 16x + batch_size: 20 + metrics: + - fastmri_psnr + - fastmri_ssim + - fastmri_nmse + regularizers: [] + crop: header +inference: + dataset: + name: FastMRI + transforms: + masking: + name: FastMRIEquispaced + accelerations: + - 4.0 + center_fractions: + - 0.08 + mode: STATIC + cropping: + crop: null + sensitivity_map_estimation: + estimate_sensitivity_maps: true + normalization: + scaling_key: masked_kspace + scale_percentile: 0.995 + use_seed: true + text_description: inference-4x + batch_size: 1 + crop: header +logging: + log_as_image: null + tensorboard: + num_images: 4 diff --git a/projects/modulated_convolution/configs/vsharp/knee/vsharp_modconv_features_triang_32_16.yaml b/projects/modulated_convolution/configs/vsharp/knee/vsharp_modconv_features_triang_32_16.yaml new file mode 100644 index 00000000..0bba5f89 --- /dev/null +++ b/projects/modulated_convolution/configs/vsharp/knee/vsharp_modconv_features_triang_32_16.yaml @@ -0,0 +1,194 @@ +model: + model_name: vsharp.vsharp.VSharpNet + num_steps: 12 + num_steps_dc_gd: 10 + image_init: SENSE + no_parameter_sharing: true + auxiliary_steps: -1 + image_model_architecture: UNET + initializer_channels: + - 32 + - 32 + - 64 + - 64 + initializer_dilations: + - 1 + - 1 + - 2 + - 4 + initializer_multiscale: 1 + initializer_activation: PRELU + conv_modulation: FEATURES + aux_in_features: 2 + log_aux: true + fc_hidden_features: + - 32 + - 16 + fc_groups: 1 + fc_activation: SOFTPLUS + num_weights: null + modulation_at_input: false + image_resnet_hidden_channels: 128 + image_resnet_num_blocks: 15 + image_resnet_batchnorm: true + image_resnet_scale: 0.1 + image_unet_num_filters: 32 + image_unet_num_pool_layers: 4 + image_unet_dropout: 0.0 + image_unet_norm_type: INSTANCE + image_unet_adain_hidden_features: null + image_didn_hidden_channels: 16 + image_didn_num_dubs: 6 + image_didn_num_convs_recon: 9 + image_conv_hidden_channels: 64 + image_conv_n_convs: 15 + image_conv_activation: ActivationType.RELU + image_conv_batchnorm: false +additional_models: + sensitivity_model: + model_name: unet.unet_2d.UnetModel2d + in_channels: 2 + out_channels: 2 + num_filters: 16 + num_pool_layers: 4 + dropout_probability: 0.0 + modulation: NONE + aux_in_features: null + fc_hidden_features: null + fc_groups: 1 + fc_activation: SIGMOID + num_weights: null +physics: + forward_operator: fft2 + backward_operator: ifft2 + use_noise_matrix: false + noise_matrix_scaling: 1.0 +training: + datasets: + - name: FastMRI + transforms: + crop: reconstruction_size + estimate_sensitivity_maps: true + scaling_key: masked_kspace + image_center_crop: false + masking: + name: FastMRIEquispaced + accelerations: + - 4 + - 16 + center_fractions: + - 0.08 + - 0.02 + range_mode: LINEAR + scale_percentile: 0.995 + use_seed: false + delete_kspace: false + random_flip_probability: 0.5 + random_rotation_probability: 0.5 + model_checkpoint: null + optimizer: Adam + lr: 0.002 + weight_decay: 0.0 + batch_size: 1 + lr_step_size: 15000 + lr_gamma: 0.8 + lr_warmup_iter: 1000 + swa_start_iter: null + num_iterations: 80001 + validation_steps: 4000 + gradient_steps: 1 + gradient_clipping: 0.0 + gradient_debug: false + loss: + crop: header + losses: + - function: l1_loss + multiplier: 1.0 + - function: ssim_loss + multiplier: 1.0 + - function: hfen_l2_norm_loss + multiplier: 1.0 + - function: hfen_l1_norm_loss + multiplier: 1.0 + - function: kspace_nmae_loss + multiplier: 1.0 + - function: kspace_nmse_loss + multiplier: 1.0 + checkpointer: + checkpoint_steps: 4000 + metrics: [] + regularizers: [] +validation: + datasets: + - name: FastMRI + transforms: + estimate_sensitivity_maps: true + scaling_key: masked_kspace + masking: + name: FastMRIEquispaced + accelerations: + - 4 + center_fractions: + - 0.08 + scale_percentile: 0.995 + use_seed: true + text_description: 4x + - name: FastMRI + transforms: + estimate_sensitivity_maps: true + scaling_key: masked_kspace + masking: + name: FastMRIEquispaced + accelerations: + - 8 + center_fractions: + - 0.04 + scale_percentile: 0.995 + use_seed: true + text_description: 8x + - name: FastMRI + transforms: + estimate_sensitivity_maps: true + scaling_key: masked_kspace + masking: + name: FastMRIEquispaced + accelerations: + - 16 + center_fractions: + - 0.02 + scale_percentile: 0.995 + use_seed: true + text_description: 16x + batch_size: 20 + metrics: + - fastmri_psnr + - fastmri_ssim + - fastmri_nmse + regularizers: [] + crop: header +inference: + dataset: + name: FastMRI + transforms: + masking: + name: FastMRIEquispaced + accelerations: + - 4.0 + center_fractions: + - 0.08 + mode: STATIC + cropping: + crop: null + sensitivity_map_estimation: + estimate_sensitivity_maps: true + normalization: + scaling_key: masked_kspace + scale_percentile: 0.995 + use_seed: true + text_description: inference-4x + batch_size: 1 + crop: header +logging: + log_as_image: null + tensorboard: + num_images: 4 diff --git a/projects/modulated_convolution/configs/vsharp/knee/vsharp_modconv_features_triang_32_16_mod_inp.yaml b/projects/modulated_convolution/configs/vsharp/knee/vsharp_modconv_features_triang_32_16_mod_inp.yaml new file mode 100644 index 00000000..626d6229 --- /dev/null +++ b/projects/modulated_convolution/configs/vsharp/knee/vsharp_modconv_features_triang_32_16_mod_inp.yaml @@ -0,0 +1,194 @@ +model: + model_name: vsharp.vsharp.VSharpNet + num_steps: 12 + num_steps_dc_gd: 10 + image_init: SENSE + no_parameter_sharing: true + auxiliary_steps: -1 + image_model_architecture: UNET + initializer_channels: + - 32 + - 32 + - 64 + - 64 + initializer_dilations: + - 1 + - 1 + - 2 + - 4 + initializer_multiscale: 1 + initializer_activation: PRELU + conv_modulation: FEATURES + aux_in_features: 2 + log_aux: true + fc_hidden_features: + - 32 + - 16 + fc_groups: 1 + fc_activation: SOFTPLUS + num_weights: null + modulation_at_input: true + image_resnet_hidden_channels: 128 + image_resnet_num_blocks: 15 + image_resnet_batchnorm: true + image_resnet_scale: 0.1 + image_unet_num_filters: 32 + image_unet_num_pool_layers: 4 + image_unet_dropout: 0.0 + image_unet_norm_type: INSTANCE + image_unet_adain_hidden_features: null + image_didn_hidden_channels: 16 + image_didn_num_dubs: 6 + image_didn_num_convs_recon: 9 + image_conv_hidden_channels: 64 + image_conv_n_convs: 15 + image_conv_activation: ActivationType.RELU + image_conv_batchnorm: false +additional_models: + sensitivity_model: + model_name: unet.unet_2d.UnetModel2d + in_channels: 2 + out_channels: 2 + num_filters: 16 + num_pool_layers: 4 + dropout_probability: 0.0 + modulation: NONE + aux_in_features: null + fc_hidden_features: null + fc_groups: 1 + fc_activation: SIGMOID + num_weights: null +physics: + forward_operator: fft2 + backward_operator: ifft2 + use_noise_matrix: false + noise_matrix_scaling: 1.0 +training: + datasets: + - name: FastMRI + transforms: + crop: reconstruction_size + estimate_sensitivity_maps: true + scaling_key: masked_kspace + image_center_crop: false + masking: + name: FastMRIEquispaced + accelerations: + - 4 + - 16 + center_fractions: + - 0.08 + - 0.02 + range_mode: LINEAR + scale_percentile: 0.995 + use_seed: false + delete_kspace: false + random_flip_probability: 0.5 + random_rotation_probability: 0.5 + model_checkpoint: null + optimizer: Adam + lr: 0.002 + weight_decay: 0.0 + batch_size: 1 + lr_step_size: 15000 + lr_gamma: 0.8 + lr_warmup_iter: 1000 + swa_start_iter: null + num_iterations: 150001 + validation_steps: 4000 + gradient_steps: 1 + gradient_clipping: 0.0 + gradient_debug: false + loss: + crop: header + losses: + - function: l1_loss + multiplier: 1.0 + - function: ssim_loss + multiplier: 1.0 + - function: hfen_l2_norm_loss + multiplier: 1.0 + - function: hfen_l1_norm_loss + multiplier: 1.0 + - function: kspace_nmae_loss + multiplier: 1.0 + - function: kspace_nmse_loss + multiplier: 1.0 + checkpointer: + checkpoint_steps: 4000 + metrics: [] + regularizers: [] +validation: + datasets: + - name: FastMRI + transforms: + estimate_sensitivity_maps: true + scaling_key: masked_kspace + masking: + name: FastMRIEquispaced + accelerations: + - 4 + center_fractions: + - 0.08 + scale_percentile: 0.995 + use_seed: true + text_description: 4x + - name: FastMRI + transforms: + estimate_sensitivity_maps: true + scaling_key: masked_kspace + masking: + name: FastMRIEquispaced + accelerations: + - 8 + center_fractions: + - 0.04 + scale_percentile: 0.995 + use_seed: true + text_description: 8x + - name: FastMRI + transforms: + estimate_sensitivity_maps: true + scaling_key: masked_kspace + masking: + name: FastMRIEquispaced + accelerations: + - 16 + center_fractions: + - 0.02 + scale_percentile: 0.995 + use_seed: true + text_description: 16x + batch_size: 20 + metrics: + - fastmri_psnr + - fastmri_ssim + - fastmri_nmse + regularizers: [] + crop: header +inference: + dataset: + name: FastMRI + transforms: + masking: + name: FastMRIEquispaced + accelerations: + - 4.0 + center_fractions: + - 0.08 + mode: STATIC + cropping: + crop: null + sensitivity_map_estimation: + estimate_sensitivity_maps: true + normalization: + scaling_key: masked_kspace + scale_percentile: 0.995 + use_seed: true + text_description: inference-4x + batch_size: 1 + crop: header +logging: + log_as_image: null + tensorboard: + num_images: 4 diff --git a/projects/modulated_convolution/configs/vsharp/knee/vsharp_modconv_features_triang_32_8.yaml b/projects/modulated_convolution/configs/vsharp/knee/vsharp_modconv_features_triang_32_8.yaml new file mode 100644 index 00000000..b599ca9a --- /dev/null +++ b/projects/modulated_convolution/configs/vsharp/knee/vsharp_modconv_features_triang_32_8.yaml @@ -0,0 +1,194 @@ +model: + model_name: vsharp.vsharp.VSharpNet + num_steps: 12 + num_steps_dc_gd: 10 + image_init: SENSE + no_parameter_sharing: true + auxiliary_steps: -1 + image_model_architecture: UNET + initializer_channels: + - 32 + - 32 + - 64 + - 64 + initializer_dilations: + - 1 + - 1 + - 2 + - 4 + initializer_multiscale: 1 + initializer_activation: PRELU + conv_modulation: FEATURES + aux_in_features: 2 + log_aux: true + fc_hidden_features: + - 32 + - 8 + fc_groups: 1 + fc_activation: SOFTPLUS + num_weights: null + modulation_at_input: false + image_resnet_hidden_channels: 128 + image_resnet_num_blocks: 15 + image_resnet_batchnorm: true + image_resnet_scale: 0.1 + image_unet_num_filters: 32 + image_unet_num_pool_layers: 4 + image_unet_dropout: 0.0 + image_unet_norm_type: INSTANCE + image_unet_adain_hidden_features: null + image_didn_hidden_channels: 16 + image_didn_num_dubs: 6 + image_didn_num_convs_recon: 9 + image_conv_hidden_channels: 64 + image_conv_n_convs: 15 + image_conv_activation: ActivationType.RELU + image_conv_batchnorm: false +additional_models: + sensitivity_model: + model_name: unet.unet_2d.UnetModel2d + in_channels: 2 + out_channels: 2 + num_filters: 16 + num_pool_layers: 4 + dropout_probability: 0.0 + modulation: NONE + aux_in_features: null + fc_hidden_features: null + fc_groups: 1 + fc_activation: SIGMOID + num_weights: null +physics: + forward_operator: fft2 + backward_operator: ifft2 + use_noise_matrix: false + noise_matrix_scaling: 1.0 +training: + datasets: + - name: FastMRI + transforms: + crop: reconstruction_size + estimate_sensitivity_maps: true + scaling_key: masked_kspace + image_center_crop: false + masking: + name: FastMRIEquispaced + accelerations: + - 4 + - 16 + center_fractions: + - 0.08 + - 0.02 + range_mode: LINEAR + scale_percentile: 0.995 + use_seed: false + delete_kspace: false + random_flip_probability: 0.5 + random_rotation_probability: 0.5 + model_checkpoint: null + optimizer: Adam + lr: 0.002 + weight_decay: 0.0 + batch_size: 1 + lr_step_size: 15000 + lr_gamma: 0.8 + lr_warmup_iter: 1000 + swa_start_iter: null + num_iterations: 80001 + validation_steps: 4000 + gradient_steps: 1 + gradient_clipping: 0.0 + gradient_debug: false + loss: + crop: header + losses: + - function: l1_loss + multiplier: 1.0 + - function: ssim_loss + multiplier: 1.0 + - function: hfen_l2_norm_loss + multiplier: 1.0 + - function: hfen_l1_norm_loss + multiplier: 1.0 + - function: kspace_nmae_loss + multiplier: 1.0 + - function: kspace_nmse_loss + multiplier: 1.0 + checkpointer: + checkpoint_steps: 4000 + metrics: [] + regularizers: [] +validation: + datasets: + - name: FastMRI + transforms: + estimate_sensitivity_maps: true + scaling_key: masked_kspace + masking: + name: FastMRIEquispaced + accelerations: + - 4 + center_fractions: + - 0.08 + scale_percentile: 0.995 + use_seed: true + text_description: 4x + - name: FastMRI + transforms: + estimate_sensitivity_maps: true + scaling_key: masked_kspace + masking: + name: FastMRIEquispaced + accelerations: + - 8 + center_fractions: + - 0.04 + scale_percentile: 0.995 + use_seed: true + text_description: 8x + - name: FastMRI + transforms: + estimate_sensitivity_maps: true + scaling_key: masked_kspace + masking: + name: FastMRIEquispaced + accelerations: + - 16 + center_fractions: + - 0.02 + scale_percentile: 0.995 + use_seed: true + text_description: 16x + batch_size: 20 + metrics: + - fastmri_psnr + - fastmri_ssim + - fastmri_nmse + regularizers: [] + crop: header +inference: + dataset: + name: FastMRI + transforms: + masking: + name: FastMRIEquispaced + accelerations: + - 4.0 + center_fractions: + - 0.08 + mode: STATIC + cropping: + crop: null + sensitivity_map_estimation: + estimate_sensitivity_maps: true + normalization: + scaling_key: masked_kspace + scale_percentile: 0.995 + use_seed: true + text_description: inference-4x + batch_size: 1 + crop: header +logging: + log_as_image: null + tensorboard: + num_images: 4 diff --git a/projects/modulated_convolution/configs/vsharp/knee/vsharp_modconv_partial_in_triang.yaml b/projects/modulated_convolution/configs/vsharp/knee/vsharp_modconv_partial_in_triang.yaml new file mode 100644 index 00000000..f95d6315 --- /dev/null +++ b/projects/modulated_convolution/configs/vsharp/knee/vsharp_modconv_partial_in_triang.yaml @@ -0,0 +1,194 @@ +model: + model_name: vsharp.vsharp.VSharpNet + num_steps: 12 + num_steps_dc_gd: 10 + image_init: SENSE + no_parameter_sharing: true + auxiliary_steps: -1 + image_model_architecture: UNET + initializer_channels: + - 32 + - 32 + - 64 + - 64 + initializer_dilations: + - 1 + - 1 + - 2 + - 4 + initializer_multiscale: 1 + initializer_activation: PRELU + conv_modulation: PARTIAL_IN + aux_in_features: 2 + log_aux: true + fc_hidden_features: + - 32 + - 32 + fc_groups: 1 + fc_activation: SOFTPLUS + num_weights: null + modulation_at_input: false + image_resnet_hidden_channels: 128 + image_resnet_num_blocks: 15 + image_resnet_batchnorm: true + image_resnet_scale: 0.1 + image_unet_num_filters: 32 + image_unet_num_pool_layers: 4 + image_unet_dropout: 0.0 + image_unet_norm_type: INSTANCE + image_unet_adain_hidden_features: null + image_didn_hidden_channels: 16 + image_didn_num_dubs: 6 + image_didn_num_convs_recon: 9 + image_conv_hidden_channels: 64 + image_conv_n_convs: 15 + image_conv_activation: ActivationType.RELU + image_conv_batchnorm: false +additional_models: + sensitivity_model: + model_name: unet.unet_2d.UnetModel2d + in_channels: 2 + out_channels: 2 + num_filters: 16 + num_pool_layers: 4 + dropout_probability: 0.0 + modulation: NONE + aux_in_features: null + fc_hidden_features: null + fc_groups: 1 + fc_activation: SIGMOID + num_weights: null +physics: + forward_operator: fft2 + backward_operator: ifft2 + use_noise_matrix: false + noise_matrix_scaling: 1.0 +training: + datasets: + - name: FastMRI + transforms: + crop: reconstruction_size + estimate_sensitivity_maps: true + scaling_key: masked_kspace + image_center_crop: false + masking: + name: FastMRIEquispaced + accelerations: + - 4 + - 16 + center_fractions: + - 0.08 + - 0.02 + range_mode: LINEAR + scale_percentile: 0.995 + use_seed: false + delete_kspace: false + random_flip_probability: 0.5 + random_rotation_probability: 0.5 + model_checkpoint: null + optimizer: Adam + lr: 0.002 + weight_decay: 0.0 + batch_size: 1 + lr_step_size: 15000 + lr_gamma: 0.8 + lr_warmup_iter: 1000 + swa_start_iter: null + num_iterations: 80001 + validation_steps: 4000 + gradient_steps: 1 + gradient_clipping: 0.0 + gradient_debug: false + loss: + crop: header + losses: + - function: l1_loss + multiplier: 1.0 + - function: ssim_loss + multiplier: 1.0 + - function: hfen_l2_norm_loss + multiplier: 1.0 + - function: hfen_l1_norm_loss + multiplier: 1.0 + - function: kspace_nmae_loss + multiplier: 1.0 + - function: kspace_nmse_loss + multiplier: 1.0 + checkpointer: + checkpoint_steps: 4000 + metrics: [] + regularizers: [] +validation: + datasets: + - name: FastMRI + transforms: + estimate_sensitivity_maps: true + scaling_key: masked_kspace + masking: + name: FastMRIEquispaced + accelerations: + - 4 + center_fractions: + - 0.08 + scale_percentile: 0.995 + use_seed: true + text_description: 4x + - name: FastMRI + transforms: + estimate_sensitivity_maps: true + scaling_key: masked_kspace + masking: + name: FastMRIEquispaced + accelerations: + - 8 + center_fractions: + - 0.04 + scale_percentile: 0.995 + use_seed: true + text_description: 8x + - name: FastMRI + transforms: + estimate_sensitivity_maps: true + scaling_key: masked_kspace + masking: + name: FastMRIEquispaced + accelerations: + - 16 + center_fractions: + - 0.02 + scale_percentile: 0.995 + use_seed: true + text_description: 16x + batch_size: 20 + metrics: + - fastmri_psnr + - fastmri_ssim + - fastmri_nmse + regularizers: [] + crop: header +inference: + dataset: + name: FastMRI + transforms: + masking: + name: FastMRIEquispaced + accelerations: + - 4.0 + center_fractions: + - 0.08 + mode: STATIC + cropping: + crop: null + sensitivity_map_estimation: + estimate_sensitivity_maps: true + normalization: + scaling_key: masked_kspace + scale_percentile: 0.995 + use_seed: true + text_description: inference-4x + batch_size: 1 + crop: header +logging: + log_as_image: null + tensorboard: + num_images: 4 diff --git a/projects/modulated_convolution/configs/vsharp/knee/vsharp_triang.yaml b/projects/modulated_convolution/configs/vsharp/knee/vsharp_triang.yaml new file mode 100644 index 00000000..3a8d4de9 --- /dev/null +++ b/projects/modulated_convolution/configs/vsharp/knee/vsharp_triang.yaml @@ -0,0 +1,194 @@ +model: + model_name: vsharp.vsharp.VSharpNet + num_steps: 12 + num_steps_dc_gd: 10 + image_init: SENSE + no_parameter_sharing: true + auxiliary_steps: -1 + image_model_architecture: UNET + initializer_channels: + - 32 + - 32 + - 64 + - 64 + initializer_dilations: + - 1 + - 1 + - 2 + - 4 + initializer_multiscale: 1 + initializer_activation: PRELU + conv_modulation: NONE + aux_in_features: 2 + log_aux: true + fc_hidden_features: + - 32 + - 32 + fc_groups: 1 + fc_activation: SOFTPLUS + num_weights: null + modulation_at_input: false + image_resnet_hidden_channels: 128 + image_resnet_num_blocks: 15 + image_resnet_batchnorm: true + image_resnet_scale: 0.1 + image_unet_num_filters: 32 + image_unet_num_pool_layers: 4 + image_unet_dropout: 0.0 + image_unet_norm_type: INSTANCE + image_unet_adain_hidden_features: null + image_didn_hidden_channels: 16 + image_didn_num_dubs: 6 + image_didn_num_convs_recon: 9 + image_conv_hidden_channels: 64 + image_conv_n_convs: 15 + image_conv_activation: ActivationType.RELU + image_conv_batchnorm: false +additional_models: + sensitivity_model: + model_name: unet.unet_2d.UnetModel2d + in_channels: 2 + out_channels: 2 + num_filters: 16 + num_pool_layers: 4 + dropout_probability: 0.0 + modulation: NONE + aux_in_features: null + fc_hidden_features: null + fc_groups: 1 + fc_activation: SIGMOID + num_weights: null +physics: + forward_operator: fft2 + backward_operator: ifft2 + use_noise_matrix: false + noise_matrix_scaling: 1.0 +training: + datasets: + - name: FastMRI + transforms: + crop: reconstruction_size + estimate_sensitivity_maps: true + scaling_key: masked_kspace + image_center_crop: false + masking: + name: FastMRIEquispaced + accelerations: + - 4 + - 16 + center_fractions: + - 0.08 + - 0.02 + range_mode: LINEAR + scale_percentile: 0.995 + use_seed: false + delete_kspace: false + random_flip_probability: 0.5 + random_rotation_probability: 0.5 + model_checkpoint: null + optimizer: Adam + lr: 0.002 + weight_decay: 0.0 + batch_size: 1 + lr_step_size: 15000 + lr_gamma: 0.8 + lr_warmup_iter: 1000 + swa_start_iter: null + num_iterations: 80001 + validation_steps: 4000 + gradient_steps: 1 + gradient_clipping: 0.0 + gradient_debug: false + loss: + crop: header + losses: + - function: l1_loss + multiplier: 1.0 + - function: ssim_loss + multiplier: 1.0 + - function: hfen_l2_norm_loss + multiplier: 1.0 + - function: hfen_l1_norm_loss + multiplier: 1.0 + - function: kspace_nmae_loss + multiplier: 1.0 + - function: kspace_nmse_loss + multiplier: 1.0 + checkpointer: + checkpoint_steps: 4000 + metrics: [] + regularizers: [] +validation: + datasets: + - name: FastMRI + transforms: + estimate_sensitivity_maps: true + scaling_key: masked_kspace + masking: + name: FastMRIEquispaced + accelerations: + - 4 + center_fractions: + - 0.08 + scale_percentile: 0.995 + use_seed: true + text_description: 4x + - name: FastMRI + transforms: + estimate_sensitivity_maps: true + scaling_key: masked_kspace + masking: + name: FastMRIEquispaced + accelerations: + - 8 + center_fractions: + - 0.04 + scale_percentile: 0.995 + use_seed: true + text_description: 8x + - name: FastMRI + transforms: + estimate_sensitivity_maps: true + scaling_key: masked_kspace + masking: + name: FastMRIEquispaced + accelerations: + - 16 + center_fractions: + - 0.02 + scale_percentile: 0.995 + use_seed: true + text_description: 16x + batch_size: 20 + metrics: + - fastmri_psnr + - fastmri_ssim + - fastmri_nmse + regularizers: [] + crop: header +inference: + dataset: + name: FastMRI + transforms: + masking: + name: FastMRIEquispaced + accelerations: + - 4.0 + center_fractions: + - 0.08 + mode: STATIC + cropping: + crop: null + sensitivity_map_estimation: + estimate_sensitivity_maps: true + normalization: + scaling_key: masked_kspace + scale_percentile: 0.995 + use_seed: true + text_description: inference-4x + batch_size: 1 + crop: header +logging: + log_as_image: null + tensorboard: + num_images: 4 diff --git a/projects/modulated_convolution/configs/vsharp/prostate/vsharp_modconv_features_triang.yaml b/projects/modulated_convolution/configs/vsharp/prostate/vsharp_modconv_features_triang.yaml new file mode 100644 index 00000000..de0c1356 --- /dev/null +++ b/projects/modulated_convolution/configs/vsharp/prostate/vsharp_modconv_features_triang.yaml @@ -0,0 +1,192 @@ +model: + model_name: vsharp.vsharp.VSharpNet + num_steps: 12 + num_steps_dc_gd: 10 + image_init: SENSE + no_parameter_sharing: true + auxiliary_steps: -1 + image_model_architecture: UNET + initializer_channels: + - 32 + - 32 + - 64 + - 64 + initializer_dilations: + - 1 + - 1 + - 2 + - 4 + initializer_multiscale: 1 + initializer_activation: PRELU + conv_modulation: FEATURES + aux_in_features: 2 + log_aux: true + fc_hidden_features: + - 32 + - 32 + fc_groups: 1 + fc_activation: SOFTPLUS + num_weights: null + modulation_at_input: false + image_resnet_hidden_channels: 128 + image_resnet_num_blocks: 15 + image_resnet_batchnorm: true + image_resnet_scale: 0.1 + image_unet_num_filters: 32 + image_unet_num_pool_layers: 4 + image_unet_dropout: 0.0 + image_unet_norm_type: INSTANCE + image_unet_adain_hidden_features: null + image_didn_hidden_channels: 16 + image_didn_num_dubs: 6 + image_didn_num_convs_recon: 9 + image_conv_hidden_channels: 64 + image_conv_n_convs: 15 + image_conv_activation: ActivationType.RELU + image_conv_batchnorm: false +additional_models: + sensitivity_model: + model_name: unet.unet_2d.UnetModel2d + in_channels: 2 + out_channels: 2 + num_filters: 16 + num_pool_layers: 4 + dropout_probability: 0.0 + modulation: NONE + aux_in_features: null + fc_hidden_features: null + fc_groups: 1 + fc_activation: SIGMOID + num_weights: null +physics: + forward_operator: fft2 + backward_operator: ifft2 + use_noise_matrix: false + noise_matrix_scaling: 1.0 +training: + datasets: + - name: FastMRI + transforms: + crop: reconstruction_size + estimate_sensitivity_maps: true + scaling_key: masked_kspace + image_center_crop: false + masking: + name: FastMRIEquispaced + accelerations: + - 4 + - 16 + center_fractions: + - 0.08 + - 0.02 + range_mode: LINEAR + scale_percentile: 0.995 + use_seed: false + delete_kspace: false + model_checkpoint: null + optimizer: Adam + lr: 0.002 + weight_decay: 0.0 + batch_size: 1 + lr_step_size: 30000 + lr_gamma: 0.8 + lr_warmup_iter: 1000 + swa_start_iter: null + num_iterations: 150000 + validation_steps: 4000 + gradient_steps: 1 + gradient_clipping: 0.0 + gradient_debug: false + loss: + crop: header + losses: + - function: l1_loss + multiplier: 1.0 + - function: ssim_loss + multiplier: 1.0 + - function: hfen_l2_norm_loss + multiplier: 1.0 + - function: hfen_l1_norm_loss + multiplier: 1.0 + - function: kspace_nmae_loss + multiplier: 1.0 + - function: kspace_nmse_loss + multiplier: 1.0 + checkpointer: + checkpoint_steps: 4000 + metrics: [] + regularizers: [] +validation: + datasets: + - name: FastMRI + transforms: + estimate_sensitivity_maps: true + scaling_key: masked_kspace + masking: + name: FastMRIEquispaced + accelerations: + - 4 + center_fractions: + - 0.08 + scale_percentile: 0.995 + use_seed: true + text_description: 4x + - name: FastMRI + transforms: + estimate_sensitivity_maps: true + scaling_key: masked_kspace + masking: + name: FastMRIEquispaced + accelerations: + - 8 + center_fractions: + - 0.04 + scale_percentile: 0.995 + use_seed: true + text_description: 8x + - name: FastMRI + transforms: + estimate_sensitivity_maps: true + scaling_key: masked_kspace + masking: + name: FastMRIEquispaced + accelerations: + - 16 + center_fractions: + - 0.02 + scale_percentile: 0.995 + use_seed: true + text_description: 16x + batch_size: 20 + metrics: + - fastmri_psnr + - fastmri_ssim + - fastmri_nmse + regularizers: [] + crop: header +inference: + dataset: + name: FastMRI + transforms: + masking: + name: FastMRIEquispaced + accelerations: + - 4.0 + center_fractions: + - 0.08 + mode: STATIC + cropping: + crop: null + sensitivity_map_estimation: + estimate_sensitivity_maps: true + normalization: + scaling_key: masked_kspace + scale_percentile: 0.995 + use_seed: true + text_description: inference-4x + batch_size: 1 + crop: header +logging: + log_as_image: null + tensorboard: + num_images: 4 diff --git a/projects/modulated_convolution/configs/vsharp/prostate/vsharp_modconv_features_triang_32_16.yaml b/projects/modulated_convolution/configs/vsharp/prostate/vsharp_modconv_features_triang_32_16.yaml new file mode 100644 index 00000000..b02a3a84 --- /dev/null +++ b/projects/modulated_convolution/configs/vsharp/prostate/vsharp_modconv_features_triang_32_16.yaml @@ -0,0 +1,192 @@ +model: + model_name: vsharp.vsharp.VSharpNet + num_steps: 12 + num_steps_dc_gd: 10 + image_init: SENSE + no_parameter_sharing: true + auxiliary_steps: -1 + image_model_architecture: UNET + initializer_channels: + - 32 + - 32 + - 64 + - 64 + initializer_dilations: + - 1 + - 1 + - 2 + - 4 + initializer_multiscale: 1 + initializer_activation: PRELU + conv_modulation: FEATURES + aux_in_features: 2 + log_aux: true + fc_hidden_features: + - 32 + - 16 + fc_groups: 1 + fc_activation: SOFTPLUS + num_weights: null + modulation_at_input: false + image_resnet_hidden_channels: 128 + image_resnet_num_blocks: 15 + image_resnet_batchnorm: true + image_resnet_scale: 0.1 + image_unet_num_filters: 32 + image_unet_num_pool_layers: 4 + image_unet_dropout: 0.0 + image_unet_norm_type: INSTANCE + image_unet_adain_hidden_features: null + image_didn_hidden_channels: 16 + image_didn_num_dubs: 6 + image_didn_num_convs_recon: 9 + image_conv_hidden_channels: 64 + image_conv_n_convs: 15 + image_conv_activation: ActivationType.RELU + image_conv_batchnorm: false +additional_models: + sensitivity_model: + model_name: unet.unet_2d.UnetModel2d + in_channels: 2 + out_channels: 2 + num_filters: 16 + num_pool_layers: 4 + dropout_probability: 0.0 + modulation: NONE + aux_in_features: null + fc_hidden_features: null + fc_groups: 1 + fc_activation: SIGMOID + num_weights: null +physics: + forward_operator: fft2 + backward_operator: ifft2 + use_noise_matrix: false + noise_matrix_scaling: 1.0 +training: + datasets: + - name: FastMRI + transforms: + crop: reconstruction_size + estimate_sensitivity_maps: true + scaling_key: masked_kspace + image_center_crop: false + masking: + name: FastMRIEquispaced + accelerations: + - 4 + - 16 + center_fractions: + - 0.08 + - 0.02 + range_mode: LINEAR + scale_percentile: 0.995 + use_seed: false + delete_kspace: false + model_checkpoint: null + optimizer: Adam + lr: 0.002 + weight_decay: 0.0 + batch_size: 1 + lr_step_size: 30000 + lr_gamma: 0.8 + lr_warmup_iter: 1000 + swa_start_iter: null + num_iterations: 150000 + validation_steps: 4000 + gradient_steps: 1 + gradient_clipping: 0.0 + gradient_debug: false + loss: + crop: header + losses: + - function: l1_loss + multiplier: 1.0 + - function: ssim_loss + multiplier: 1.0 + - function: hfen_l2_norm_loss + multiplier: 1.0 + - function: hfen_l1_norm_loss + multiplier: 1.0 + - function: kspace_nmae_loss + multiplier: 1.0 + - function: kspace_nmse_loss + multiplier: 1.0 + checkpointer: + checkpoint_steps: 4000 + metrics: [] + regularizers: [] +validation: + datasets: + - name: FastMRI + transforms: + estimate_sensitivity_maps: true + scaling_key: masked_kspace + masking: + name: FastMRIEquispaced + accelerations: + - 4 + center_fractions: + - 0.08 + scale_percentile: 0.995 + use_seed: true + text_description: 4x + - name: FastMRI + transforms: + estimate_sensitivity_maps: true + scaling_key: masked_kspace + masking: + name: FastMRIEquispaced + accelerations: + - 8 + center_fractions: + - 0.04 + scale_percentile: 0.995 + use_seed: true + text_description: 8x + - name: FastMRI + transforms: + estimate_sensitivity_maps: true + scaling_key: masked_kspace + masking: + name: FastMRIEquispaced + accelerations: + - 16 + center_fractions: + - 0.02 + scale_percentile: 0.995 + use_seed: true + text_description: 16x + batch_size: 20 + metrics: + - fastmri_psnr + - fastmri_ssim + - fastmri_nmse + regularizers: [] + crop: header +inference: + dataset: + name: FastMRI + transforms: + masking: + name: FastMRIEquispaced + accelerations: + - 4.0 + center_fractions: + - 0.08 + mode: STATIC + cropping: + crop: null + sensitivity_map_estimation: + estimate_sensitivity_maps: true + normalization: + scaling_key: masked_kspace + scale_percentile: 0.995 + use_seed: true + text_description: inference-4x + batch_size: 1 + crop: header +logging: + log_as_image: null + tensorboard: + num_images: 4 diff --git a/projects/modulated_convolution/configs/vsharp/prostate/vsharp_modconv_features_triang_32_8.yaml b/projects/modulated_convolution/configs/vsharp/prostate/vsharp_modconv_features_triang_32_8.yaml new file mode 100644 index 00000000..fe961f49 --- /dev/null +++ b/projects/modulated_convolution/configs/vsharp/prostate/vsharp_modconv_features_triang_32_8.yaml @@ -0,0 +1,192 @@ +model: + model_name: vsharp.vsharp.VSharpNet + num_steps: 12 + num_steps_dc_gd: 10 + image_init: SENSE + no_parameter_sharing: true + auxiliary_steps: -1 + image_model_architecture: UNET + initializer_channels: + - 32 + - 32 + - 64 + - 64 + initializer_dilations: + - 1 + - 1 + - 2 + - 4 + initializer_multiscale: 1 + initializer_activation: PRELU + conv_modulation: FEATURES + aux_in_features: 2 + log_aux: true + fc_hidden_features: + - 32 + - 8 + fc_groups: 1 + fc_activation: SOFTPLUS + num_weights: null + modulation_at_input: false + image_resnet_hidden_channels: 128 + image_resnet_num_blocks: 15 + image_resnet_batchnorm: true + image_resnet_scale: 0.1 + image_unet_num_filters: 32 + image_unet_num_pool_layers: 4 + image_unet_dropout: 0.0 + image_unet_norm_type: INSTANCE + image_unet_adain_hidden_features: null + image_didn_hidden_channels: 16 + image_didn_num_dubs: 6 + image_didn_num_convs_recon: 9 + image_conv_hidden_channels: 64 + image_conv_n_convs: 15 + image_conv_activation: ActivationType.RELU + image_conv_batchnorm: false +additional_models: + sensitivity_model: + model_name: unet.unet_2d.UnetModel2d + in_channels: 2 + out_channels: 2 + num_filters: 16 + num_pool_layers: 4 + dropout_probability: 0.0 + modulation: NONE + aux_in_features: null + fc_hidden_features: null + fc_groups: 1 + fc_activation: SIGMOID + num_weights: null +physics: + forward_operator: fft2 + backward_operator: ifft2 + use_noise_matrix: false + noise_matrix_scaling: 1.0 +training: + datasets: + - name: FastMRI + transforms: + crop: reconstruction_size + estimate_sensitivity_maps: true + scaling_key: masked_kspace + image_center_crop: false + masking: + name: FastMRIEquispaced + accelerations: + - 4 + - 16 + center_fractions: + - 0.08 + - 0.02 + range_mode: LINEAR + scale_percentile: 0.995 + use_seed: false + delete_kspace: false + model_checkpoint: null + optimizer: Adam + lr: 0.002 + weight_decay: 0.0 + batch_size: 1 + lr_step_size: 30000 + lr_gamma: 0.8 + lr_warmup_iter: 1000 + swa_start_iter: null + num_iterations: 150000 + validation_steps: 4000 + gradient_steps: 1 + gradient_clipping: 0.0 + gradient_debug: false + loss: + crop: header + losses: + - function: l1_loss + multiplier: 1.0 + - function: ssim_loss + multiplier: 1.0 + - function: hfen_l2_norm_loss + multiplier: 1.0 + - function: hfen_l1_norm_loss + multiplier: 1.0 + - function: kspace_nmae_loss + multiplier: 1.0 + - function: kspace_nmse_loss + multiplier: 1.0 + checkpointer: + checkpoint_steps: 4000 + metrics: [] + regularizers: [] +validation: + datasets: + - name: FastMRI + transforms: + estimate_sensitivity_maps: true + scaling_key: masked_kspace + masking: + name: FastMRIEquispaced + accelerations: + - 4 + center_fractions: + - 0.08 + scale_percentile: 0.995 + use_seed: true + text_description: 4x + - name: FastMRI + transforms: + estimate_sensitivity_maps: true + scaling_key: masked_kspace + masking: + name: FastMRIEquispaced + accelerations: + - 8 + center_fractions: + - 0.04 + scale_percentile: 0.995 + use_seed: true + text_description: 8x + - name: FastMRI + transforms: + estimate_sensitivity_maps: true + scaling_key: masked_kspace + masking: + name: FastMRIEquispaced + accelerations: + - 16 + center_fractions: + - 0.02 + scale_percentile: 0.995 + use_seed: true + text_description: 16x + batch_size: 20 + metrics: + - fastmri_psnr + - fastmri_ssim + - fastmri_nmse + regularizers: [] + crop: header +inference: + dataset: + name: FastMRI + transforms: + masking: + name: FastMRIEquispaced + accelerations: + - 4.0 + center_fractions: + - 0.08 + mode: STATIC + cropping: + crop: null + sensitivity_map_estimation: + estimate_sensitivity_maps: true + normalization: + scaling_key: masked_kspace + scale_percentile: 0.995 + use_seed: true + text_description: inference-4x + batch_size: 1 + crop: header +logging: + log_as_image: null + tensorboard: + num_images: 4 diff --git a/projects/modulated_convolution/configs/vsharp/prostate/vsharp_modconv_partial_in_triang.yaml b/projects/modulated_convolution/configs/vsharp/prostate/vsharp_modconv_partial_in_triang.yaml new file mode 100644 index 00000000..96f8d1c3 --- /dev/null +++ b/projects/modulated_convolution/configs/vsharp/prostate/vsharp_modconv_partial_in_triang.yaml @@ -0,0 +1,192 @@ +model: + model_name: vsharp.vsharp.VSharpNet + num_steps: 12 + num_steps_dc_gd: 10 + image_init: SENSE + no_parameter_sharing: true + auxiliary_steps: -1 + image_model_architecture: UNET + initializer_channels: + - 32 + - 32 + - 64 + - 64 + initializer_dilations: + - 1 + - 1 + - 2 + - 4 + initializer_multiscale: 1 + initializer_activation: PRELU + conv_modulation: PARTIAL_IN + aux_in_features: 2 + log_aux: true + fc_hidden_features: + - 32 + - 32 + fc_groups: 1 + fc_activation: SOFTPLUS + num_weights: null + modulation_at_input: false + image_resnet_hidden_channels: 128 + image_resnet_num_blocks: 15 + image_resnet_batchnorm: true + image_resnet_scale: 0.1 + image_unet_num_filters: 32 + image_unet_num_pool_layers: 4 + image_unet_dropout: 0.0 + image_unet_norm_type: INSTANCE + image_unet_adain_hidden_features: null + image_didn_hidden_channels: 16 + image_didn_num_dubs: 6 + image_didn_num_convs_recon: 9 + image_conv_hidden_channels: 64 + image_conv_n_convs: 15 + image_conv_activation: ActivationType.RELU + image_conv_batchnorm: false +additional_models: + sensitivity_model: + model_name: unet.unet_2d.UnetModel2d + in_channels: 2 + out_channels: 2 + num_filters: 16 + num_pool_layers: 4 + dropout_probability: 0.0 + modulation: NONE + aux_in_features: null + fc_hidden_features: null + fc_groups: 1 + fc_activation: SIGMOID + num_weights: null +physics: + forward_operator: fft2 + backward_operator: ifft2 + use_noise_matrix: false + noise_matrix_scaling: 1.0 +training: + datasets: + - name: FastMRI + transforms: + crop: reconstruction_size + estimate_sensitivity_maps: true + scaling_key: masked_kspace + image_center_crop: false + masking: + name: FastMRIEquispaced + accelerations: + - 4 + - 16 + center_fractions: + - 0.08 + - 0.02 + range_mode: LINEAR + scale_percentile: 0.995 + use_seed: false + delete_kspace: false + model_checkpoint: null + optimizer: Adam + lr: 0.002 + weight_decay: 0.0 + batch_size: 1 + lr_step_size: 30000 + lr_gamma: 0.8 + lr_warmup_iter: 1000 + swa_start_iter: null + num_iterations: 150000 + validation_steps: 4000 + gradient_steps: 1 + gradient_clipping: 0.0 + gradient_debug: false + loss: + crop: header + losses: + - function: l1_loss + multiplier: 1.0 + - function: ssim_loss + multiplier: 1.0 + - function: hfen_l2_norm_loss + multiplier: 1.0 + - function: hfen_l1_norm_loss + multiplier: 1.0 + - function: kspace_nmae_loss + multiplier: 1.0 + - function: kspace_nmse_loss + multiplier: 1.0 + checkpointer: + checkpoint_steps: 4000 + metrics: [] + regularizers: [] +validation: + datasets: + - name: FastMRI + transforms: + estimate_sensitivity_maps: true + scaling_key: masked_kspace + masking: + name: FastMRIEquispaced + accelerations: + - 4 + center_fractions: + - 0.08 + scale_percentile: 0.995 + use_seed: true + text_description: 4x + - name: FastMRI + transforms: + estimate_sensitivity_maps: true + scaling_key: masked_kspace + masking: + name: FastMRIEquispaced + accelerations: + - 8 + center_fractions: + - 0.04 + scale_percentile: 0.995 + use_seed: true + text_description: 8x + - name: FastMRI + transforms: + estimate_sensitivity_maps: true + scaling_key: masked_kspace + masking: + name: FastMRIEquispaced + accelerations: + - 16 + center_fractions: + - 0.02 + scale_percentile: 0.995 + use_seed: true + text_description: 16x + batch_size: 20 + metrics: + - fastmri_psnr + - fastmri_ssim + - fastmri_nmse + regularizers: [] + crop: header +inference: + dataset: + name: FastMRI + transforms: + masking: + name: FastMRIEquispaced + accelerations: + - 4.0 + center_fractions: + - 0.08 + mode: STATIC + cropping: + crop: null + sensitivity_map_estimation: + estimate_sensitivity_maps: true + normalization: + scaling_key: masked_kspace + scale_percentile: 0.995 + use_seed: true + text_description: inference-4x + batch_size: 1 + crop: header +logging: + log_as_image: null + tensorboard: + num_images: 4 diff --git a/projects/modulated_convolution/configs/vsharp/prostate/vsharp_triang.yaml b/projects/modulated_convolution/configs/vsharp/prostate/vsharp_triang.yaml new file mode 100644 index 00000000..53ff3760 --- /dev/null +++ b/projects/modulated_convolution/configs/vsharp/prostate/vsharp_triang.yaml @@ -0,0 +1,192 @@ +model: + model_name: vsharp.vsharp.VSharpNet + num_steps: 12 + num_steps_dc_gd: 10 + image_init: SENSE + no_parameter_sharing: true + auxiliary_steps: -1 + image_model_architecture: UNET + initializer_channels: + - 32 + - 32 + - 64 + - 64 + initializer_dilations: + - 1 + - 1 + - 2 + - 4 + initializer_multiscale: 1 + initializer_activation: PRELU + conv_modulation: NONE + aux_in_features: 2 + log_aux: true + fc_hidden_features: + - 32 + - 32 + fc_groups: 1 + fc_activation: SOFTPLUS + num_weights: null + modulation_at_input: false + image_resnet_hidden_channels: 128 + image_resnet_num_blocks: 15 + image_resnet_batchnorm: true + image_resnet_scale: 0.1 + image_unet_num_filters: 32 + image_unet_num_pool_layers: 4 + image_unet_dropout: 0.0 + image_unet_norm_type: INSTANCE + image_unet_adain_hidden_features: null + image_didn_hidden_channels: 16 + image_didn_num_dubs: 6 + image_didn_num_convs_recon: 9 + image_conv_hidden_channels: 64 + image_conv_n_convs: 15 + image_conv_activation: ActivationType.RELU + image_conv_batchnorm: false +additional_models: + sensitivity_model: + model_name: unet.unet_2d.UnetModel2d + in_channels: 2 + out_channels: 2 + num_filters: 16 + num_pool_layers: 4 + dropout_probability: 0.0 + modulation: NONE + aux_in_features: null + fc_hidden_features: null + fc_groups: 1 + fc_activation: SIGMOID + num_weights: null +physics: + forward_operator: fft2 + backward_operator: ifft2 + use_noise_matrix: false + noise_matrix_scaling: 1.0 +training: + datasets: + - name: FastMRI + transforms: + crop: reconstruction_size + estimate_sensitivity_maps: true + scaling_key: masked_kspace + image_center_crop: false + masking: + name: FastMRIEquispaced + accelerations: + - 4 + - 16 + center_fractions: + - 0.08 + - 0.02 + range_mode: LINEAR + scale_percentile: 0.995 + use_seed: false + delete_kspace: false + model_checkpoint: null + optimizer: Adam + lr: 0.002 + weight_decay: 0.0 + batch_size: 1 + lr_step_size: 30000 + lr_gamma: 0.8 + lr_warmup_iter: 1000 + swa_start_iter: null + num_iterations: 150000 + validation_steps: 4000 + gradient_steps: 1 + gradient_clipping: 0.0 + gradient_debug: false + loss: + crop: header + losses: + - function: l1_loss + multiplier: 1.0 + - function: ssim_loss + multiplier: 1.0 + - function: hfen_l2_norm_loss + multiplier: 1.0 + - function: hfen_l1_norm_loss + multiplier: 1.0 + - function: kspace_nmae_loss + multiplier: 1.0 + - function: kspace_nmse_loss + multiplier: 1.0 + checkpointer: + checkpoint_steps: 4000 + metrics: [] + regularizers: [] +validation: + datasets: + - name: FastMRI + transforms: + estimate_sensitivity_maps: true + scaling_key: masked_kspace + masking: + name: FastMRIEquispaced + accelerations: + - 4 + center_fractions: + - 0.08 + scale_percentile: 0.995 + use_seed: true + text_description: 4x + - name: FastMRI + transforms: + estimate_sensitivity_maps: true + scaling_key: masked_kspace + masking: + name: FastMRIEquispaced + accelerations: + - 8 + center_fractions: + - 0.04 + scale_percentile: 0.995 + use_seed: true + text_description: 8x + - name: FastMRI + transforms: + estimate_sensitivity_maps: true + scaling_key: masked_kspace + masking: + name: FastMRIEquispaced + accelerations: + - 16 + center_fractions: + - 0.02 + scale_percentile: 0.995 + use_seed: true + text_description: 16x + batch_size: 20 + metrics: + - fastmri_psnr + - fastmri_ssim + - fastmri_nmse + regularizers: [] + crop: header +inference: + dataset: + name: FastMRI + transforms: + masking: + name: FastMRIEquispaced + accelerations: + - 4.0 + center_fractions: + - 0.08 + mode: STATIC + cropping: + crop: null + sensitivity_map_estimation: + estimate_sensitivity_maps: true + normalization: + scaling_key: masked_kspace + scale_percentile: 0.995 + use_seed: true + text_description: inference-4x + batch_size: 1 + crop: header +logging: + log_as_image: null + tensorboard: + num_images: 4 diff --git a/projects/modulated_convolution/figures/knee_reconstruction.png b/projects/modulated_convolution/figures/knee_reconstruction.png new file mode 100644 index 00000000..0440f771 Binary files /dev/null and b/projects/modulated_convolution/figures/knee_reconstruction.png differ diff --git a/projects/modulated_convolution/figures/modconv_architecture.png b/projects/modulated_convolution/figures/modconv_architecture.png new file mode 100644 index 00000000..523d8746 Binary files /dev/null and b/projects/modulated_convolution/figures/modconv_architecture.png differ diff --git a/projects/modulated_convolution/figures/modulation_mlp.png b/projects/modulated_convolution/figures/modulation_mlp.png new file mode 100644 index 00000000..49c361f9 Binary files /dev/null and b/projects/modulated_convolution/figures/modulation_mlp.png differ diff --git a/tests/checkpointer_test.py b/tests/checkpointer_test.py index f08b0fe9..88fa8528 100644 --- a/tests/checkpointer_test.py +++ b/tests/checkpointer_test.py @@ -60,7 +60,14 @@ def create_checkpointables(*keys): [], ["sensitivity_model", "optimizer", "something_which_is_not_stored"], ["sensitivity_model", "optimizer"], - ["sensitivity_model", "optimizer", "__author__", "__version__", "__datetime__", "__mixed_precision__"], + [ + "sensitivity_model", + "optimizer", + "__author__", + "__version__", + "__datetime__", + "__mixed_precision__", + ], ], ) def test_checkpointer(checkpoint_ids, checkpointables_keys): @@ -68,7 +75,11 @@ def test_checkpointer(checkpoint_ids, checkpointables_keys): for checkpoint_id in checkpoint_ids: checkpointables = create_checkpointables(checkpointables_keys) - checkpointer = Checkpointer(save_directory=pathlib.Path(tempdir), save_to_disk=True, **checkpointables) + checkpointer = Checkpointer( + save_directory=pathlib.Path(tempdir), + save_to_disk=True, + **checkpointables, + ) # Test save function checkpointer.save(iteration=checkpoint_id) # Test load function diff --git a/tests/tests_common/subsample_test.py b/tests/tests_common/subsample_test.py index 4cf399ff..fb075346 100644 --- a/tests/tests_common/subsample_test.py +++ b/tests/tests_common/subsample_test.py @@ -22,6 +22,7 @@ import torch from direct.common.subsample import * +from direct.utils.distributions import triangular_distribution @pytest.mark.parametrize( @@ -136,7 +137,12 @@ def test_mask_reuse_kt(mask_func, center_fracs, accelerations, batch_size, shape @pytest.mark.parametrize( "mask_func", - [FastMRIRandomMaskFunc, FastMRIEquispacedMaskFunc, FastMRIMagicMaskFunc, Gaussian1DMaskFunc], + [ + FastMRIRandomMaskFunc, + FastMRIEquispacedMaskFunc, + FastMRIMagicMaskFunc, + Gaussian1DMaskFunc, + ], ) @pytest.mark.parametrize( "center_fracs, accelerations, batch_size, dim", @@ -171,7 +177,12 @@ def test_cartesian_mask_low_freqs(mask_func, center_fracs, accelerations, batch_ @pytest.mark.parametrize( "mask_func", - [FastMRIRandomMaskFunc, FastMRIEquispacedMaskFunc, FastMRIMagicMaskFunc, Gaussian1DMaskFunc], + [ + FastMRIRandomMaskFunc, + FastMRIEquispacedMaskFunc, + FastMRIMagicMaskFunc, + Gaussian1DMaskFunc, + ], ) @pytest.mark.parametrize( "shape, center_fractions, accelerations, mode", @@ -199,7 +210,12 @@ def test_apply_mask_cartesian(mask_func, shape, center_fractions, accelerations, @pytest.mark.parametrize( "mask_func", - [FastMRIRandomMaskFunc, FastMRIEquispacedMaskFunc, FastMRIMagicMaskFunc, Gaussian1DMaskFunc], + [ + FastMRIRandomMaskFunc, + FastMRIEquispacedMaskFunc, + FastMRIMagicMaskFunc, + Gaussian1DMaskFunc, + ], ) @pytest.mark.parametrize( "shape, center_fractions, accelerations", @@ -534,3 +550,60 @@ def test_apply_kt_mask(mask_func, shape, accelerations, center_fractions): assert all(not np.allclose(mask[:, _], mask[:, _ + 1]) for _ in range(shape[1] - 1)) assert all(np.allclose(acs_mask[:, _], acs_mask[:, _ + 1]) for _ in range(shape[1] - 1)) + + +def test_linear_range_triangular_sampling_bias(): + mask_func = FastMRIRandomMaskFunc( + center_fractions=[0.04, 0.08], + accelerations=[4.0, 8.0], + range_mode=RangeMode.LINEAR, + ) + samples = [] + for seed in range(500): + _, acceleration, _ = mask_func((1, 320, 320, 2), seed=seed, return_acceleration=True) + samples.append(acceleration) + + mean_accel = float(np.mean(samples)) + assert mean_accel > 5.5 + + +def test_return_acceleration_on_poisson_mask(): + mask_func = VariableDensityPoissonMaskFunc( + center_fractions=[0.08, 0.04], + accelerations=[4.0, 8.0], + ) + mask, acceleration, center_fraction = mask_func((1, 320, 320, 2), seed=42, return_acceleration=True) + assert mask.dtype == torch.bool + assert acceleration in (4.0, 8.0) + assert 0.0 < center_fraction <= 0.25 + + +def test_build_masking_function_linear_range(): + mask_func = build_masking_function( + name="FastMRIRandom", + center_fractions=[0.04, 0.08], + accelerations=[4.0, 8.0], + range_mode=RangeMode.LINEAR, + ) + assert mask_func.range_mode == RangeMode.LINEAR + + +def test_equispaced_linear_range_no_invalid_randint(): + mask_func = FastMRIEquispacedMaskFunc( + center_fractions=[0.08, 0.02], + accelerations=[4, 16], + range_mode=RangeMode.LINEAR, + ) + shape = (1, 640, 368, 2) + for seed in range(1000): + mask, acceleration, center_fraction = mask_func(shape, seed=seed, return_acceleration=True) + assert mask.dtype == torch.bool + assert acceleration > 0 + assert center_fraction > 0 + + +def test_triangular_distribution_default_rng(): + samples = triangular_distribution(4.0, 8.0, n=100) + assert samples.shape == (100,) + assert np.all(samples >= 4.0) + assert np.all(samples <= 8.0) diff --git a/tests/tests_data/mri_transforms_test.py b/tests/tests_data/mri_transforms_test.py index 1ce75bd6..9cdf7bc6 100644 --- a/tests/tests_data/mri_transforms_test.py +++ b/tests/tests_data/mri_transforms_test.py @@ -91,7 +91,12 @@ def _mask_func(shape, seed=None, return_acs=False): [ (["key1"], [True], {}, {"key1": True}), (["key1", "key2"], [True, False], {}, {"key1": True, "key2": False}), - (["key1"], [True], {"existing_key": "existing_value"}, {"existing_key": "existing_value", "key1": True}), + ( + ["key1"], + [True], + {"existing_key": "existing_value"}, + {"existing_key": "existing_value", "key1": True}, + ), ], ) def test_add_boolean_keys_module(keys, values, sample, expected): @@ -183,7 +188,9 @@ def test_CreateSamplingMask(shape, return_acs, use_shape): sample = create_sample(shape) transform = CreateSamplingMask( - mask_func=_mask_func, shape=shape[1:-1] if use_shape else None, return_acs=return_acs + mask_func=_mask_func, + shape=shape[1:-1] if use_shape else None, + return_acs=return_acs, ) sample = transform(sample) print(sample["kspace"].shape, sample["sampling_mask"].shape) @@ -198,6 +205,25 @@ def test_CreateSamplingMask(shape, return_acs, use_shape): assert "acs_mask" in sample +def test_CreateSamplingMask_returns_acceleration(): + from direct.common.subsample import FastMRIEquispacedMaskFunc, RangeMode + + shape = (4, 32, 32, 2) + sample = create_sample(shape) + mask_func = FastMRIEquispacedMaskFunc( + center_fractions=[0.08, 0.04], + accelerations=[4.0, 8.0], + range_mode=RangeMode.UNIFORM, + ) + transform = CreateSamplingMask(mask_func=mask_func) + sample = transform(sample) + + assert "acceleration" in sample + assert "center_fraction" in sample + assert sample["acceleration"].numel() == 1 + assert sample["center_fraction"].numel() == 1 + + @pytest.mark.parametrize( "shape", [(4, 32, 32), (3, 10, 16)], @@ -222,7 +248,14 @@ def test_ApplyMask(shape): ) @pytest.mark.parametrize( "crop", - [(10, 5, 6), "reconstruction_size", "[10, 5, 6]", "(10, 5, 6)", None, "invalid_key"], + [ + (10, 5, 6), + "reconstruction_size", + "[10, 5, 6]", + "(10, 5, 6)", + None, + "invalid_key", + ], ) @pytest.mark.parametrize( "image_space_center_crop", @@ -323,7 +356,14 @@ def test_PadKspace(shape, pad_shape): ) @pytest.mark.parametrize( "crop", - [(10, 5, 6), "reconstruction_size", "[10, 5, 6]", "(10, 5, 6)", None, "invalid_key"], + [ + (10, 5, 6), + "reconstruction_size", + "[10, 5, 6]", + "(10, 5, 6)", + None, + "invalid_key", + ], ) @pytest.mark.parametrize( "image_space_center_crop", @@ -630,7 +670,14 @@ def test_CompressCoil(shape, compress_coils): @pytest.mark.parametrize( "shape, pad_coils", - [[(3, 10, 16), 5], [(5, 7, 6), 5], [(4, 5, 5), 2], [(4, 5, 5), None], [(3, 4, 6, 4), 4], [(5, 3, 3, 4), 3]], + [ + [(3, 10, 16), 5], + [(5, 7, 6), 5], + [(4, 5, 5), 2], + [(4, 5, 5), None], + [(3, 4, 6, 4), 4], + [(5, 3, 3, 4), 3], + ], ) @pytest.mark.parametrize( "key", @@ -775,7 +822,14 @@ def test_build_mri_transforms(shape, spatial_dims, estimate_body_coil_image, ima assert all( key in sample.keys() - for key in ["sampling_mask", "sensitivity_map", "target", "masked_kspace", "scaling_diff", "scaling_factor"] + for key in [ + "sampling_mask", + "sensitivity_map", + "target", + "masked_kspace", + "scaling_diff", + "scaling_factor", + ] ) assert sample["masked_kspace"].shape == shape + (2,) assert sample["sensitivity_map"].shape == shape + (2,) diff --git a/tests/tests_data/transforms_test.py b/tests/tests_data/transforms_test.py index d1589d68..1936b5f4 100644 --- a/tests/tests_data/transforms_test.py +++ b/tests/tests_data/transforms_test.py @@ -22,7 +22,7 @@ def create_input(shape): - data = np.random.randn(*shape).copy() + data = np.random.RandomState(0).randn(*shape).copy() data = torch.from_numpy(data).float() return data @@ -491,7 +491,11 @@ def test_complex_random_crop(shapes, crop_shape, sampler, sigma, expect_error, c if expect_error: with pytest.raises(ValueError): samples = transforms.complex_random_crop( - data_list, crop_shape, sampler=sampler, sigma=sigma, contiguous=contiguous + data_list, + crop_shape, + sampler=sampler, + sigma=sigma, + contiguous=contiguous, ) else: data_list = transforms.complex_random_crop( diff --git a/tests/tests_functionals/gradloss_test.py b/tests/tests_functionals/gradloss_test.py index cf14f818..47cba474 100644 --- a/tests/tests_functionals/gradloss_test.py +++ b/tests/tests_functionals/gradloss_test.py @@ -36,8 +36,8 @@ def test_nmse(image): image_batch.append(image) image_noise_batch.append(image_noise) - image_batch_torch = torch.tensor(image_batch) - image_noise_batch_torch = torch.tensor(image_noise_batch) + image_batch_torch = torch.tensor(np.array(image_batch)) + image_noise_batch_torch = torch.tensor(np.array(image_noise_batch)) grad_loss_l1 = SobelGradL1Loss(image_batch_torch, image_noise_batch_torch) grad_loss_l2 = SobelGradL2Loss(image_batch_torch, image_noise_batch_torch) diff --git a/tests/tests_nn/auxiliary_data_test.py b/tests/tests_nn/auxiliary_data_test.py new file mode 100644 index 00000000..d57d98c1 --- /dev/null +++ b/tests/tests_nn/auxiliary_data_test.py @@ -0,0 +1,202 @@ +# Copyright 2026 AI for Oncology Research Group. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from dataclasses import dataclass +from typing import Optional + +import pytest +import torch + +from direct.nn.conv.modulated import ( + AuxiliaryFeature, + ModConvType, + prepare_auxiliary_data, + register_auxiliary_feature, + resolve_auxiliary_features, +) + + +@dataclass +class _Cfg: + conv_modulation: ModConvType + aux_in_features: int + log_aux: bool + auxiliary_features: Optional[tuple[str, ...]] = None + + +def _batch_data() -> dict[str, torch.Tensor]: + return { + "acceleration": torch.tensor([4.0, 8.0]), + "center_fraction": torch.tensor([0.08, 0.04]), + "field_strength": torch.tensor([3.0, 1.5]), + } + + +def test_prepare_auxiliary_data_returns_none_without_modulation_config(): + cfg = object() + assert prepare_auxiliary_data(_batch_data(), cfg) is None # type: ignore[arg-type] + + +def test_prepare_auxiliary_data_returns_none_without_modulation(): + cfg = _Cfg(conv_modulation=ModConvType.NONE, aux_in_features=2, log_aux=False) + assert prepare_auxiliary_data(_batch_data(), cfg) is None + + +def test_prepare_auxiliary_data_for_adain_without_modulation(): + @dataclass + class _AdaINCfg: + conv_modulation: ModConvType = ModConvType.NONE + aux_in_features: int = 2 + log_aux: bool = True + auxiliary_features: Optional[tuple[str, ...]] = None + image_unet_norm_type: str = "ADAIN" + + auxiliary_data = prepare_auxiliary_data(_batch_data(), _AdaINCfg()) + assert auxiliary_data is not None + assert auxiliary_data.shape == (2, 2) + + +def test_prepare_auxiliary_data_two_features_without_log(): + cfg = _Cfg(conv_modulation=ModConvType.FEATURES, aux_in_features=2, log_aux=False) + auxiliary_data = prepare_auxiliary_data(_batch_data(), cfg) + + assert auxiliary_data is not None + assert auxiliary_data.shape == (2, 2) + torch.testing.assert_close( + auxiliary_data, + torch.tensor([[4.0, 0.08], [8.0, 0.04]]), + ) + + +def test_prepare_auxiliary_data_two_features_with_log(): + cfg = _Cfg(conv_modulation=ModConvType.FEATURES, aux_in_features=2, log_aux=True) + auxiliary_data = prepare_auxiliary_data(_batch_data(), cfg) + + assert auxiliary_data is not None + torch.testing.assert_close( + auxiliary_data, + torch.log(torch.tensor([[4.0, 8.0], [8.0, 4.0]])), + ) + + +def test_prepare_auxiliary_data_three_features(): + cfg = _Cfg(conv_modulation=ModConvType.FEATURES, aux_in_features=3, log_aux=False) + auxiliary_data = prepare_auxiliary_data(_batch_data(), cfg) + + assert auxiliary_data is not None + assert auxiliary_data.shape == (2, 3) + torch.testing.assert_close( + auxiliary_data, + torch.tensor([[4.0, 0.08, 3.0], [8.0, 0.04, 1.5]]), + ) + + +def test_prepare_auxiliary_data_explicit_feature_list(): + cfg = _Cfg( + conv_modulation=ModConvType.FEATURES, + aux_in_features=2, + log_aux=False, + auxiliary_features=("field_strength", "acceleration"), + ) + auxiliary_data = prepare_auxiliary_data(_batch_data(), cfg) + + assert auxiliary_data is not None + torch.testing.assert_close( + auxiliary_data, + torch.tensor([[3.0, 4.0], [1.5, 8.0]]), + ) + + +def test_prepare_auxiliary_data_missing_feature_raises(): + cfg = _Cfg(conv_modulation=ModConvType.FEATURES, aux_in_features=2, log_aux=False) + data = {"acceleration": torch.tensor([4.0])} + + with pytest.raises(KeyError, match="center_fraction"): + prepare_auxiliary_data(data, cfg) + + +def test_prepare_auxiliary_data_invalid_aux_in_features_raises(): + cfg = _Cfg(conv_modulation=ModConvType.FEATURES, aux_in_features=4, log_aux=False) + + with pytest.raises(ValueError, match="exceeds the number of default"): + prepare_auxiliary_data(_batch_data(), cfg) + + +def test_resolve_auxiliary_features_unknown_name_raises(): + with pytest.raises(ValueError, match="Unknown auxiliary feature"): + resolve_auxiliary_features(("acceleration", "unknown_feature"), aux_in_features=2) + + +def test_resolve_auxiliary_features_length_mismatch_raises(): + with pytest.raises(ValueError, match="auxiliary_features has length"): + resolve_auxiliary_features(("acceleration",), aux_in_features=2) + + +def test_resolve_auxiliary_features_non_positive_raises(): + with pytest.raises(ValueError, match="aux_in_features must be positive"): + resolve_auxiliary_features(None, aux_in_features=0) + + +def test_register_auxiliary_feature(): + feature = AuxiliaryFeature("custom_feature") + register_auxiliary_feature(feature) + resolved = resolve_auxiliary_features(("custom_feature",), aux_in_features=1) + assert resolved[0].key == "custom_feature" + + +def test_prepare_auxiliary_data_string_modulation(): + cfg = _Cfg(conv_modulation="features", aux_in_features=2, log_aux=False) # type: ignore[arg-type] + auxiliary_data = prepare_auxiliary_data(_batch_data(), cfg) + assert auxiliary_data is not None + assert auxiliary_data.shape == (2, 2) + + +def test_prepare_auxiliary_data_missing_aux_in_features(): + class _CfgNoAux: + conv_modulation = ModConvType.FEATURES + aux_in_features = None + log_aux = False + auxiliary_features = None + + assert prepare_auxiliary_data(_batch_data(), _CfgNoAux()) is None + + +def test_prepare_auxiliary_data_scalar_feature(): + cfg = _Cfg(conv_modulation=ModConvType.FEATURES, aux_in_features=1, log_aux=False) + data = {"acceleration": torch.tensor(4.0)} + auxiliary_data = prepare_auxiliary_data(data, cfg) + assert auxiliary_data is not None + assert auxiliary_data.shape == (1, 1) + + +def test_prepare_auxiliary_data_column_feature(): + cfg = _Cfg(conv_modulation=ModConvType.FEATURES, aux_in_features=1, log_aux=False) + data = {"acceleration": torch.tensor([[4.0], [8.0]])} + auxiliary_data = prepare_auxiliary_data(data, cfg) + assert auxiliary_data is not None + torch.testing.assert_close(auxiliary_data, torch.tensor([[4.0], [8.0]])) + + +def test_prepare_auxiliary_data_invalid_feature_shape_raises(): + cfg = _Cfg(conv_modulation=ModConvType.FEATURES, aux_in_features=1, log_aux=False) + data = {"acceleration": torch.randn(2, 4, 4)} + with pytest.raises(ValueError, match="Expected auxiliary feature"): + prepare_auxiliary_data(data, cfg) + + +def test_prepare_auxiliary_data_explicit_features_override(): + cfg = _Cfg(conv_modulation=ModConvType.FEATURES, aux_in_features=1, log_aux=False) + features = (AuxiliaryFeature("acceleration"),) + auxiliary_data = prepare_auxiliary_data(_batch_data(), cfg, features=features) + assert auxiliary_data is not None + torch.testing.assert_close(auxiliary_data, torch.tensor([[4.0], [8.0]])) diff --git a/tests/tests_nn/modulated_conv_test.py b/tests/tests_nn/modulated_conv_test.py new file mode 100644 index 00000000..c4c9e27f --- /dev/null +++ b/tests/tests_nn/modulated_conv_test.py @@ -0,0 +1,412 @@ +# Copyright 2026 AI for Oncology Research Group. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pytest +import torch + +from direct.nn.conv.conv import Conv2d +from direct.nn.conv.modulated import ( + ModConv2d, + ModConv2dBias, + ModConv3d, + ModConvActivation, + ModConvTranspose2d, + ModConvTranspose3d, + ModConvType, + ModulationParams, + mod_conv2d, + mod_conv3d, + mod_conv_transpose2d, + mod_conv_transpose3d, +) +from direct.nn.didn.didn import DIDN +from direct.nn.mwcnn.mwcnn import MWCNN +from direct.nn.unet.unet_2d import UnetModel2d + +MODULATION_TYPES = [ + ModConvType.NONE, + ModConvType.FEATURES, + ModConvType.PARTIAL_IN, + ModConvType.PARTIAL_OUT, + ModConvType.SUM, +] + + +@pytest.mark.parametrize("modulation", MODULATION_TYPES) +@pytest.mark.parametrize("bias", [ModConv2dBias.PARAM, ModConv2dBias.NONE]) +def test_modconv2d(modulation, bias): + batch, in_ch, out_ch, h, w = 2, 4, 8, 16, 16 + aux_feat = 3 + x = torch.randn(batch, in_ch, h, w) + y = torch.randn(batch, aux_feat) + + kwargs = dict( + in_channels=in_ch, + out_channels=out_ch, + kernel_size=3, + padding=1, + bias=bias, + modulation=modulation, + ) + if modulation != ModConvType.NONE: + kwargs.update(aux_in_features=aux_feat, fc_hidden_features=(16,), fc_groups=1) + if modulation == ModConvType.SUM: + kwargs["num_weights"] = 3 + + model = ModConv2d(**kwargs) + if modulation == ModConvType.NONE: + out = model(x) + else: + out = model(x, y) + + assert out.shape == (batch, out_ch, h, w) + + +@pytest.mark.parametrize("modulation", MODULATION_TYPES) +def test_modconv_transpose2d(modulation): + batch, in_ch, out_ch, h, w = 2, 8, 4, 8, 8 + aux_feat = 3 + x = torch.randn(batch, in_ch, h, w) + y = torch.randn(batch, aux_feat) + + kwargs = dict( + in_channels=in_ch, + out_channels=out_ch, + kernel_size=2, + stride=2, + modulation=modulation, + ) + if modulation != ModConvType.NONE: + kwargs.update(aux_in_features=aux_feat, fc_hidden_features=(16,), fc_groups=1) + if modulation == ModConvType.SUM: + kwargs["num_weights"] = 3 + + model = ModConvTranspose2d(**kwargs) + if modulation == ModConvType.NONE: + out = model(x) + else: + out = model(x, y) + + assert out.shape == (batch, out_ch, h * 2, w * 2) + + +@pytest.mark.parametrize("modulation", [ModConvType.NONE, ModConvType.FEATURES, ModConvType.SUM]) +def test_modconv3d(modulation): + batch, in_ch, out_ch, d, h, w = 2, 4, 8, 4, 8, 8 + aux_feat = 3 + x = torch.randn(batch, in_ch, d, h, w) + y = torch.randn(batch, aux_feat) + + kwargs = dict( + in_channels=in_ch, + out_channels=out_ch, + kernel_size=3, + padding=1, + modulation=modulation, + ) + if modulation != ModConvType.NONE: + kwargs.update(aux_in_features=aux_feat, fc_hidden_features=(16,), fc_groups=1) + if modulation == ModConvType.SUM: + kwargs["num_weights"] = 3 + + model = ModConv3d(**kwargs) + if modulation == ModConvType.NONE: + out = model(x) + else: + out = model(x, y) + + assert out.shape == (batch, out_ch, d, h, w) + + +@pytest.mark.parametrize("modulation", [ModConvType.NONE, ModConvType.FEATURES, ModConvType.SUM]) +def test_modconv_transpose3d(modulation): + batch, in_ch, out_ch, d, h, w = 2, 8, 4, 4, 4, 4 + aux_feat = 3 + x = torch.randn(batch, in_ch, d, h, w) + y = torch.randn(batch, aux_feat) + + kwargs = dict( + in_channels=in_ch, + out_channels=out_ch, + kernel_size=2, + stride=2, + modulation=modulation, + ) + if modulation != ModConvType.NONE: + kwargs.update(aux_in_features=aux_feat, fc_hidden_features=(16,), fc_groups=1) + if modulation == ModConvType.SUM: + kwargs["num_weights"] = 3 + + model = ModConvTranspose3d(**kwargs) + if modulation == ModConvType.NONE: + out = model(x) + else: + out = model(x, y) + + assert out.shape == (batch, out_ch, d * 2, h * 2, w * 2) + + +@pytest.mark.parametrize("modulation", [ModConvType.NONE, ModConvType.FEATURES]) +def test_conv2d_cascade(modulation): + batch, in_ch, out_ch, h, w = 2, 2, 4, 16, 16 + aux_feat = 3 + x = torch.randn(batch, in_ch, h, w) + y = torch.randn(batch, aux_feat) + + kwargs = dict( + in_channels=in_ch, + out_channels=out_ch, + hidden_channels=8, + n_convs=3, + modulation=modulation, + ) + if modulation != ModConvType.NONE: + kwargs.update(aux_in_features=aux_feat, fc_hidden_features=(16,), fc_groups=1) + + model = Conv2d(**kwargs) + if modulation == ModConvType.NONE: + out = model(x) + else: + out = model(x, y) + + assert out.shape == (batch, out_ch, h, w) + + +@pytest.mark.parametrize("modulation", [ModConvType.NONE, ModConvType.FEATURES]) +def test_didn(modulation): + batch, in_ch, out_ch, h, w = 1, 2, 2, 32, 32 + aux_feat = 3 + x = torch.randn(batch, in_ch, h, w) + y = torch.randn(batch, aux_feat) + + kwargs = dict( + in_channels=in_ch, + out_channels=out_ch, + hidden_channels=8, + num_dubs=2, + num_convs_recon=3, + modulation=modulation, + ) + if modulation != ModConvType.NONE: + kwargs.update(aux_in_features=aux_feat, fc_hidden_features=(8,), fc_groups=1) + + model = DIDN(**kwargs) + if modulation == ModConvType.NONE: + out = model(x) + else: + out = model(x, y) + + assert out.shape == (batch, out_ch, h, w) + + +@pytest.mark.parametrize("modulation", [ModConvType.NONE, ModConvType.FEATURES]) +def test_mwcnn(modulation): + batch, in_ch, h, w = 1, 2, 32, 32 + aux_feat = 3 + x = torch.randn(batch, in_ch, h, w) + y = torch.randn(batch, aux_feat) + + kwargs = dict( + input_channels=in_ch, + first_conv_hidden_channels=8, + num_scales=2, + modulation=modulation, + ) + if modulation != ModConvType.NONE: + kwargs.update(aux_in_features=aux_feat, fc_hidden_features=(8,), fc_groups=1) + + model = MWCNN(**kwargs) + if modulation == ModConvType.NONE: + out = model(x) + else: + out = model(x, y) + + assert out.shape == (batch, in_ch, h, w) + + +@pytest.mark.parametrize("modulation", [ModConvType.NONE, ModConvType.FEATURES]) +def test_modconv2d_gradient_flow(modulation): + batch, in_ch, out_ch, h, w = 2, 4, 8, 8, 8 + aux_feat = 3 + x = torch.randn(batch, in_ch, h, w, requires_grad=True) + y = torch.randn(batch, aux_feat, requires_grad=True) + + kwargs = dict( + in_channels=in_ch, + out_channels=out_ch, + kernel_size=3, + padding=1, + modulation=modulation, + ) + if modulation != ModConvType.NONE: + kwargs.update(aux_in_features=aux_feat, fc_hidden_features=(16,), fc_groups=1) + + model = ModConv2d(**kwargs) + if modulation == ModConvType.NONE: + out = model(x) + else: + out = model(x, y) + + loss = out.sum() + loss.backward() + + assert x.grad is not None + if modulation != ModConvType.NONE: + assert y.grad is not None + + +@pytest.mark.parametrize("fc_groups", [1, 2]) +def test_modconv2d_fc_groups(fc_groups): + batch, in_ch, out_ch, h, w = 2, 4, 8, 8, 8 + aux_feat = 3 + x = torch.randn(batch, in_ch, h, w) + y = torch.randn(batch, aux_feat) + + model = ModConv2d( + in_channels=in_ch, + out_channels=out_ch, + kernel_size=3, + padding=1, + modulation=ModConvType.FEATURES, + aux_in_features=aux_feat, + fc_hidden_features=(16,), + fc_groups=fc_groups, + ) + out = model(x, y) + assert out.shape == (batch, out_ch, h, w) + + +def test_modconv2d_learned_bias(): + batch, in_ch, out_ch, h, w = 2, 4, 8, 8, 8 + aux_feat = 3 + x = torch.randn(batch, in_ch, h, w) + y = torch.randn(batch, aux_feat) + + model = ModConv2d( + in_channels=in_ch, + out_channels=out_ch, + kernel_size=3, + padding=1, + modulation=ModConvType.FEATURES, + bias=ModConv2dBias.LEARNED, + aux_in_features=aux_feat, + fc_hidden_features=(16,), + fc_groups=1, + ) + out = model(x, y) + assert out.shape == (batch, out_ch, h, w) + + +@pytest.mark.parametrize( + "modulation", + [ModConvType.FULL, ModConvType.PARTIAL_IN, ModConvType.PARTIAL_OUT], +) +@pytest.mark.parametrize("fc_groups", [1, 2]) +def test_modconv2d_weight_modulation_types(modulation, fc_groups): + batch, in_ch, out_ch, h, w = 2, 4, 8, 16, 16 + aux_feat = 3 + x = torch.randn(batch, in_ch, h, w) + y = torch.randn(batch, aux_feat) + + model = ModConv2d( + in_channels=in_ch, + out_channels=out_ch, + kernel_size=3, + padding=1, + modulation=modulation, + aux_in_features=aux_feat, + fc_hidden_features=(16, 8), + fc_groups=fc_groups, + fc_activation=ModConvActivation.SOFTPLUS, + ) + out = model(x, y) + assert out.shape == (batch, out_ch, h, w) + + +def test_mod_conv_factories(): + params = ModulationParams( + modulation=ModConvType.FEATURES, + aux_in_features=2, + fc_hidden_features=(8,), + fc_activation=ModConvActivation.SOFTPLUS, + ) + x2d = torch.randn(1, 4, 8, 8) + y = torch.randn(1, 2) + x3d = torch.randn(1, 4, 4, 8, 8) + + conv2d = mod_conv2d(4, 8, kernel_size=3, padding=1, modulation_params=params) + assert conv2d(x2d, y).shape == (1, 8, 8, 8) + + conv_t2d = mod_conv_transpose2d(8, 4, kernel_size=2, stride=2, modulation_params=params) + assert conv_t2d(torch.randn(1, 8, 8, 8), y).shape == (1, 4, 16, 16) + + conv3d = mod_conv3d(4, 8, kernel_size=3, padding=1, modulation_params=params) + assert conv3d(x3d, y).shape == (1, 8, 4, 8, 8) + + conv_t3d = mod_conv_transpose3d(8, 4, kernel_size=2, stride=2, modulation_params=params) + assert conv_t3d(torch.randn(1, 8, 4, 8, 8), y).shape == (1, 4, 8, 16, 16) + + +def test_modconv2d_init_validation(): + with pytest.raises(ValueError, match="aux_in_features"): + ModConv2d(4, 8, 3, modulation=ModConvType.FEATURES) + with pytest.raises(ValueError, match="fc_hidden_features"): + ModConv2d(4, 8, 3, modulation=ModConvType.FEATURES, aux_in_features=2) + with pytest.raises(ValueError, match="num_weights"): + ModConv2d( + 4, + 8, + 3, + modulation=ModConvType.SUM, + aux_in_features=2, + fc_hidden_features=(8,), + ) + with pytest.raises(ValueError, match="LEARNED requires modulation"): + ModConv2d(4, 8, 3, bias=ModConv2dBias.LEARNED) + + +def test_unet2d_modulated_forward(): + model = UnetModel2d( + in_channels=2, + out_channels=2, + num_filters=8, + num_pool_layers=2, + dropout_probability=0.0, + modulation=ModConvType.FEATURES, + aux_in_features=2, + fc_hidden_features=(8,), + ) + x = torch.randn(1, 2, 32, 32) + y = torch.randn(1, 2) + out = model(x, y) + assert out.shape == x.shape + + +def test_didn_modulation_params(): + params = ModulationParams( + modulation=ModConvType.FEATURES, + aux_in_features=2, + fc_hidden_features=(8,), + ) + model = DIDN( + in_channels=2, + out_channels=2, + hidden_channels=8, + num_dubs=2, + num_convs_recon=3, + modulation_params=params, + ) + x = torch.randn(1, 2, 32, 32) + y = torch.randn(1, 2) + out = model(x, y) + assert out.shape == x.shape diff --git a/tests/tests_utils/dataset_field_strength_test.py b/tests/tests_utils/dataset_field_strength_test.py new file mode 100644 index 00000000..0a8d39f9 --- /dev/null +++ b/tests/tests_utils/dataset_field_strength_test.py @@ -0,0 +1,50 @@ +# Copyright 2026 AI for Oncology Research Group. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for field-strength parsing from dataset filenames.""" + +import numpy as np +import pytest + +from direct.utils.dataset import maybe_attach_field_strength, parse_field_strength_tesla + + +@pytest.mark.parametrize( + ("filename", "expected"), + [ + ("2025_train_Center005_UIH_15T_umr670_P001_lge_lax_2ch.mat", 1.5), + ("2025_train_Center001_UIH_30T_umr780_P001_cine_sax.mat", 3.0), + ("subject_055T_scan.mat", 0.55), + ("scanner_70T_data.h5", 7.0), + ("prefix_7T_suffix.h5", 7.0), + ("/data/path/Center_15T_file.mat", 1.5), + ("file1000000.h5", None), + ("no_field_token.h5", None), + ("P015_cine.mat", None), # digits without trailing T + ], +) +def test_parse_field_strength_tesla(filename, expected): + assert parse_field_strength_tesla(filename) == expected + + +def test_maybe_attach_field_strength_adds_key(): + sample = {"filename": "2025_train_Center001_UIH_30T_umr780_P001.mat"} + maybe_attach_field_strength(sample) + assert "field_strength" in sample + np.testing.assert_allclose(sample["field_strength"], np.array([3.0])) + + +def test_maybe_attach_field_strength_skips_when_absent(): + sample = {"filename": "file1000000.h5"} + maybe_attach_field_strength(sample) + assert "field_strength" not in sample