diff --git a/.prospector.yml b/.prospector.yml
new file mode 100644
index 00000000..c269d432
--- /dev/null
+++ b/.prospector.yml
@@ -0,0 +1,22 @@
+# Prospector profile for Codacy. CI uses ruff + ty; this file aligns Codacy with those
+# conventions. Numpydoc-style docstrings (Parameters/Returns/References sections) conflict
+# with strict pydocstyle (D203, D412, D413, D417, …), so pydocstyle is disabled here.
+
+pydocstyle:
+ run: false
+
+pylint:
+ disable:
+ - too-many-arguments
+ - too-many-positional-arguments
+ - too-many-locals
+ - too-many-branches
+ - import-outside-toplevel
+ - consider-using-from-import
+ - line-too-long
+ options:
+ max-line-length: 120
+
+mccabe:
+ options:
+ max-complexity: 20
diff --git a/direct/common/subsample.py b/direct/common/subsample.py
index 5df40a81..926fe04c 100644
--- a/direct/common/subsample.py
+++ b/direct/common/subsample.py
@@ -14,7 +14,8 @@
"""DIRECT samplers module.
This module contains classes for creating sub-sampling masks. The masks are used to sample k-space data in MRI
-reconstruction. The masks are created by selecting a subset of samples from the input k-space data."""
+reconstruction. The masks are created by selecting a subset of samples from the input k-space data.
+"""
# Code and comments can be shared with code of FastMRI under the same MIT license:
# https://github.com/facebookresearch/fastMRI/
@@ -37,8 +38,9 @@
from direct.common._gaussian import gaussian_mask_1d, gaussian_mask_2d # pylint: disable=no-name-in-module
from direct.common._poisson import poisson as _poisson # pylint: disable=no-name-in-module
from direct.environment import DIRECT_CACHE_DIR
-from direct.types import DirectEnum, MaskFuncMode, Number, TensorOrNdarray
+from direct.types import DirectEnum, MaskFuncMode, Number, RangeMode, TensorOrNdarray
from direct.utils import str_to_class
+from direct.utils.distributions import triangular_distribution
from direct.utils.io import download_url
# pylint: disable=arguments-differ
@@ -65,6 +67,7 @@
"KtUniformMaskFunc",
"MagicMaskFunc",
"MaskFuncMode",
+ "RangeMode",
"RadialMaskFunc",
"RandomMaskFunc",
"SpiralMaskFunc",
@@ -89,86 +92,104 @@ def temp_seed(rng, seed):
class BaseMaskFunc:
- """:class:`BaseMaskFunc` is the base class to create a sub-sampling mask of a given shape.
-
- Parameters
- ----------
- accelerations : Union[list[Number], tuple[Number, ...]]
- Amount of under-sampling_mask. An acceleration of 4 retains 25% of the k-space, the method is given by
- mask_type. Has to be the same length as center_fractions if uniform_range is not True.
- center_fractions : Union[list[float], tuple[float, ...]], optional
- Fraction of low-frequency columns (float < 1.0) or number of low-frequence columns (integer) to be retained.
- If multiple values are provided, then one of these numbers is chosen uniformly each time. If uniform_range
- is True, then two values should be given. Default: None.
- uniform_range : bool
- If True then an acceleration will be uniformly sampled between the two values. Default: False.
- mode : MaskFuncMode
- Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE.
- If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be
- broadcasted to the shape by expanding other dimensions with 1, if applicable. If MaskFuncMode.DYNAMIC,
- this expects the shape to have more then 3 dimensions, and the mask will be created for each time frame
- along the fourth last dimension. Similarly for MaskFuncMode.MULTISLICE, the mask will be created for each
- slice along the fourth last dimension. Default: MaskFuncMode.STATIC.
+ """Base class to create a sub-sampling mask of a given shape.
+
+ Args:
+ accelerations: Amount of under-sampling. An acceleration of 4 retains 25% of
+ k-space. Must match ``center_fractions`` length when ``range_mode`` is
+ ``RangeMode.DISCRETE``.
+ center_fractions: Fraction of low-frequency columns (float < 1.0) or number of
+ low-frequency columns (int) to retain. If ``range_mode`` is UNIFORM or LINEAR,
+ two values should be given. Defaults to None.
+ range_mode: How accelerations are sampled. ``DISCRETE`` picks from the configured
+ values; ``UNIFORM`` samples uniformly between the two endpoints; ``LINEAR``
+ samples from a triangular distribution biased toward higher acceleration.
+ Defaults to ``RangeMode.UNIFORM``.
+ mode: Mask function mode (``STATIC``, ``DYNAMIC``, or ``MULTISLICE``).
+ Defaults to ``MaskFuncMode.STATIC``.
"""
def __init__(
self,
accelerations: Union[list[Number], tuple[Number, ...]],
center_fractions: Optional[Union[list[float], tuple[float, ...]]] = None,
- uniform_range: bool = True,
+ range_mode: RangeMode = RangeMode.UNIFORM,
mode: MaskFuncMode = MaskFuncMode.STATIC,
) -> None:
- """Inits :class:`BaseMaskFunc`.
-
- Parameters
- ----------
- accelerations : Union[list[Number], tuple[Number, ...]]
- Amount of under-sampling_mask. An acceleration of 4 retains 25% of the k-space, the method is given by
- mask_type. Has to be the same length as center_fractions if uniform_range is not True.
- center_fractions : Union[list[float], tuple[float, ...]], optional
- Fraction of low-frequency columns (float < 1.0) or number of low-frequence columns (integer) to be retained.
- If multiple values are provided, then one of these numbers is chosen uniformly each time. If uniform_range
- is True, then two values should be given. Default: None.
- uniform_range : bool
- If True then an acceleration will be uniformly sampled between the two values. Default: False.
- mode : MaskFuncMode
- Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE.
- If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be
- broadcasted to the shape by expanding other dimensions with 1, if applicable. If MaskFuncMode.DYNAMIC,
- this expects the shape to have more then 3 dimensions, and the mask will be created for each time frame
- along the fourth last dimension. Similarly for MaskFuncMode.MULTISLICE, the mask will be created for each
- slice along the fourth last dimension. Default: MaskFuncMode.STATIC.
+ """Initialize :class:`BaseMaskFunc`.
+
+ Args:
+ accelerations: Amount of under-sampling. An acceleration of 4 retains 25% of
+ k-space. Must match ``center_fractions`` length when ``range_mode`` is
+ ``RangeMode.DISCRETE``.
+ center_fractions: Fraction of low-frequency columns (float < 1.0) or number of
+ low-frequency columns (int) to retain. If ``range_mode`` is UNIFORM or LINEAR,
+ two values should be given. Defaults to None.
+ range_mode: How accelerations are sampled. ``DISCRETE`` picks from the configured
+ values; ``UNIFORM`` samples uniformly between the two endpoints; ``LINEAR``
+ samples from a triangular distribution biased toward higher acceleration.
+ Defaults to ``RangeMode.UNIFORM``.
+ mode: Mask function mode (``STATIC``, ``DYNAMIC``, or ``MULTISLICE``).
+ Defaults to ``MaskFuncMode.STATIC``.
+
+ Raises:
+ ValueError: If ``range_mode`` is UNIFORM or LINEAR and ``center_fractions`` /
+ ``accelerations`` are not length-2, or if their lengths differ for DISCRETE.
"""
if center_fractions is not None:
- if len([center_fractions]) != len([accelerations]):
+ if range_mode in (RangeMode.UNIFORM, RangeMode.LINEAR):
+ if len(center_fractions) != 2 or len(accelerations) != 2:
+ raise ValueError(
+ "When range_mode is UNIFORM or LINEAR, both center_fractions and "
+ f"accelerations must have length two. Got center_fractions={center_fractions} "
+ f"and accelerations={accelerations}."
+ )
+ elif len(center_fractions) != len(accelerations):
raise ValueError(
f"Number of center fractions should match number of accelerations. "
- f"Got {len([center_fractions])} {len([accelerations])}."
+ f"Got {len(center_fractions)} and {len(accelerations)}."
)
self.center_fractions = center_fractions
self.accelerations = accelerations
-
- self.uniform_range = uniform_range
+ self.range_mode = range_mode
self.mode = mode
-
self.rng = np.random.RandomState()
+ self._last_acceleration: Optional[float] = None
+ self._last_center_fraction: Optional[float] = None
- def _choose_index(self) -> int:
- """Picks a random index into ``self.accelerations``.
-
- Raises
- ------
- ValueError
- If no accelerations have been configured.
- NotImplementedError
- If uniform range is requested but not yet implemented.
- """
+ def _draw_acceleration_value(self) -> Number:
if not self.accelerations:
raise ValueError("No accelerations configured for this mask function.")
- if self.uniform_range:
- raise NotImplementedError("Uniform range is not yet implemented.")
- return int(self.rng.randint(0, len(self.accelerations)))
+ if self.range_mode == RangeMode.UNIFORM:
+ accel = [float(value) for value in self.accelerations]
+ return self.rng.uniform(min(accel), max(accel))
+ if self.range_mode == RangeMode.LINEAR:
+ accel = [float(value) for value in self.accelerations]
+ return triangular_distribution(min(accel), max(accel), 1, self.rng)[0]
+ return self.accelerations[int(self.rng.randint(0, len(self.accelerations)))]
+
+ def _draw_acceleration_pair(self) -> tuple[Number, Number]:
+ if self.center_fractions is None:
+ raise ValueError(
+ "_draw_acceleration_pair requires center_fractions; use choose_acceleration_only "
+ "for mask functions that do not configure center fractions."
+ )
+ if self.range_mode == RangeMode.UNIFORM:
+ accel = [float(value) for value in self.accelerations]
+ acceleration = self.rng.uniform(min(accel), max(accel))
+ center_values = [float(value) for value in self.center_fractions]
+ center_fraction = self.rng.uniform(min(center_values), max(center_values))
+ elif self.range_mode == RangeMode.LINEAR:
+ accel = [float(value) for value in self.accelerations]
+ acceleration = triangular_distribution(min(accel), max(accel), 1, self.rng)[0]
+ center_values = [float(value) for value in self.center_fractions]
+ center_fraction = self.rng.uniform(min(center_values), max(center_values))
+ else:
+ choice = int(self.rng.randint(0, len(self.accelerations)))
+ acceleration = self.accelerations[choice]
+ center_fraction = self.center_fractions[choice] # type: ignore[index]
+ return acceleration, center_fraction
def choose_acceleration(self) -> tuple[Number, Number]:
"""Chooses an acceleration and matching center fraction.
@@ -188,12 +209,21 @@ def choose_acceleration(self) -> tuple[Number, Number]:
"choose_acceleration requires center_fractions; use choose_acceleration_only "
"for mask functions that do not configure center fractions."
)
- choice = self._choose_index()
- return self.center_fractions[choice], self.accelerations[choice]
+
+ acceleration, center_fraction = self._draw_acceleration_pair()
+ center_fraction = float(center_fraction)
+ if center_fraction < 1.0:
+ center_fraction = min(1.0 / float(acceleration), center_fraction)
+ self._last_acceleration = float(acceleration)
+ self._last_center_fraction = float(center_fraction)
+ return center_fraction, acceleration
def choose_acceleration_only(self) -> Number:
"""Chooses an acceleration without an associated center fraction."""
- return self.accelerations[self._choose_index()]
+ acceleration = self._draw_acceleration_value()
+ self._last_acceleration = float(acceleration)
+ self._last_center_fraction = min(1.0 / float(acceleration), 1.0)
+ return acceleration
@abstractmethod
def mask_func(self, *args, **kwargs) -> torch.Tensor:
@@ -220,9 +250,9 @@ def _reshape_and_add_coil_axis(self, mask: TensorOrNdarray, shape: Sequence[int]
(nt or num_slices, num_rows, num_cols) if mode is MaskFuncMode.DYNAMIC or
MaskFuncMode.MULTISLICE to be reshaped.
shape : tuple of ints
- Shape of the output array after reshaping. Expects shape to be (\*, num_rows, num_cols, channels) for
- mode MaskFuncMode.STATIC, and (\*, nt or num_slices, num_rows, num_cols, channels) for mode
- MaskFuncMode.DYNAMIC where \* is any number of dimensions.
+ Shape of the output array after reshaping. Expects shape to be (\\*, num_rows, num_cols, channels) for
+ mode MaskFuncMode.STATIC, and (\\*, nt or num_slices, num_rows, num_cols, channels) for mode
+ MaskFuncMode.DYNAMIC where \\* is any number of dimensions.
Returns
-------
@@ -250,7 +280,9 @@ def _reshape_and_add_coil_axis(self, mask: TensorOrNdarray, shape: Sequence[int]
return mask
- def __call__(self, shape: tuple[int, ...], *args, **kwargs) -> torch.Tensor:
+ def __call__(
+ self, shape: tuple[int, ...], *args, **kwargs
+ ) -> Union[torch.Tensor, tuple[torch.Tensor, float, float]]:
"""Calls the mask function.
Parameters
@@ -265,16 +297,36 @@ def __call__(self, shape: tuple[int, ...], *args, **kwargs) -> torch.Tensor:
Returns
-------
- torch.Tensor
- Sampling mask.
+ torch.Tensor or tuple
+ Sampling mask, or ``(mask, acceleration, center_fraction)`` when
+ ``return_acceleration`` is True.
"""
if len(shape) < 3:
raise ValueError("Shape should have 3 or more dimensions.")
if self.mode in [MaskFuncMode.DYNAMIC, MaskFuncMode.MULTISLICE] and len(shape) < 4:
raise ValueError("Shape should have 4 or more dimensions for dynamic or multislice mode.")
+ return_acceleration = kwargs.pop("return_acceleration", False)
+ self._last_acceleration = None
+ self._last_center_fraction = None
+
mask = self.mask_func(shape, *args, **kwargs)
- return mask
+
+ if not return_acceleration:
+ return mask
+
+ if (
+ isinstance(mask, tuple)
+ and len(mask) == 3
+ and isinstance(mask[1], (float, int))
+ and isinstance(mask[2], (float, int))
+ ):
+ return mask
+
+ if self._last_acceleration is None or self._last_center_fraction is None:
+ return mask
+
+ return mask, self._last_acceleration, self._last_center_fraction
class CartesianVerticalMaskFunc(BaseMaskFunc):
@@ -290,8 +342,8 @@ class CartesianVerticalMaskFunc(BaseMaskFunc):
Amount of under-sampling.
center_fractions : Union[list[float], tuple[float, ...]]
Fraction of low-frequency columns (float < 1.0) or number of low-frequence columns (integer) to be retained.
- uniform_range : bool, optional
- If True then an acceleration will be uniformly sampled between the two values, by default False.
+ range_mode : RangeMode, optional
+ How accelerations are sampled. Defaults to RangeMode.DISCRETE.
mode : MaskFuncMode
Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE.
If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be
@@ -305,7 +357,7 @@ def __init__(
self,
accelerations: Union[list[Number], tuple[Number, ...]],
center_fractions: Union[list[float], tuple[float, ...]],
- uniform_range: bool = False,
+ range_mode: RangeMode = RangeMode.DISCRETE,
mode: MaskFuncMode = MaskFuncMode.STATIC,
) -> None:
"""Inits :class:`CartesianVerticalMaskFunc`.
@@ -316,8 +368,8 @@ def __init__(
Amount of under-sampling.
center_fractions : Union[list[float], tuple[float, ...]]
Fraction of low-frequency columns (float < 1.0) or number of low-frequence columns (integer) to be retained.
- uniform_range : bool, optional
- If True then an acceleration will be uniformly sampled between the two values, by default False.
+ range_mode : RangeMode, optional
+ How accelerations are sampled. Defaults to RangeMode.DISCRETE.
mode : MaskFuncMode
Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE.
If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be
@@ -329,7 +381,7 @@ def __init__(
super().__init__(
accelerations=accelerations,
center_fractions=center_fractions,
- uniform_range=uniform_range,
+ range_mode=range_mode,
mode=mode,
)
@@ -420,8 +472,8 @@ class RandomMaskFunc(CartesianVerticalMaskFunc):
center_fractions : Union[list[Number], tuple[Number, ...]]
If < 1.0 this corresponds to the fraction of low-frequency columns to be retained.
If >= 1 (integer) this corresponds to the exact number of low-frequency columns to be retained.
- uniform_range : bool, optional
- If True then an acceleration will be uniformly sampled between the two values, by default False.
+ range_mode : RangeMode, optional
+ How accelerations are sampled. Defaults to RangeMode.DISCRETE.
mode : MaskFuncMode
Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE.
If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be
@@ -435,7 +487,7 @@ def __init__(
self,
accelerations: Union[list[Number], tuple[Number, ...]],
center_fractions: Union[list[Number], tuple[Number, ...]],
- uniform_range: bool = False,
+ range_mode: RangeMode = RangeMode.DISCRETE,
mode: MaskFuncMode = MaskFuncMode.STATIC,
) -> None:
"""Inits :class:`RandomMaskFunc`.
@@ -447,8 +499,8 @@ def __init__(
center_fractions : Union[list[Number], tuple[Number, ...]]
If < 1.0 this corresponds to the fraction of low-frequency columns to be retained.
If >= 1 (integer) this corresponds to the exact number of low-frequency columns to be retained.
- uniform_range : bool, optional
- If True then an acceleration will be uniformly sampled between the two values, by default False.
+ range_mode : RangeMode, optional
+ How accelerations are sampled. Defaults to RangeMode.DISCRETE.
mode : MaskFuncMode
Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE.
If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be
@@ -460,7 +512,7 @@ def __init__(
super().__init__(
accelerations=accelerations,
center_fractions=center_fractions,
- uniform_range=uniform_range,
+ range_mode=range_mode,
mode=mode,
)
@@ -483,7 +535,6 @@ def mask_func(
Seed for the random number generator. Setting the seed ensures the same mask is generated
each time for the same shape. Default: None.
-
Returns
-------
mask : torch.Tensor
@@ -544,8 +595,8 @@ class FastMRIRandomMaskFunc(RandomMaskFunc):
Amount of under-sampling.
center_fractions : Union[list[float], tuple[float, ...]]
Fraction of low-frequency columns (float < 1.0) to be retained.
- uniform_range : bool, optional
- If True then an acceleration will be uniformly sampled between the two values, by default False.
+ range_mode : RangeMode, optional
+ How accelerations are sampled. Defaults to RangeMode.DISCRETE.
mode : MaskFuncMode
Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE.
If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be
@@ -559,7 +610,7 @@ def __init__(
self,
accelerations: Union[list[Number], tuple[Number, ...]],
center_fractions: Union[list[float], tuple[float, ...]],
- uniform_range: bool = False,
+ range_mode: RangeMode = RangeMode.DISCRETE,
mode: MaskFuncMode = MaskFuncMode.STATIC,
) -> None:
"""Inits :class:`FastMRIRandomMaskFunc`.
@@ -570,8 +621,8 @@ def __init__(
Amount of under-sampling.
center_fractions : Union[list[float], tuple[float, ...]]
Fraction of low-frequency columns (float < 1.0) to be retained.
- uniform_range : bool, optional
- If True then an acceleration will be uniformly sampled between the two values, by default False.
+ range_mode : RangeMode, optional
+ How accelerations are sampled. Defaults to RangeMode.DISCRETE.
mode : MaskFuncMode
Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE.
If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be
@@ -588,7 +639,7 @@ def __init__(
super().__init__(
accelerations=accelerations,
center_fractions=center_fractions,
- uniform_range=uniform_range,
+ range_mode=range_mode,
mode=mode,
)
@@ -603,11 +654,11 @@ class CartesianRandomMaskFunc(RandomMaskFunc):
----------
accelerations : Union[list[Number], tuple[Number, ...]]
Amount of under-sampling_mask. An acceleration of 4 retains 25% of the k-space, the method is given by
- mask_type. Has to be the same length as center_fractions if uniform_range is not True.
+ mask_type. Has to be the same length as center_fractions if range_mode is RangeMode.DISCRETE.
center_fractions : Union[list[int], tuple[int, ...]]
Number of low-frequence (center) columns to be retained.
- uniform_range : bool
- If True then an acceleration will be uniformly sampled between the two values. Default: False.
+ range_mode : RangeMode
+ How accelerations are sampled. Defaults to RangeMode.DISCRETE.
mode : MaskFuncMode
Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE.
If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be
@@ -621,7 +672,7 @@ def __init__(
self,
accelerations: Union[list[Number], tuple[Number, ...]],
center_fractions: Union[list[int], tuple[int, ...]],
- uniform_range: bool = False,
+ range_mode: RangeMode = RangeMode.DISCRETE,
mode: MaskFuncMode = MaskFuncMode.STATIC,
) -> None:
"""Inits :class:`CartesianRandomMaskFunc`.
@@ -630,11 +681,11 @@ def __init__(
----------
accelerations : Union[list[Number], tuple[Number, ...]]
Amount of under-sampling_mask. An acceleration of 4 retains 25% of the k-space, the method is given by
- mask_type. Has to be the same length as center_fractions if uniform_range is not True.
+ mask_type. Has to be the same length as center_fractions if range_mode is RangeMode.DISCRETE.
center_fractions : Union[list[int], tuple[int, ...]]
Number of low-frequence (center) columns to be retained.
- uniform_range : bool
- If True then an acceleration will be uniformly sampled between the two values. Default: False.
+ range_mode : RangeMode
+ How accelerations are sampled. Defaults to RangeMode.DISCRETE.
mode : MaskFuncMode
Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE.
If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be
@@ -651,7 +702,7 @@ def __init__(
super().__init__(
accelerations=accelerations,
center_fractions=list(center_fractions),
- uniform_range=uniform_range,
+ range_mode=range_mode,
mode=mode,
)
@@ -683,8 +734,8 @@ class EquispacedMaskFunc(CartesianVerticalMaskFunc):
center_fractions : Union[list[Number], tuple[Number, ...]]
If < 1.0 this corresponds to the fraction of low-frequency columns to be retained.
If >= 1 (integer) this corresponds to the exact number of low-frequency columns to be retained.
- uniform_range : bool, optional
- If True then an acceleration will be uniformly sampled between the two values, by default False.
+ range_mode : RangeMode, optional
+ How accelerations are sampled. Defaults to RangeMode.DISCRETE.
mode : MaskFuncMode
Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE.
If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be
@@ -698,7 +749,7 @@ def __init__(
self,
accelerations: Union[list[Number], tuple[Number, ...]],
center_fractions: Union[list[Number], tuple[Number, ...]],
- uniform_range: bool = False,
+ range_mode: RangeMode = RangeMode.DISCRETE,
mode: MaskFuncMode = MaskFuncMode.STATIC,
) -> None:
"""Inits :class:`EquispacedMaskFunc`.
@@ -710,8 +761,8 @@ def __init__(
center_fractions : Union[list[Number], tuple[Number, ...]]
If < 1.0 this corresponds to the fraction of low-frequency columns to be retained.
If >= 1 (integer) this corresponds to the exact number of low-frequency columns to be retained.
- uniform_range : bool, optional
- If True then an acceleration will be uniformly sampled between the two values, by default False.
+ range_mode : RangeMode, optional
+ How accelerations are sampled. Defaults to RangeMode.DISCRETE.
mode : MaskFuncMode
Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE.
If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be
@@ -723,7 +774,7 @@ def __init__(
super().__init__(
accelerations=accelerations,
center_fractions=center_fractions,
- uniform_range=uniform_range,
+ range_mode=range_mode,
mode=mode,
)
@@ -771,12 +822,17 @@ def mask_func(
if return_acs:
return self._reshape_and_add_coil_axis(self._broadcast_mask(mask, num_rows), shape)
+ # When the ACS already covers the target acceleration, skip equispaced lines.
+ if num_low_freqs - num_cols // acceleration >= 0:
+ return self._reshape_and_add_coil_axis(self._broadcast_mask(mask, num_rows), shape)
+
# determine acceleration rate by adjusting for the number of low frequencies
adjusted_accel = (acceleration * (num_low_freqs - num_cols)) / (num_low_freqs * acceleration - num_cols)
+ offset_high = max(1, round(adjusted_accel))
mask = mask.reshape(num_slc_or_time, -1) # In case mode != MaskFuncMode.STATIC:
for i in range(num_slc_or_time):
- offset = self.rng.randint(0, round(adjusted_accel))
+ offset = self.rng.randint(0, offset_high)
accel_samples = np.arange(offset, num_cols - 1, adjusted_accel)
accel_samples = np.around(accel_samples).astype(np.uint)
mask[i, accel_samples] = True
@@ -809,8 +865,8 @@ class FastMRIEquispacedMaskFunc(EquispacedMaskFunc):
Amount of under-sampling.
center_fractions : Union[list[float], tuple[float, ...]]
Fraction (< 1.0) of low-frequency columns to be retained.
- uniform_range : bool, optional
- If True then an acceleration will be uniformly sampled between the two values, by default False.
+ range_mode : RangeMode, optional
+ How accelerations are sampled. Defaults to RangeMode.DISCRETE.
mode : MaskFuncMode
Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE.
If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be
@@ -824,7 +880,7 @@ def __init__(
self,
accelerations: Union[list[Number], tuple[Number, ...]],
center_fractions: Union[list[float], tuple[float, ...]],
- uniform_range: bool = False,
+ range_mode: RangeMode = RangeMode.DISCRETE,
mode: MaskFuncMode = MaskFuncMode.STATIC,
) -> None:
"""Inits :class:`FastMRIEquispacedMaskFunc`.
@@ -835,8 +891,8 @@ def __init__(
Amount of under-sampling.
center_fractions : Union[list[float], tuple[float, ...]]
Fraction (< 1.0) of low-frequency columns to be retained.
- uniform_range : bool, optional
- If True then an acceleration will be uniformly sampled between the two values, by default False.
+ range_mode : RangeMode, optional
+ How accelerations are sampled. Defaults to RangeMode.DISCRETE.
mode : MaskFuncMode
Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE.
If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be
@@ -853,7 +909,7 @@ def __init__(
super().__init__(
accelerations=accelerations,
center_fractions=center_fractions,
- uniform_range=uniform_range,
+ range_mode=range_mode,
mode=mode,
)
@@ -868,11 +924,11 @@ class CartesianEquispacedMaskFunc(EquispacedMaskFunc):
----------
accelerations : Union[list[Number], tuple[Number, ...]]
Amount of under-sampling_mask. An acceleration of 4 retains 25% of the k-space, the method is given by
- mask_type. Has to be the same length as center_fractions if uniform_range is not True.
+ mask_type. Has to be the same length as center_fractions if range_mode is RangeMode.DISCRETE.
center_fractions : Union[list[int], tuple[int, ...]]
Number of low-frequence (center) columns to be retained.
- uniform_range : bool
- If True then an acceleration will be uniformly sampled between the two values. Default: False.
+ range_mode : RangeMode
+ How accelerations are sampled. Defaults to RangeMode.DISCRETE.
mode : MaskFuncMode
Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE.
If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be
@@ -886,7 +942,7 @@ def __init__(
self,
accelerations: Union[list[Number], tuple[Number, ...]],
center_fractions: Union[list[int], tuple[int, ...]],
- uniform_range: bool = False,
+ range_mode: RangeMode = RangeMode.DISCRETE,
mode: MaskFuncMode = MaskFuncMode.STATIC,
) -> None:
"""Inits :class:`CartesianEquispacedMaskFunc`.
@@ -895,11 +951,11 @@ def __init__(
----------
accelerations : Union[list[Number], tuple[Number, ...]]
Amount of under-sampling_mask. An acceleration of 4 retains 25% of the k-space, the method is given by
- mask_type. Has to be the same length as center_fractions if uniform_range is not True.
+ mask_type. Has to be the same length as center_fractions if range_mode is RangeMode.DISCRETE.
center_fractions : Union[list[int], tuple[int, ...]]
Number of low-frequence (center) columns to be retained.
- uniform_range : bool
- If True then an acceleration will be uniformly sampled between the two values. Default: False.
+ range_mode : RangeMode
+ How accelerations are sampled. Defaults to RangeMode.DISCRETE.
mode : MaskFuncMode
Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE.
If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be
@@ -916,7 +972,7 @@ def __init__(
super().__init__(
accelerations=accelerations,
center_fractions=list(center_fractions),
- uniform_range=uniform_range,
+ range_mode=range_mode,
mode=mode,
)
@@ -937,12 +993,12 @@ class MagicMaskFunc(CartesianVerticalMaskFunc):
----------
accelerations : Union[list[Number], tuple[Number, ...]]
Amount of under-sampling_mask. An acceleration of 4 retains 25% of the k-space, the method is given by
- mask_type. Has to be the same length as center_fractions if uniform_range is not True.
+ mask_type. Has to be the same length as center_fractions if range_mode is RangeMode.DISCRETE.
center_fractions : Union[list[Number], tuple[Number, ...]]
If < 1.0 this corresponds to the fraction of low-frequency columns to be retained.
If >= 1 (integer) this corresponds to the exact number of low-frequency columns to be retained.
- uniform_range : bool
- If True then an acceleration will be uniformly sampled between the two values. Default: False.
+ range_mode : RangeMode
+ How accelerations are sampled. Defaults to RangeMode.DISCRETE.
mode : MaskFuncMode
Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE.
If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be
@@ -956,7 +1012,7 @@ def __init__(
self,
accelerations: Union[list[Number], tuple[Number, ...]],
center_fractions: Union[list[Number], tuple[Number, ...]],
- uniform_range: bool = False,
+ range_mode: RangeMode = RangeMode.DISCRETE,
mode: MaskFuncMode = MaskFuncMode.STATIC,
) -> None:
"""Inits :class:`MagicMaskFunc`.
@@ -965,12 +1021,12 @@ def __init__(
----------
accelerations : Union[list[Number], tuple[Number, ...]]
Amount of under-sampling_mask. An acceleration of 4 retains 25% of the k-space, the method is given by
- mask_type. Has to be the same length as center_fractions if uniform_range is not True.
+ mask_type. Has to be the same length as center_fractions if range_mode is RangeMode.DISCRETE.
center_fractions : Union[list[Number], tuple[Number, ...]]
If < 1.0 this corresponds to the fraction of low-frequency columns to be retained.
If >= 1.0 this corresponds to the exact number of low-frequency columns to be retained.
- uniform_range : bool
- If True then an acceleration will be uniformly sampled between the two values. Default: False.
+ range_mode : RangeMode
+ How accelerations are sampled. Defaults to RangeMode.DISCRETE.
mode : MaskFuncMode
Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE.
If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be
@@ -982,7 +1038,7 @@ def __init__(
super().__init__(
accelerations=accelerations,
center_fractions=center_fractions,
- uniform_range=uniform_range,
+ range_mode=range_mode,
mode=mode,
)
@@ -1044,6 +1100,11 @@ def mask_func(
adjusted_acceleration = int(round(num_cols / adjusted_target_cols_to_sample))
acs_mask = acs_mask.reshape(num_slc_or_time, -1) # In case mode != MaskFuncMode.STATIC:
+
+ if adjusted_target_cols_to_sample <= 0:
+ mask = acs_mask.squeeze()
+ return self._reshape_and_add_coil_axis(self._broadcast_mask(mask, num_rows), shape)
+
mask = []
for i in range(num_slc_or_time):
offset = self.rng.randint(0, high=adjusted_acceleration)
@@ -1087,11 +1148,11 @@ class FastMRIMagicMaskFunc(MagicMaskFunc):
----------
accelerations : Union[list[Number], tuple[Number, ...]]
Amount of under-sampling_mask. An acceleration of 4 retains 25% of the k-space, the method is given by
- mask_type. Has to be the same length as center_fractions if uniform_range is not True.
+ mask_type. Has to be the same length as center_fractions if range_mode is RangeMode.DISCRETE.
center_fractions : Union[list[float], tuple[float, ...]]
Fraction (< 1.0) of low-frequency columns to be retained.
- uniform_range : bool
- If True then an acceleration will be uniformly sampled between the two values. Default: False.
+ range_mode : RangeMode
+ How accelerations are sampled. Defaults to RangeMode.DISCRETE.
mode : MaskFuncMode
Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE.
If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be
@@ -1105,7 +1166,7 @@ def __init__(
self,
accelerations: Union[list[Number], tuple[Number, ...]],
center_fractions: Union[list[float], tuple[float, ...]],
- uniform_range: bool = False,
+ range_mode: RangeMode = RangeMode.DISCRETE,
mode: MaskFuncMode = MaskFuncMode.STATIC,
) -> None:
"""Inits :class:`FastMRIMagicMaskFunc`.
@@ -1114,11 +1175,11 @@ def __init__(
----------
accelerations : Union[list[Number], tuple[Number, ...]]
Amount of under-sampling_mask. An acceleration of 4 retains 25% of the k-space, the method is given by
- mask_type. Has to be the same length as center_fractions if uniform_range is not True.
+ mask_type. Has to be the same length as center_fractions if range_mode is RangeMode.DISCRETE.
center_fractions : Union[list[float], tuple[float, ...]]
Fraction (< 1.0) of low-frequency columns to be retained.
- uniform_range : bool
- If True then an acceleration will be uniformly sampled between the two values. Default: False.
+ range_mode : RangeMode
+ How accelerations are sampled. Defaults to RangeMode.DISCRETE.
mode : MaskFuncMode
Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE.
If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be
@@ -1136,7 +1197,7 @@ def __init__(
super().__init__(
accelerations=accelerations,
center_fractions=center_fractions,
- uniform_range=uniform_range,
+ range_mode=range_mode,
mode=mode,
)
@@ -1156,11 +1217,11 @@ class CartesianMagicMaskFunc(MagicMaskFunc):
----------
accelerations : Union[list[Number], tuple[Number, ...]]
Amount of under-sampling_mask. An acceleration of 4 retains 25% of the k-space, the method is given by
- mask_type. Has to be the same length as center_fractions if uniform_range is not True.
+ mask_type. Has to be the same length as center_fractions if range_mode is RangeMode.DISCRETE.
center_fractions : Union[list[int], tuple[int, ...]]
Number of low-frequence (center) columns to be retained.
- uniform_range : bool
- If True then an acceleration will be uniformly sampled between the two values. Default: False.
+ range_mode : RangeMode
+ How accelerations are sampled. Defaults to RangeMode.DISCRETE.
mode : MaskFuncMode
Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE.
If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be
@@ -1174,7 +1235,7 @@ def __init__(
self,
accelerations: Union[list[Number], tuple[Number, ...]],
center_fractions: Union[list[int], tuple[int, ...]],
- uniform_range: bool = False,
+ range_mode: RangeMode = RangeMode.DISCRETE,
mode: MaskFuncMode = MaskFuncMode.STATIC,
) -> None:
"""Inits :class:`CartesianMagicMaskFunc`.
@@ -1183,11 +1244,11 @@ def __init__(
----------
accelerations : Union[list[Number], tuple[Number, ...]]
Amount of under-sampling_mask. An acceleration of 4 retains 25% of the k-space, the method is given by
- mask_type. Has to be the same length as center_fractions if uniform_range is not True.
+ mask_type. Has to be the same length as center_fractions if range_mode is RangeMode.DISCRETE.
center_fractions : Union[list[int], tuple[int, ...]]
Number of low-frequence (center) columns to be retained.
- uniform_range : bool
- If True then an acceleration will be uniformly sampled between the two values. Default: False.
+ range_mode : RangeMode
+ How accelerations are sampled. Defaults to RangeMode.DISCRETE.
mode : MaskFuncMode
Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE.
If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be
@@ -1204,7 +1265,7 @@ def __init__(
super().__init__(
accelerations=accelerations,
center_fractions=list(center_fractions),
- uniform_range=uniform_range,
+ range_mode=range_mode,
mode=mode,
)
@@ -1225,7 +1286,7 @@ class CalgaryCampinasMaskFunc(BaseMaskFunc):
----------
accelerations : Union[list[int], tuple[int, ...]]
Amount of under-sampling_mask. An acceleration of 4 retains 25% of the k-space, the method is given by
- mask_type. Has to be the same length as center_fractions if uniform_range is not True.
+ mask_type. Has to be the same length as center_fractions if range_mode is RangeMode.DISCRETE.
Raises
------
@@ -1263,7 +1324,7 @@ def __init__(self, accelerations: Union[list[int], tuple[int, ...]], **kwargs) -
If the acceleration is not 5 or 10.
"""
- super().__init__(accelerations=list(accelerations), uniform_range=False)
+ super().__init__(accelerations=list(accelerations), range_mode=RangeMode.DISCRETE)
if not all(_ in [5, 10] for _ in accelerations):
raise ValueError("CalgaryCampinas only provide 5x and 10x acceleration masks.")
@@ -1403,8 +1464,8 @@ class CIRCUSMaskFunc(BaseMaskFunc):
center_fractions : Union[list[float], tuple[float, ...]], optional
Fraction (< 1.0) of low-frequency samples to be retained. If None, it will calculate the acs mask based on the
maximum masked disk in the generated mask (with a tolerance).Default: None.
- uniform_range : bool
- If True then an acceleration will be uniformly sampled between the two values. Default: False.
+ range_mode : RangeMode
+ How accelerations are sampled. Defaults to RangeMode.DISCRETE.
mode : MaskFuncMode
Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE.
If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be
@@ -1426,7 +1487,7 @@ def __init__(
subsampling_scheme: CIRCUSSamplingMode,
accelerations: Union[list[Number], tuple[Number, ...]],
center_fractions: Optional[Union[list[float], tuple[float, ...]]] = None,
- uniform_range: bool = False,
+ range_mode: RangeMode = RangeMode.DISCRETE,
mode: MaskFuncMode = MaskFuncMode.STATIC,
) -> None:
"""Inits :class:`CIRCUSMaskFunc`.
@@ -1441,8 +1502,8 @@ def __init__(
center_fractions : Union[list[float], tuple[float, ...]], optional
Fraction (< 1.0) of low-frequency samples to be retained. If None, it will calculate the acs mask
based on the maximum masked disk in the generated mask (with a tolerance).Default: None.
- uniform_range : bool
- If True then an acceleration will be uniformly sampled between the two values. Default: False.
+ range_mode : RangeMode
+ How accelerations are sampled. Defaults to RangeMode.DISCRETE.
mode : MaskFuncMode
Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE.
If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be
@@ -1458,11 +1519,14 @@ def __init__(
"""
super().__init__(
accelerations=accelerations,
- center_fractions=center_fractions if center_fractions else tuple(0 for _ in range(len(accelerations))),
- uniform_range=uniform_range,
+ center_fractions=(center_fractions if center_fractions else tuple(0 for _ in range(len(accelerations)))),
+ range_mode=range_mode,
mode=mode,
)
- if subsampling_scheme not in [CIRCUSSamplingMode.CIRCUS_RADIAL, CIRCUSSamplingMode.CIRCUS_SPIRAL]:
+ if subsampling_scheme not in [
+ CIRCUSSamplingMode.CIRCUS_RADIAL,
+ CIRCUSSamplingMode.CIRCUS_SPIRAL,
+ ]:
raise NotImplementedError(
f"Currently CIRCUSMaskFunc is only implemented for 'circus-radial' or 'circus-spiral' "
f"as a subsampling_scheme. Got subsampling_scheme={subsampling_scheme}."
@@ -1739,8 +1803,8 @@ class RadialMaskFunc(CIRCUSMaskFunc):
center_fractions : Union[list[float], tuple[float, ...]], optional
Fraction (< 1.0) of low-frequency samples to be retained. If None, it will calculate the acs mask
based on the maximum masked disk in the generated mask (with a tolerance).Default: None.
- uniform_range : bool
- If True then an acceleration will be uniformly sampled between the two values. Default: False.
+ range_mode : RangeMode
+ How accelerations are sampled. Defaults to RangeMode.DISCRETE.
mode : MaskFuncMode
Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE.
If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be
@@ -1754,7 +1818,7 @@ def __init__(
self,
accelerations: Union[list[Number], tuple[Number, ...]],
center_fractions: Optional[Union[list[float], tuple[float, ...]]] = None,
- uniform_range: bool = False,
+ range_mode: RangeMode = RangeMode.DISCRETE,
mode: MaskFuncMode = MaskFuncMode.STATIC,
) -> None:
"""Inits :class:`RadialMaskFunc`.
@@ -1766,8 +1830,8 @@ def __init__(
center_fractions : Union[list[float], tuple[float, ...]], optional
Fraction (< 1.0) of low-frequency samples to be retained. If None, it will calculate the acs mask
based on the maximum masked disk in the generated mask (with a tolerance).Default: None.
- uniform_range : bool
- If True then an acceleration will be uniformly sampled between the two values. Default: False.
+ range_mode : RangeMode
+ How accelerations are sampled. Defaults to RangeMode.DISCRETE.
mode : MaskFuncMode
Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE.
If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be
@@ -1780,7 +1844,7 @@ def __init__(
accelerations=accelerations,
center_fractions=center_fractions,
subsampling_scheme=CIRCUSSamplingMode.CIRCUS_RADIAL,
- uniform_range=uniform_range,
+ range_mode=range_mode,
mode=mode,
)
@@ -1795,8 +1859,8 @@ class SpiralMaskFunc(CIRCUSMaskFunc):
center_fractions : Union[list[float], tuple[float, ...]], optional
Fraction (< 1.0) of low-frequency samples to be retained. If None, it will calculate the acs mask
based on the maximum masked disk in the generated mask (with a tolerance).Default: None.
- uniform_range : bool
- If True then an acceleration will be uniformly sampled between the two values. Default: False.
+ range_mode : RangeMode
+ How accelerations are sampled. Defaults to RangeMode.DISCRETE.
mode : MaskFuncMode
Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE.
If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be
@@ -1810,7 +1874,7 @@ def __init__(
self,
accelerations: Union[list[Number], tuple[Number, ...]],
center_fractions: Optional[Union[list[float], tuple[float, ...]]] = None,
- uniform_range: bool = False,
+ range_mode: RangeMode = RangeMode.DISCRETE,
mode: MaskFuncMode = MaskFuncMode.STATIC,
) -> None:
"""Inits :class:`SpiralMaskFunc`.
@@ -1822,8 +1886,8 @@ def __init__(
center_fractions : Union[list[float], tuple[float, ...]], optional
Fraction (< 1.0) of low-frequency samples to be retained. If None, it will calculate the acs mask
based on the maximum masked disk in the generated mask (with a tolerance).Default: None.
- uniform_range : bool
- If True then an acceleration will be uniformly sampled between the two values. Default: False.
+ range_mode : RangeMode
+ How accelerations are sampled. Defaults to RangeMode.DISCRETE.
mode : MaskFuncMode
Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE.
If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be
@@ -1836,7 +1900,7 @@ def __init__(
accelerations=accelerations,
center_fractions=center_fractions,
subsampling_scheme=CIRCUSSamplingMode.CIRCUS_SPIRAL,
- uniform_range=uniform_range,
+ range_mode=range_mode,
mode=mode,
)
@@ -1928,7 +1992,7 @@ def __init__(
super().__init__(
accelerations=accelerations,
center_fractions=center_fractions,
- uniform_range=False,
+ range_mode=RangeMode.DISCRETE,
mode=mode,
)
self.crop_corner = crop_corner
@@ -1980,7 +2044,15 @@ def mask_func(
mask = []
for _ in range(num_slc_or_time):
- mask.append(self.poisson(num_rows, num_cols, center_fraction, acceleration, self.rng.randint(100_000)))
+ mask.append(
+ self.poisson(
+ num_rows,
+ num_cols,
+ center_fraction,
+ acceleration,
+ self.rng.randint(100_000),
+ )
+ )
mask = np.stack(mask, axis=0).squeeze()
return self._reshape_and_add_coil_axis(mask, shape)
@@ -2035,7 +2107,15 @@ def poisson(
mask = np.zeros((num_rows, num_cols), dtype=int)
- _poisson(num_rows, num_cols, self.max_attempts or 10, mask, radius_x, radius_y, seed)
+ _poisson(
+ num_rows,
+ num_cols,
+ self.max_attempts or 10,
+ mask,
+ radius_x,
+ radius_y,
+ seed,
+ )
mask = mask | centered_disk_mask((num_rows, num_cols), center_fraction)
@@ -2070,8 +2150,8 @@ class Gaussian1DMaskFunc(CartesianVerticalMaskFunc):
Amount of under-sampling.
center_fractions : Union[list[float], tuple[float, ...]]
Fraction of low-frequency columns (float < 1.0) to be retained.
- uniform_range : bool, optional
- If True then an acceleration will be uniformly sampled between the two values, by default False.
+ range_mode : RangeMode, optional
+ How accelerations are sampled. Defaults to RangeMode.DISCRETE.
mode : MaskFuncMode
Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE.
If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be
@@ -2085,7 +2165,7 @@ def __init__(
self,
accelerations: Union[list[Number], tuple[Number, ...]],
center_fractions: Union[list[float], tuple[float, ...]],
- uniform_range: bool = False,
+ range_mode: RangeMode = RangeMode.DISCRETE,
mode: MaskFuncMode = MaskFuncMode.STATIC,
) -> None:
"""Inits :class:`Gaussian1DMaskFunc`.
@@ -2096,8 +2176,8 @@ def __init__(
Amount of under-sampling.
center_fractions : Union[list[float], tuple[float, ...]]
Fraction of low-frequency columns (float < 1.0) to be retained.
- uniform_range : bool, optional
- If True then an acceleration will be uniformly sampled between the two values, by default False.
+ range_mode : RangeMode, optional
+ How accelerations are sampled. Defaults to RangeMode.DISCRETE.
mode : MaskFuncMode
Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE.
If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be
@@ -2109,7 +2189,7 @@ def __init__(
super().__init__(
accelerations=accelerations,
center_fractions=center_fractions,
- uniform_range=uniform_range,
+ range_mode=range_mode,
mode=mode,
)
@@ -2195,8 +2275,8 @@ class Gaussian2DMaskFunc(BaseMaskFunc):
Amount of under-sampling.
center_fractions : Union[list[float], tuple[float, ...]]
Fraction of low-frequency samples (float < 1.0) to be retained.
- uniform_range : bool, optional
- If True then an acceleration will be uniformly sampled between the two values, by default False.
+ range_mode : RangeMode, optional
+ How accelerations are sampled. Defaults to RangeMode.DISCRETE.
mode : MaskFuncMode
Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE.
If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be
@@ -2210,7 +2290,7 @@ def __init__(
self,
accelerations: Union[list[Number], tuple[Number, ...]],
center_fractions: Union[list[float], tuple[float, ...]],
- uniform_range: bool = False,
+ range_mode: RangeMode = RangeMode.DISCRETE,
mode: MaskFuncMode = MaskFuncMode.STATIC,
) -> None:
"""Inits :class:`Gaussian2DMaskFunc`.
@@ -2221,8 +2301,8 @@ def __init__(
Amount of under-sampling.
center_fractions : Union[list[float], tuple[float, ...]]
Fraction of low-frequency samples (float < 1.0) to be retained.
- uniform_range : bool, optional
- If True then an acceleration will be uniformly sampled between the two values, by default False.
+ range_mode : RangeMode, optional
+ How accelerations are sampled. Defaults to RangeMode.DISCRETE.
mode : MaskFuncMode
Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE.
If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be
@@ -2234,7 +2314,7 @@ def __init__(
super().__init__(
accelerations=accelerations,
center_fractions=center_fractions,
- uniform_range=uniform_range,
+ range_mode=range_mode,
mode=mode,
)
@@ -2319,15 +2399,15 @@ class KtBaseMaskFunc(BaseMaskFunc):
Amount of under-sampling.
center_fractions : Union[list[float], tuple[float, ...]]
Fraction of low-frequency columns (float < 1.0) or number of low-frequence columns (integer) to be retained.
- uniform_range : bool, optional
- If True then an acceleration will be uniformly sampled between the two values, by default False.
+ range_mode : RangeMode, optional
+ How accelerations are sampled. Defaults to RangeMode.DISCRETE.
"""
def __init__(
self,
accelerations: Union[list[Number], tuple[Number, ...]],
center_fractions: Union[list[float], tuple[float, ...]],
- uniform_range: bool = False,
+ range_mode: RangeMode = RangeMode.DISCRETE,
) -> None:
"""Inits :class:`KtBaseMaskFunc`.
@@ -2337,13 +2417,13 @@ def __init__(
Amount of under-sampling.
center_fractions : Union[list[float], tuple[float, ...]]
Fraction of low-frequency columns (float < 1.0) or number of low-frequence columns (integer) to be retained.
- uniform_range : bool, optional
- If True then an acceleration will be uniformly sampled between the two values, by default False.
+ range_mode : RangeMode, optional
+ How accelerations are sampled. Defaults to RangeMode.DISCRETE.
"""
super().__init__(
accelerations=accelerations,
center_fractions=center_fractions,
- uniform_range=uniform_range,
+ range_mode=range_mode,
mode=MaskFuncMode.DYNAMIC,
)
@@ -2378,7 +2458,10 @@ def zero_pad_to_center(array: np.ndarray, target_shape: tuple[int, ...]) -> np.n
# Calculate the slices for inserting the original array into the padded array
insert_slices = tuple(
- slice((target_dim - current_dim) // 2, (target_dim - current_dim) // 2 + current_dim)
+ slice(
+ (target_dim - current_dim) // 2,
+ (target_dim - current_dim) // 2 + current_dim,
+ )
for target_dim, current_dim in zip(target_shape, current_shape)
)
@@ -2518,7 +2601,10 @@ def crop_center(array, target_height, target_width):
# Calculate the slices for cropping the array
crop_slices = tuple(
- slice((current_dim - target_dim) // 2, (current_dim - target_dim) // 2 + target_dim)
+ slice(
+ (current_dim - target_dim) // 2,
+ (current_dim - target_dim) // 2 + target_dim,
+ )
for current_dim, target_dim in zip(current_shape, target_shape)
)
@@ -2536,8 +2622,8 @@ class KtRadialMaskFunc(KtBaseMaskFunc):
Amount of under-sampling.
center_fractions : Union[list[float], tuple[float, ...]]
Fraction of low-frequency columns (float < 1.0) or number of low-frequence columns (integer) to be retained.
- uniform_range : bool, optional
- If True then an acceleration will be uniformly sampled between the two values, by default False.
+ range_mode : RangeMode, optional
+ How accelerations are sampled. Defaults to RangeMode.DISCRETE.
crop_corner : bool, optional
If True, the mask is cropped to the corners. Default: False.
"""
@@ -2546,7 +2632,7 @@ def __init__(
self,
accelerations: Union[list[Number], tuple[Number, ...]],
center_fractions: Union[list[float], tuple[float, ...]],
- uniform_range: bool = False,
+ range_mode: RangeMode = RangeMode.DISCRETE,
crop_corner: bool = False,
) -> None:
"""Inits :class:`KtRadialMaskFunc`.
@@ -2557,15 +2643,15 @@ def __init__(
Amount of under-sampling.
center_fractions : Union[list[float], tuple[float, ...]]
Fraction of low-frequency columns (float < 1.0) or number of low-frequence columns (integer) to be retained.
- uniform_range : bool, optional
- If True then an acceleration will be uniformly sampled between the two values, by default False.
+ range_mode : RangeMode, optional
+ How accelerations are sampled. Defaults to RangeMode.DISCRETE.
crop_corner : bool, optional
If True, the mask is cropped to the corners. Default: False.
"""
super().__init__(
accelerations=accelerations,
center_fractions=center_fractions,
- uniform_range=uniform_range,
+ range_mode=range_mode,
)
self.crop_corner = crop_corner
@@ -2596,7 +2682,7 @@ def mask_func(
if len(shape) not in [4, 5]:
raise ValueError("Shape should have 4 or 5 dimensions.")
- (nt, num_rows, num_cols) = shape[-4:-1]
+ nt, num_rows, num_cols = shape[-4:-1]
with temp_seed(self.rng, seed):
center_fraction, acceleration = self.choose_acceleration()
@@ -2625,7 +2711,8 @@ def mask_func(
aux[int(temp_size / 2), :] = 1
base_mask = np.sum(
- [rotate(aux, angle, reshape=False, order=0) for angle in np.linspace(0, 180, num_beams)], axis=0
+ [rotate(aux, angle, reshape=False, order=0) for angle in np.linspace(0, 180, num_beams)],
+ axis=0,
)
mask = [self.crop_center(base_mask, num_rows, num_cols)]
@@ -2649,15 +2736,15 @@ class KtUniformMaskFunc(KtBaseMaskFunc):
Amount of under-sampling.
center_fractions : Union[list[float], tuple[float, ...]]
Fraction of low-frequency columns (float < 1.0) or number of low-frequence columns (integer) to be retained.
- uniform_range : bool, optional
- If True then an acceleration will be uniformly sampled between the two values, by default False.
+ range_mode : RangeMode, optional
+ How accelerations are sampled. Defaults to RangeMode.DISCRETE.
"""
def __init__(
self,
accelerations: Union[list[Number], tuple[Number, ...]],
center_fractions: Union[list[float], tuple[float, ...]],
- uniform_range: bool = False,
+ range_mode: RangeMode = RangeMode.DISCRETE,
) -> None:
"""Inits :class:`KtUniformMaskFunc`.
@@ -2667,13 +2754,13 @@ def __init__(
Amount of under-sampling.
center_fractions : Union[list[float], tuple[float, ...]]
Fraction of low-frequency columns (float < 1.0) or number of low-frequence columns (integer) to be retained.
- uniform_range : bool, optional
- If True then an acceleration will be uniformly sampled between the two values, by default False.
+ range_mode : RangeMode, optional
+ How accelerations are sampled. Defaults to RangeMode.DISCRETE.
"""
super().__init__(
accelerations=accelerations,
center_fractions=center_fractions,
- uniform_range=uniform_range,
+ range_mode=range_mode,
)
def mask_func(
@@ -2703,7 +2790,7 @@ def mask_func(
if len(shape) not in [4, 5]:
raise ValueError("Shape should have 4 or 5 dimensions.")
- (nt, num_rows, num_cols) = shape[-4:-1]
+ nt, num_rows, num_cols = shape[-4:-1]
with temp_seed(self.rng, seed):
center_fraction, acceleration = self.choose_acceleration()
@@ -2711,7 +2798,8 @@ def mask_func(
# Fully sampled rectangle region
acs_mask = self.zero_pad_to_center(
- np.ones((nt, num_rows, num_low_freqs)), (int(nt), int(num_rows), int(num_cols))
+ np.ones((nt, num_rows, num_low_freqs)),
+ (int(nt), int(num_rows), int(num_cols)),
)
if return_acs:
@@ -2724,8 +2812,14 @@ def mask_func(
ptmp = np.zeros(num_cols)
ttmp = np.zeros(nt)
- ptmp[np.arange(self.rng.randint(0, adjusted_acceleration), num_cols, adjusted_acceleration).astype(int)] = 1
- ttmp[np.arange(self.rng.randint(0, acceleration), nt, acceleration).astype(int)] = 1
+ ptmp[
+ np.arange(
+ self.rng.randint(0, max(1, int(adjusted_acceleration))),
+ num_cols,
+ adjusted_acceleration,
+ ).astype(int)
+ ] = 1
+ ttmp[np.arange(self.rng.randint(0, max(1, int(acceleration))), nt, acceleration).astype(int)] = 1
top_mat = toeplitz(ptmp, ttmp)
ind = np.where(top_mat.ravel())[0]
@@ -2755,8 +2849,8 @@ class KtGaussian1DMaskFunc(KtBaseMaskFunc):
Amount of under-sampling.
center_fractions : Union[list[float], tuple[float, ...]]
Fraction of low-frequency columns (float < 1.0) or number of low-frequence columns (integer) to be retained.
- uniform_range : bool, optional
- If True then an acceleration will be uniformly sampled between the two values, by default False.
+ range_mode : RangeMode, optional
+ How accelerations are sampled. Defaults to RangeMode.DISCRETE.
alpha : float, optional
0 < alpha < 1 controls sampling density; 0: uniform density, 1: maximally non-uniform density.
Default: 0.28.
@@ -2768,7 +2862,7 @@ def __init__(
self,
accelerations: Union[list[Number], tuple[Number, ...]],
center_fractions: Union[list[float], tuple[float, ...]],
- uniform_range: bool = False,
+ range_mode: RangeMode = RangeMode.DISCRETE,
alpha: float = 0.28,
std_scale: float = 5.0,
) -> None:
@@ -2780,8 +2874,8 @@ def __init__(
Amount of under-sampling.
center_fractions : Union[list[float], tuple[float, ...]]
Fraction of low-frequency columns (float < 1.0) or number of low-frequence columns (integer) to be retained.
- uniform_range : bool, optional
- If True then an acceleration will be uniformly sampled between the two values, by default False.
+ range_mode : RangeMode, optional
+ How accelerations are sampled. Defaults to RangeMode.DISCRETE.
alpha : float, optional
0 < alpha < 1 controls sampling density; 0: uniform density, 1: maximally non-uniform density.
Default: 0.28.
@@ -2791,7 +2885,7 @@ def __init__(
super().__init__(
accelerations=accelerations,
center_fractions=center_fractions,
- uniform_range=uniform_range,
+ range_mode=range_mode,
)
self.alpha = alpha
self.std_scale = std_scale
@@ -2823,7 +2917,7 @@ def mask_func(
if len(shape) not in [4, 5]:
raise ValueError("Shape should have 4 or 5 dimensions.")
- (nt, num_rows, num_cols) = shape[-4:-1]
+ nt, num_rows, num_cols = shape[-4:-1]
with temp_seed(self.rng, seed):
center_fraction, acceleration = self.choose_acceleration()
@@ -2831,7 +2925,8 @@ def mask_func(
# Fully sampled rectangle region
acs_mask = self.zero_pad_to_center(
- np.ones((nt, num_rows, num_low_freqs)), (int(nt), int(num_rows), int(num_cols))
+ np.ones((nt, num_rows, num_low_freqs)),
+ (int(nt), int(num_rows), int(num_cols)),
)
if return_acs:
@@ -2858,7 +2953,11 @@ def mask_func(
n_tmp = tr - len(a)
prob_tmp = prob.copy()
prob_tmp[a] = 0
- p_tmp = self.rng.choice(np.arange(-num_cols // 2, num_cols // 2), n_tmp, p=prob_tmp / prob_tmp.sum())
+ p_tmp = self.rng.choice(
+ np.arange(-num_cols // 2, num_cols // 2),
+ n_tmp,
+ p=prob_tmp / prob_tmp.sum(),
+ )
ti[ind : ind + n_tmp] = i
ph[ind : ind + n_tmp] = p_tmp
ind += n_tmp
@@ -2876,7 +2975,9 @@ def mask_func(
return self._reshape_and_add_coil_axis(mask, shape)
-def integerize_seed(seed: Union[None, int, tuple[int, ...], list[int], Iterable[int]]) -> int:
+def integerize_seed(
+ seed: Union[None, int, tuple[int, ...], list[int], Iterable[int]],
+) -> int:
"""Returns an integer seed.
If input is integer, will return the input. If input is None, will return a random integer seed.
@@ -2933,9 +3034,9 @@ def build_masking_function(
name: str,
accelerations: Union[list[Number], tuple[Number, ...]],
center_fractions: Optional[Union[list[Number], tuple[Number, ...]]] = None,
- uniform_range: bool = False,
+ range_mode: RangeMode = RangeMode.DISCRETE,
mode: MaskFuncMode = MaskFuncMode.STATIC,
- **kwargs: dict[str, Any],
+ **kwargs: Any,
) -> BaseMaskFunc:
"""Builds a mask function.
@@ -2947,10 +3048,10 @@ def build_masking_function(
Amount of under-sampling.
center_fractions : Union[list[Number], tuple[Number, ...]], optional
Fraction of low-frequency columns (float < 1.0) or number of low-frequence columns (integer) to be retained.
- If multiple values are provided, then one of these numbers is chosen uniformly each time. If uniform_range
- is True, then two values should be given, by default None.
- uniform_range : bool, optional
- If True then an acceleration will be uniformly sampled between the two values, by default False.
+ If multiple values are provided, then one of these numbers is chosen uniformly each time. If range_mode is UNIFORM or LINEAR,
+ then two values should be given, by default None.
+ range_mode : RangeMode, optional
+ How accelerations are sampled. Defaults to RangeMode.DISCRETE.
mode : MaskFuncMode, optional
Mode of the mask function. Can be MaskFuncMode.STATIC, MaskFuncMode.DYNAMIC, or MaskFuncMode.MULTISLICE.
If MaskFuncMode.STATIC, then a single mask is created independent of the requested shape, and will be
@@ -2981,7 +3082,7 @@ def build_masking_function(
kwargs.update(
{
"center_fractions": center_fractions,
- "uniform_range": uniform_range,
+ "range_mode": range_mode,
"mode": mode,
}
)
@@ -2995,4 +3096,7 @@ def build_masking_function(
# Create the MaskFunc instance with the prepared arguments
mask_func = MaskFunc(**init_args)
+ if isinstance(mask_func, BaseMaskFunc) and "range_mode" not in init_args:
+ mask_func.range_mode = range_mode
+
return mask_func
diff --git a/direct/common/subsample_config.py b/direct/common/subsample_config.py
index 56393f43..a640cfa1 100644
--- a/direct/common/subsample_config.py
+++ b/direct/common/subsample_config.py
@@ -19,7 +19,7 @@
from omegaconf import MISSING
from direct.config.defaults import BaseConfig
-from direct.types import MaskFuncMode
+from direct.types import MaskFuncMode, RangeMode
@dataclass
@@ -27,7 +27,7 @@ class MaskingConfig(BaseConfig):
name: str = MISSING
accelerations: tuple[float, ...] = (5.0,)
center_fractions: Optional[tuple[float, ...]] = (0.1,)
- uniform_range: bool = False
+ range_mode: RangeMode = RangeMode.DISCRETE
mode: MaskFuncMode = MaskFuncMode.STATIC
val_accelerations: tuple[float, ...] = (5.0, 10.0)
diff --git a/direct/data/datasets.py b/direct/data/datasets.py
index 60566777..88e266ee 100644
--- a/direct/data/datasets.py
+++ b/direct/data/datasets.py
@@ -35,7 +35,7 @@
from direct.data.sens import simulate_sensitivity_maps
from direct.types import PathOrString
from direct.utils import remove_keys, str_to_class
-from direct.utils.dataset import get_filenames_for_datasets
+from direct.utils.dataset import get_filenames_for_datasets, maybe_attach_field_strength
logger = logging.getLogger(__name__)
@@ -245,6 +245,8 @@ def __getitem__(self, index: int) -> dict[str, Any]:
if sample["kspace"].ndim == 2: # Singlecoil data does not always have coils at the first axis.
sample["kspace"] = sample["kspace"][np.newaxis, ...]
+ maybe_attach_field_strength(sample)
+
if self.transform:
sample = self.transform(sample)
@@ -612,7 +614,9 @@ def __init__(
self.logger.error(e)
raise ValueError(e)
filenames = get_filenames_for_datasets(
- lists=filenames_lists, files_root=filenames_lists_root, data_root=data_root
+ lists=filenames_lists,
+ files_root=filenames_lists_root,
+ data_root=data_root,
)
self.logger.info("Attempting to load %s filenames from list(s).", len(filenames))
else:
@@ -804,7 +808,11 @@ def __getitem__(self, index: int) -> dict[str, Any]:
if kspace.ndim == 2: # Single-coil data.
kspace = kspace[np.newaxis, ...]
- sample: dict[str, Any] = {"kspace": kspace, "filename": str(filename), "slice_no": slice_no}
+ sample: dict[str, Any] = {
+ "kspace": kspace,
+ "filename": str(filename),
+ "slice_no": slice_no,
+ }
if self.compute_mask or (any("mask" in key for key in extra_data)):
nx, ny = kspace.shape[-2:]
@@ -834,7 +842,11 @@ def __getitem__(self, index: int) -> dict[str, Any]:
sample.update(extra_data)
shape = kspace.shape
- sample["reconstruction_size"] = (int(np.round(shape[-2] / 3)), int(np.round(shape[-1] / 2)), 1)
+ sample["reconstruction_size"] = (
+ int(np.round(shape[-2] / 3)),
+ int(np.round(shape[-1] / 2)),
+ 1,
+ )
if self.kspace_context:
# Add context dimension in reconstruction size without any crop
@@ -843,6 +855,8 @@ def __getitem__(self, index: int) -> dict[str, Any]:
# If context put coil dim first
sample["kspace"] = np.swapaxes(sample["kspace"], 0, 1)
+ maybe_attach_field_strength(sample)
+
if self.transform:
sample = self.transform(sample) # ty: ignore[invalid-argument-type]
@@ -1028,7 +1042,7 @@ def __init__(
"""
self.logger = logging.getLogger(type(self).__name__)
- (self.nx, self.ny, self.nz) = (shape, shape, shape) if isinstance(shape, int) else tuple(shape)
+ self.nx, self.ny, self.nz = (shape, shape, shape) if isinstance(shape, int) else tuple(shape)
self.num_coils = num_coils
assert intensity in self.IMAGE_INTENSITIES, (
@@ -1097,7 +1111,11 @@ def sample_image(self, idx: int) -> np.ndarray: # pylint: disable=too-many-loca
# Put ellipses where they need to be
for j in range(self.ellipsoids.shape[0]):
- center_x, center_y, center_z = self.center_xs[j], self.center_ys[j], self.center_zs[j]
+ center_x, center_y, center_z = (
+ self.center_xs[j],
+ self.center_ys[j],
+ self.center_zs[j],
+ )
a, b, c = self.half_ax_as[j], self.half_ax_bs[j], self.half_ax_cs[j]
ct0, st0 = ct[j], st[j]
@@ -1141,6 +1159,7 @@ def __getitem__(self, index: int) -> dict[str, Any]:
kspace = self.fft(image)
sample = {"kspace": kspace, "filename": self.name, "slice_no": index}
+ maybe_attach_field_strength(sample)
if self.transform is not None:
sample = self.transform(sample)
@@ -1154,15 +1173,15 @@ def default_mr_ellipsoid_parameters() -> np.ndarray:
-------
ellipsoids : np.ndarray
Array containing the parameters for the ellipsoids used to construct the phantom.
- Each row of the form [x, y, z, a, b, c, \theta, m_0, A, C, T1, T2, \chi] represents an ellipsoid, where:
+ Each row of the form [x, y, z, a, b, c, theta, m_0, A, C, T1, T2, chi] represents an ellipsoid, where:
* (x, y, z): denotes the center of the ellipsoid
* (a, b, c): denote the lengths of the semi-major axis aligned with the x, y, z-axis, respectively
- * \theta: denotes the rotation angle of the ellipsoid in rads
+ * theta: denotes the rotation angle of the ellipsoid in rads
* m_0: denotes the spin density
* (A, C): denote the T1 parameters
- * T1: denotes the T1 value if explicit, otherwise T1 = A \times B_0^{C}
+ * T1: denotes the T1 value if explicit, otherwise T1 = A * B_0^{C}
* T2: denotes the T2 value
- * \chi: denotes the \chi value
+ * chi: denotes the chi value
References
----------
@@ -1171,23 +1190,138 @@ def default_mr_ellipsoid_parameters() -> np.ndarray:
"""
params = _mr_relaxation_parameters()
- ellipsoids = np.zeros((SheppLoganDataset.DEFAULT_NUM_ELLIPSOIDS, SheppLoganDataset.ELLIPSOID_NUM_PARAMS))
+ ellipsoids = np.zeros(
+ (
+ SheppLoganDataset.DEFAULT_NUM_ELLIPSOIDS,
+ SheppLoganDataset.ELLIPSOID_NUM_PARAMS,
+ )
+ )
ellipsoids[0, :] = [0, 0, 0, 0.72, 0.95, 0.93, 0, 0.8, *params["scalp"]]
ellipsoids[1, :] = [0, 0, 0, 0.69, 0.92, 0.9, 0, 0.12, *params["marrow"]]
ellipsoids[2, :] = [0, -0.0184, 0, 0.6624, 0.874, 0.88, 0, 0.98, *params["csf"]]
- ellipsoids[3, :] = [0, -0.0184, 0, 0.6524, 0.864, 0.87, 0, 0.745, *params["gray-matter"]]
- ellipsoids[4, :] = [-0.22, 0, -0.25, 0.41, 0.16, 0.21, np.deg2rad(-72), 0.98, *params["csf"]]
- ellipsoids[5, :] = [0.22, 0, -0.25, 0.31, 0.11, 0.22, np.deg2rad(72), 0.98, *params["csf"]]
- ellipsoids[6, :] = [0, 0.35, -0.25, 0.21, 0.25, 0.35, 0, 0.617, *params["white-matter"]]
- ellipsoids[7, :] = [0, 0.1, -0.25, 0.046, 0.046, 0.046, 0, 0.95, *params["tumor"]]
- ellipsoids[8, :] = [-0.08, -0.605, -0.25, 0.046, 0.023, 0.02, 0, 0.95, *params["tumor"]]
- ellipsoids[9, :] = [0.06, -0.605, -0.25, 0.046, 0.023, 0.02, np.deg2rad(-90), 0.95, *params["tumor"]]
- ellipsoids[10, :] = [0, -0.1, -0.25, 0.046, 0.046, 0.046, 0, 0.95, *params["tumor"]]
- ellipsoids[11, :] = [0, -0.605, -0.25, 0.023, 0.023, 0.023, 0, 0.95, *params["tumor"]]
- ellipsoids[12, :] = [0.06, -0.105, 0.0625, 0.056, 0.04, 0.1, np.deg2rad(-90), 0.93, *params["tumor"]]
+ ellipsoids[3, :] = [
+ 0,
+ -0.0184,
+ 0,
+ 0.6524,
+ 0.864,
+ 0.87,
+ 0,
+ 0.745,
+ *params["gray-matter"],
+ ]
+ ellipsoids[4, :] = [
+ -0.22,
+ 0,
+ -0.25,
+ 0.41,
+ 0.16,
+ 0.21,
+ np.deg2rad(-72),
+ 0.98,
+ *params["csf"],
+ ]
+ ellipsoids[5, :] = [
+ 0.22,
+ 0,
+ -0.25,
+ 0.31,
+ 0.11,
+ 0.22,
+ np.deg2rad(72),
+ 0.98,
+ *params["csf"],
+ ]
+ ellipsoids[6, :] = [
+ 0,
+ 0.35,
+ -0.25,
+ 0.21,
+ 0.25,
+ 0.35,
+ 0,
+ 0.617,
+ *params["white-matter"],
+ ]
+ ellipsoids[7, :] = [
+ 0,
+ 0.1,
+ -0.25,
+ 0.046,
+ 0.046,
+ 0.046,
+ 0,
+ 0.95,
+ *params["tumor"],
+ ]
+ ellipsoids[8, :] = [
+ -0.08,
+ -0.605,
+ -0.25,
+ 0.046,
+ 0.023,
+ 0.02,
+ 0,
+ 0.95,
+ *params["tumor"],
+ ]
+ ellipsoids[9, :] = [
+ 0.06,
+ -0.605,
+ -0.25,
+ 0.046,
+ 0.023,
+ 0.02,
+ np.deg2rad(-90),
+ 0.95,
+ *params["tumor"],
+ ]
+ ellipsoids[10, :] = [
+ 0,
+ -0.1,
+ -0.25,
+ 0.046,
+ 0.046,
+ 0.046,
+ 0,
+ 0.95,
+ *params["tumor"],
+ ]
+ ellipsoids[11, :] = [
+ 0,
+ -0.605,
+ -0.25,
+ 0.023,
+ 0.023,
+ 0.023,
+ 0,
+ 0.95,
+ *params["tumor"],
+ ]
+ ellipsoids[12, :] = [
+ 0.06,
+ -0.105,
+ 0.0625,
+ 0.056,
+ 0.04,
+ 0.1,
+ np.deg2rad(-90),
+ 0.93,
+ *params["tumor"],
+ ]
ellipsoids[13, :] = [0, 0.1, 0.625, 0.056, 0.056, 0.1, 0, 0.98, *params["csf"]]
- ellipsoids[14, :] = [0.56, -0.4, -0.25, 0.2, 0.03, 0.1, np.deg2rad(70), 0.85, *params["blood-clot"]]
+ ellipsoids[14, :] = [
+ 0.56,
+ -0.4,
+ -0.25,
+ 0.2,
+ 0.03,
+ 0.1,
+ np.deg2rad(70),
+ 0.85,
+ *params["blood-clot"],
+ ]
# Need to subtract some ellipses here...
ellipsoids_neg = np.zeros(ellipsoids.shape)
@@ -1491,10 +1625,20 @@ def build_dataset_from_input(
f"Got {kwargs.get('initial_images')} and {kwargs.get('initial_kspaces')}."
)
if "initial_images" in kwargs:
- pass_h5s = {"initial_image": (dataset_config.input_image_key, kwargs.get("initial_images"))}
+ pass_h5s = {
+ "initial_image": (
+ dataset_config.input_image_key,
+ kwargs.get("initial_images"),
+ )
+ }
del kwargs["initial_images"]
elif "initial_kspaces" in kwargs:
- pass_h5s = {"initial_kspace": (dataset_config.input_kspace_key, kwargs.get("initial_kspaces"))}
+ pass_h5s = {
+ "initial_kspace": (
+ dataset_config.input_kspace_key,
+ kwargs.get("initial_kspaces"),
+ )
+ }
del kwargs["initial_kspaces"]
if pass_h5s is not None:
kwargs.update({"pass_h5s": pass_h5s})
@@ -1503,7 +1647,8 @@ def build_dataset_from_input(
# case the arguments in kwargs.
# For example, `data_root` can be passed both from the command line and in the configuration file.
config_kwargs = remove_keys(
- dict(dataset_config), ["name", "transforms"] + list(kwargs.keys() & dict(dataset_config).keys())
+ dict(dataset_config),
+ ["name", "transforms"] + list(kwargs.keys() & dict(dataset_config).keys()),
)
dataset = build_dataset(
name=dataset_config.name, # type: ignore
diff --git a/direct/data/h5_data.py b/direct/data/h5_data.py
index ce55130e..4212af25 100644
--- a/direct/data/h5_data.py
+++ b/direct/data/h5_data.py
@@ -22,7 +22,7 @@
from direct.types import PathOrString
from direct.utils import cast_as_path
-from direct.utils.dataset import get_filenames_for_datasets
+from direct.utils.dataset import get_filenames_for_datasets, maybe_attach_field_strength
logger = logging.getLogger(__name__)
@@ -248,6 +248,7 @@ def __getitem__(self, index: int) -> Dict[str, Any]:
raise ValueError(f"Trying to add key {key} to sample dict, but this key already exists.")
sample[key] = curr_slice
+ maybe_attach_field_strength(sample)
return sample
def get_slice_data(self, filename, slice_no, key="kspace", pass_attrs=False, extra_keys=None):
diff --git a/direct/data/mri_transforms.py b/direct/data/mri_transforms.py
index 653255be..a6570ad8 100644
--- a/direct/data/mri_transforms.py
+++ b/direct/data/mri_transforms.py
@@ -12,7 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""The `direct.data.mri_transforms` module contains mri transformations utilized to transform or augment k-space data,
-used for DIRECT's training pipeline. They can be also used individually by importing them into python scripts."""
+used for DIRECT's training pipeline. They can be also used individually by importing them into python scripts.
+"""
from __future__ import annotations
@@ -210,9 +211,7 @@ def __call__(self, sample: dict[str, Any]) -> dict[str, Any]:
else (
(-1,)
if self.flip == "vertical"
- else (-2, -1)
- if self.flip == "both"
- else (random.SystemRandom().choice([-2, -1]),)
+ else ((-2, -1) if self.flip == "both" else (random.SystemRandom().choice([-2, -1]),))
)
)
@@ -285,6 +284,13 @@ def __call__(self, sample: dict[str, Any]) -> dict[str, Any]:
return sample
+def _mask_func_supports_return_acceleration(mask_func: Callable) -> bool:
+ """Check whether a mask callable can return sampled acceleration metadata."""
+ from direct.common.subsample import BaseMaskFunc
+
+ return isinstance(mask_func, BaseMaskFunc)
+
+
class CreateSamplingMask(DirectTransform):
"""Data Transformer for training MRI reconstruction models.
@@ -341,13 +347,22 @@ def __call__(self, sample: dict[str, Any]) -> dict[str, Any]:
seed = None if not self.use_seed else tuple(map(ord, str(sample["filename"])))
- sampling_mask = self.mask_func(shape=shape, seed=seed, return_acs=False)
+ mask_kwargs = {"shape": shape, "seed": seed, "return_acs": False}
+ if _mask_func_supports_return_acceleration(self.mask_func):
+ mask_kwargs["return_acceleration"] = True
+ sampling_mask, acceleration, center_fraction = self.mask_func(**mask_kwargs)
+ if "padding" in sample:
+ sampling_mask = T.apply_padding(sampling_mask, sample["padding"])
- if "padding" in sample:
- sampling_mask = T.apply_padding(sampling_mask, sample["padding"])
+ sample["sampling_mask"] = sampling_mask
+ sample["acceleration"] = torch.tensor([acceleration], dtype=sample["kspace"].dtype)
+ sample["center_fraction"] = torch.tensor([center_fraction], dtype=sample["kspace"].dtype)
+ else:
+ sampling_mask = self.mask_func(**mask_kwargs)
+ if "padding" in sample:
+ sampling_mask = T.apply_padding(sampling_mask, sample["padding"])
- # Shape 3D: (1, 1, height, width, 1), 2D: (1, height, width, 1)
- sample["sampling_mask"] = sampling_mask
+ sample["sampling_mask"] = sampling_mask
if self.return_acs:
sample["acs_mask"] = self.mask_func(shape=shape, seed=seed, return_acs=True)
@@ -475,7 +490,7 @@ def __init__(
else:
self.crop_func = functools.partial(
T.complex_random_crop,
- sampler=random_crop_sampler_type if random_crop_sampler_type is not None else "uniform",
+ sampler=(random_crop_sampler_type if random_crop_sampler_type is not None else "uniform"),
sigma=random_crop_sampler_gaussian_sigma,
)
self.random_crop_sampler_use_seed = random_crop_sampler_use_seed
@@ -656,7 +671,9 @@ def __call__(self, sample: dict[str, Any]) -> dict[str, Any]:
backprojected_kspace = backprojected_kspace.unsqueeze(0)
rescaled_backprojected_kspace = T.complex_image_resize(
- backprojected_kspace, self.shape, self.rescale_mode if self.rescale_mode is not None else "nearest"
+ backprojected_kspace,
+ self.shape,
+ self.rescale_mode if self.rescale_mode is not None else "nearest",
)
if (kspace.ndim == 4) or (kspace.ndim == 5 and not self.rescale_2d_if_3d):
@@ -917,14 +934,14 @@ def forward(self, sample: dict[str, Any]) -> dict[str, Any]:
Parameters
----------
sample: dict[str, Any]
- Contains key kspace_key with value a torch.Tensor of shape (coil,\*spatial_dims, complex=2).
+ Contains key kspace_key with value a torch.Tensor of shape (coil, \\*spatial_dims, complex=2).
Returns
-------
sample: dict
- Contains key target_key with value a torch.Tensor of shape (\*spatial_dims) if `type_reconstruction` is
+ Contains key target_key with value a torch.Tensor of shape (\\*spatial_dims) if `type_reconstruction` is
ReconstructionType.RSS, ReconstructionType.COMPLEX_MOD, ReconstructionType.SENSE_MOD,
- and of shape (\*spatial_dims, complex_dim=2) otherwise.
+ and of shape (\\*spatial_dims, complex_dim=2) otherwise.
"""
kspace_data = sample[self.kspace_key]
dim = self.spatial_dims.TWO_D if kspace_data.ndim == 5 else self.spatial_dims.THREE_D
@@ -1281,7 +1298,10 @@ def forward(self, sample: dict[str, Any]) -> dict[str, Any]:
if ndim == 6:
compressed_k_space = compressed_k_space.reshape(
- shape[0] // num_slice_or_time, num_slice_or_time, self.num_coils, *shape[2:]
+ shape[0] // num_slice_or_time,
+ num_slice_or_time,
+ self.num_coils,
+ *shape[2:],
).permute(0, 2, 1, 3, 4)
compressed_k_space = torch.view_as_real(compressed_k_space)
@@ -2240,7 +2260,9 @@ def build_supervised_mri_transforms(
]
mri_transforms += [
ComputeScalingFactor(
- normalize_key=scaling_key, percentile=scale_percentile, scaling_factor_key=TransformKey.SCALING_FACTOR
+ normalize_key=scaling_key,
+ percentile=scale_percentile,
+ scaling_factor_key=TransformKey.SCALING_FACTOR,
),
Normalize(
scaling_factor_key=TransformKey.SCALING_FACTOR,
@@ -2483,8 +2505,8 @@ def build_mri_transforms(
sensitivity_maps_espirit_kernel_size=sensitivity_maps_espirit_kernel_size,
sensitivity_maps_espirit_crop=sensitivity_maps_espirit_crop,
sensitivity_maps_espirit_max_iters=sensitivity_maps_espirit_max_iters,
- delete_acs_mask=delete_acs_mask if transforms_type == TransformsType.SUPERVISED else False,
- delete_kspace=delete_kspace if transforms_type == TransformsType.SUPERVISED else False,
+ delete_acs_mask=(delete_acs_mask if transforms_type == TransformsType.SUPERVISED else False),
+ delete_kspace=(delete_kspace if transforms_type == TransformsType.SUPERVISED else False),
image_recon_type=image_recon_type,
compress_coils=compress_coils,
pad_coils=pad_coils,
diff --git a/direct/data/sens.py b/direct/data/sens.py
index 90c74d8f..28994bfe 100644
--- a/direct/data/sens.py
+++ b/direct/data/sens.py
@@ -18,7 +18,10 @@
def simulate_sensitivity_maps(
- shape: Union[List[int], Tuple[int, ...]], num_coils: int, var: float = 1, seed: Optional[int] = None
+ shape: Union[List[int], Tuple[int, ...]],
+ num_coils: int,
+ var: float = 1,
+ seed: Optional[int] = None,
) -> np.ndarray:
r"""Simulates coil sensitivities using bi-variate or tri-variate gaussian distribution.
@@ -36,7 +39,7 @@ def simulate_sensitivity_maps(
Returns
-------
sensitivity_map : nd.array
- Simulated coil sensitivity maps of shape (num_coils, \*shape).
+ Simulated coil sensitivity maps of shape (num_coils, \\*shape).
Notes
-----
diff --git a/direct/data/transforms.py b/direct/data/transforms.py
index 0b0dd9a3..61bd979b 100644
--- a/direct/data/transforms.py
+++ b/direct/data/transforms.py
@@ -9,7 +9,8 @@
This module contains functions for complex-valued data manipulation in PyTorch. This includes functions for complex
multiplication, division, modulus, fft, ifft, fftshift, ifftshift, and more. The functions are designed to work with
complex-valued data where the last axis denotes the real and imaginary parts respectively. The functions are designed to
-work with complex-valued data where the last axis denotes the real and imaginary parts respectively."""
+work with complex-valued data where the last axis denotes the real and imaginary parts respectively.
+"""
from __future__ import annotations
@@ -74,12 +75,12 @@ def view_as_complex(data):
----------
data: torch.Tensor
Input data with torch.dtype torch.float64 and torch.float32 with complex axis (last) of dimension 2
- and of shape (N, \*, 2).
+ and of shape (N, \\*, 2).
Returns
-------
complex_valued_data: torch.Tensor
- Output complex-valued data of shape (N, \*) with complex torch.dtype.
+ Output complex-valued data of shape (N, \\*) with complex torch.dtype.
"""
return torch.view_as_complex(data)
@@ -93,12 +94,12 @@ def view_as_real(data):
Parameters
----------
data: torch.Tensor
- Input data with complex torch.dtype of shape (N, \*).
+ Input data with complex torch.dtype of shape (N, \\*).
Returns
-------
real_valued_data: torch.Tensor
- Output real-valued data of shape (N, \*, 2).
+ Output real-valued data of shape (N, \\*, 2).
"""
return torch.view_as_real(data)
@@ -119,7 +120,7 @@ def fft2(
Parameters
----------
data: torch.Tensor
- Complex-valued input tensor. Should be of shape (\*, 2) and dim is in \*.
+ Complex-valued input tensor. Should be of shape (\\*, 2) and dim is in \\*.
dim: tuple, list or int
Dimensions over which to compute. Should be positive. Negative indexing not supported
Default is (1, 2), corresponding to ('height', 'width').
@@ -179,7 +180,7 @@ def ifft2(
Parameters
----------
data: torch.Tensor
- Complex-valued input tensor. Should be of shape (\*, 2) and dim is in \*.
+ Complex-valued input tensor. Should be of shape (\\*, 2) and dim is in \\*.
dim: tuple, list or int
Dimensions over which to compute. Should be positive. Negative indexing not supported
Default is (1, 2), corresponding to ( 'height', 'width').
@@ -460,10 +461,12 @@ def complex_division(input_tensor: torch.Tensor, other_tensor: torch.Tensor) ->
denominator = other_tensor[..., 0] ** 2 + other_tensor[..., 1] ** 2
real_part = safe_divide(
- input_tensor[..., 0] * other_tensor[..., 0] + input_tensor[..., 1] * other_tensor[..., 1], denominator
+ input_tensor[..., 0] * other_tensor[..., 0] + input_tensor[..., 1] * other_tensor[..., 1],
+ denominator,
)
imaginary_part = safe_divide(
- input_tensor[..., 1] * other_tensor[..., 0] - input_tensor[..., 0] * other_tensor[..., 1], denominator
+ input_tensor[..., 1] * other_tensor[..., 0] - input_tensor[..., 0] * other_tensor[..., 1],
+ denominator,
)
division = torch.cat(
@@ -569,7 +572,7 @@ def apply_padding(
Parameters
----------
data : torch.Tensor
- Batched or not input to be padded of shape (`batch`, \*, `height`, `width`, \*).
+ Batched or not input to be padded of shape (`batch`, \\*, `height`, `width`, \\*).
padding : torch.Tensor or None
Binary tensor of shape (`batch`, 1, `height`, `width`, 1). Entries in `padding` with non-zero value
point to samples in `data` that will be zero-padded. If None, `data` will be returned.
@@ -873,12 +876,12 @@ def crop_to_acs(acs_mask: torch.Tensor, kspace: torch.Tensor) -> torch.Tensor:
acs_mask : torch.Tensor
Autocalibration mask of shape (height, width).
kspace : torch.Tensor
- K-space of shape (coil, height, width, \*).
+ K-space of shape (coil, height, width, \\*).
Returns
-------
torch.Tensor
- Cropped k-space of shape (coil, height', width', \*), where height' and width' are the new dimensions derived
+ Cropped k-space of shape (coil, height', width', \\*), where height' and width' are the new dimensions derived
from the acs_mask.
"""
nonzero_idxs = torch.nonzero(acs_mask)
diff --git a/direct/engine.py b/direct/engine.py
index ec0d5622..5e1d6fb2 100644
--- a/direct/engine.py
+++ b/direct/engine.py
@@ -518,7 +518,11 @@ def validation_loop(
)
storage.add_image(f"{key_prefix}target", visualize_target)
- self.logger.info("Done evaluation of %s at iteration %s.", str(curr_dataset_name), str(iter_idx))
+ self.logger.info(
+ "Done evaluation of %s at iteration %s.",
+ str(curr_dataset_name),
+ str(iter_idx),
+ )
self.model.train()
def process_slices_for_visualization(self, visualize_slices, visualize_target):
@@ -526,7 +530,7 @@ def process_slices_for_visualization(self, visualize_slices, visualize_target):
# Compute the difference as well, and normalize for visualization
difference_slices = [a - b for a, b in zip(visualize_slices, visualize_target)]
# Normalize slices
- difference_slices = [(d / np.abs(d)) * 0.5 + 0.5 for d in difference_slices]
+ difference_slices = [(d / d.abs().clamp(min=1e-8)) * 0.5 + 0.5 for d in difference_slices]
visualize_slices = [normalize_image(image) for image in visualize_slices]
# Visualize slices, and crop to the largest volume
diff --git a/direct/nn/adain/__init__.py b/direct/nn/adain/__init__.py
new file mode 100644
index 00000000..1be13d14
--- /dev/null
+++ b/direct/nn/adain/__init__.py
@@ -0,0 +1,18 @@
+# Copyright 2026 AI for Oncology Research Group. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""direct.nn.adain module."""
+
+from direct.nn.adain.adain import AdaIN2d, AdaIN3d, NormType
+
+__all__ = ["AdaIN2d", "AdaIN3d", "NormType"]
diff --git a/direct/nn/adain/adain.py b/direct/nn/adain/adain.py
new file mode 100644
index 00000000..4ce4c134
--- /dev/null
+++ b/direct/nn/adain/adain.py
@@ -0,0 +1,168 @@
+# Copyright 2026 AI for Oncology Research Group. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Adaptive Instance Normalization (AdaIN) modules for 2D and 3D tensors based on [1]_.
+
+References
+----------
+
+.. [1] Moriakov, N., Yiasemis, G., Sonke, J.-J. & Teuwen, J. (2026). Conditional Learned Reconstruction for
+ Medical Imaging. Proceedings of The 9th International Conference on Medical Imaging with Deep Learning,
+ PMLR 315:754-780. https://proceedings.mlr.press/v315/moriakov26a.html
+"""
+
+from __future__ import annotations
+
+from enum import Enum
+
+import torch
+from torch import nn
+
+__all__ = ["AdaIN2d", "AdaIN3d", "NormType"]
+
+
+class NormType(str, Enum):
+ INSTANCE = "instance"
+ ADAIN = "adain"
+
+
+class AdaIN2d(nn.Module):
+ """Adaptive Instance Normalization for 2D tensors based on [1]_.
+
+ Given input x of shape (B, C, H, W) and auxiliary vector y of shape (B, F),
+ produces per-sample, per-channel affine parameters from y.
+
+ References
+ ----------
+
+ .. [1] Moriakov, N., Yiasemis, G., Sonke, J.-J. & Teuwen, J. (2026). Conditional Learned Reconstruction for
+ Medical Imaging. Proceedings of The 9th International Conference on Medical Imaging with Deep Learning,
+ PMLR 315:754-780. https://proceedings.mlr.press/v315/moriakov26a.html
+ """
+
+ def __init__(
+ self,
+ num_channels: int,
+ aux_in_features: int,
+ hidden_features: int | tuple[int, ...] | None = None,
+ activation: nn.Module | None = None,
+ eps: float = 1e-5,
+ use_one_plus_gamma: bool = True,
+ ):
+ super().__init__()
+ self.num_channels = num_channels
+ self.eps = eps
+ self.use_one_plus_gamma = use_one_plus_gamma
+
+ if activation is None:
+ activation = nn.SiLU()
+
+ if hidden_features is None:
+ hidden = []
+ elif isinstance(hidden_features, int):
+ hidden = [hidden_features]
+ else:
+ hidden = list(hidden_features)
+
+ layers: list[nn.Module] = []
+ in_f = aux_in_features
+ for h in hidden:
+ layers += [nn.Linear(in_f, h), activation]
+ in_f = h
+ layers += [nn.Linear(in_f, 2 * num_channels)]
+ self.mlp = nn.Sequential(*layers)
+
+ if isinstance(self.mlp[-1], nn.Linear):
+ nn.init.zeros_(self.mlp[-1].weight)
+ nn.init.zeros_(self.mlp[-1].bias)
+
+ def forward(self, x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
+ mean = x.mean(dim=(2, 3), keepdim=True)
+ var = x.var(dim=(2, 3), keepdim=True, unbiased=False)
+ x_norm = (x - mean) / torch.sqrt(var + self.eps)
+
+ params = self.mlp(y)
+ gamma, beta = params.chunk(2, 1)
+
+ gamma = gamma.view(-1, self.num_channels, 1, 1)
+ beta = beta.view(-1, self.num_channels, 1, 1)
+
+ if self.use_one_plus_gamma:
+ return x_norm * (1.0 + gamma) + beta
+ return x_norm * gamma + beta
+
+
+class AdaIN3d(nn.Module):
+ """Adaptive Instance Normalization for 3D tensors based on [1]_.
+
+ Given input x of shape (B, C, Z, H, W) and auxiliary vector y of shape (B, F),
+ produces per-sample, per-channel affine parameters from y.
+
+ References
+ ----------
+
+ .. [1] Moriakov, N., Yiasemis, G., Sonke, J.-J. & Teuwen, J. (2026). Conditional Learned Reconstruction for
+ Medical Imaging. Proceedings of The 9th International Conference on Medical Imaging with Deep Learning,
+ PMLR 315:754-780. https://proceedings.mlr.press/v315/moriakov26a.html
+ """
+
+ def __init__(
+ self,
+ num_channels: int,
+ aux_in_features: int,
+ hidden_features: int | tuple[int, ...] | None = None,
+ activation: nn.Module | None = None,
+ eps: float = 1e-5,
+ use_one_plus_gamma: bool = True,
+ ):
+ super().__init__()
+ self.num_channels = num_channels
+ self.eps = eps
+ self.use_one_plus_gamma = use_one_plus_gamma
+
+ if activation is None:
+ activation = nn.SiLU()
+
+ if hidden_features is None:
+ hidden = []
+ elif isinstance(hidden_features, int):
+ hidden = [hidden_features]
+ else:
+ hidden = list(hidden_features)
+
+ layers: list[nn.Module] = []
+ in_f = aux_in_features
+ for h in hidden:
+ layers += [nn.Linear(in_f, h), activation]
+ in_f = h
+ layers += [nn.Linear(in_f, 2 * num_channels)]
+ self.mlp = nn.Sequential(*layers)
+
+ if isinstance(self.mlp[-1], nn.Linear):
+ nn.init.zeros_(self.mlp[-1].weight)
+ nn.init.zeros_(self.mlp[-1].bias)
+
+ def forward(self, x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
+ mean = x.mean(dim=(2, 3, 4), keepdim=True)
+ var = x.var(dim=(2, 3, 4), keepdim=True, unbiased=False)
+ x_norm = (x - mean) / torch.sqrt(var + self.eps)
+
+ params = self.mlp(y)
+ gamma, beta = params.chunk(2, dim=-1)
+
+ gamma = gamma.view(-1, self.num_channels, 1, 1, 1)
+ beta = beta.view(-1, self.num_channels, 1, 1, 1)
+
+ if self.use_one_plus_gamma:
+ return x_norm * (1.0 + gamma) + beta
+ return x_norm * gamma + beta
diff --git a/direct/nn/conv/conv.py b/direct/nn/conv/conv.py
index 5df2547d..abb56f75 100644
--- a/direct/nn/conv/conv.py
+++ b/direct/nn/conv/conv.py
@@ -11,16 +11,27 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-from typing import List
+from __future__ import annotations
+
+from typing import List, Optional
import torch
import torch.nn as nn
+from direct.nn.conv.modulated import (
+ ModConv2dBias,
+ ModConvActivation,
+ ModConvType,
+ ModulationParams,
+ mod_conv2d,
+)
+
class Conv2d(nn.Module):
"""Implementation of a simple cascade of 2D convolutions.
If `batchnorm` is set to True, batch normalization layer is applied after each convolution.
+ Supports modulated convolutions when `modulation` is not ModConvType.NONE.
"""
def __init__(
@@ -31,6 +42,13 @@ def __init__(
n_convs: int = 3,
activation: nn.Module = nn.PReLU(),
batchnorm: bool = False,
+ modulation: ModConvType = ModConvType.NONE,
+ modulation_params: Optional[ModulationParams] = None,
+ aux_in_features: Optional[int] = None,
+ fc_hidden_features: Optional[tuple[int] | int] = None,
+ fc_groups: int = 1,
+ fc_activation: ModConvActivation = ModConvActivation.SIGMOID,
+ num_weights: Optional[int] = None,
):
"""Inits :class:`Conv2d`.
@@ -48,37 +66,90 @@ def __init__(
Activation function.
batchnorm: bool
If True a batch normalization layer is applied after every convolution.
+ modulation : ModConvType
+ Modulation type. Default: ModConvType.NONE.
+ aux_in_features : int, optional
+ Number of features in the auxiliary input for modulation.
+ fc_hidden_features : int or tuple of int, optional
+ Hidden features in the modulation MLP.
+ fc_groups : int
+ Groups for modulation MLP output interpolation. Default: 1.
+ fc_activation : ModConvActivation
+ Activation after modulation MLP. Default: ModConvActivation.SIGMOID.
+ num_weights : int, optional
+ Number of weight bases for ModConvType.SUM.
"""
super().__init__()
- conv: List[nn.Module] = []
+ if modulation_params is None:
+ modulation_params = ModulationParams(
+ modulation=modulation,
+ aux_in_features=aux_in_features,
+ fc_hidden_features=fc_hidden_features,
+ fc_groups=fc_groups,
+ fc_activation=fc_activation,
+ num_weights=num_weights,
+ )
+ self.modulation = modulation_params.modulation
+
+ conv_layers: List[nn.Module] = []
+ norm_layers: List[Optional[nn.Module]] = []
+ act_layers: List[Optional[nn.Module]] = []
+
for idx in range(n_convs):
- conv.append(
- nn.Conv2d(
- in_channels if idx == 0 else hidden_channels,
- hidden_channels if idx != n_convs - 1 else out_channels,
+ ic = in_channels if idx == 0 else hidden_channels
+ oc = hidden_channels if idx != n_convs - 1 else out_channels
+
+ conv_layers.append(
+ mod_conv2d(
+ ic,
+ oc,
kernel_size=3,
padding=1,
+ bias=ModConv2dBias.PARAM,
+ modulation_params=modulation_params,
)
)
if batchnorm:
- conv.append(nn.BatchNorm2d(hidden_channels if idx != n_convs - 1 else out_channels, eps=1e-4))
+ norm_layers.append(nn.BatchNorm2d(oc, eps=1e-4))
+ else:
+ norm_layers.append(None)
if idx != n_convs - 1:
- conv.append(activation)
- self.conv = nn.Sequential(*conv)
+ act_layers.append(activation)
+ else:
+ act_layers.append(None)
+
+ self.conv_layers = nn.ModuleList(conv_layers)
+ self.norm_layers = nn.ModuleList([m for m in norm_layers if m is not None]) if batchnorm else None
+ self.act_layers = act_layers
+ self.n_convs = n_convs
+ self.batchnorm = batchnorm
- def forward(self, x: torch.Tensor) -> torch.Tensor:
+ def forward(self, x: torch.Tensor, y: Optional[torch.Tensor] = None) -> torch.Tensor:
"""Performs the forward pass of :class:`Conv2d`.
Parameters
----------
x: torch.Tensor
Input tensor.
+ y: torch.Tensor, optional
+ Auxiliary signal for modulation of shape (N, aux_in_features).
Returns
-------
out: torch.Tensor
Convoluted output.
"""
- out = self.conv(x)
- return out
+ norm_idx = 0
+ for idx in range(self.n_convs):
+ if self.modulation != ModConvType.NONE:
+ x = self.conv_layers[idx](x, y)
+ else:
+ x = self.conv_layers[idx](x)
+ if self.batchnorm and self.norm_layers is not None:
+ x = self.norm_layers[norm_idx](x)
+ norm_idx += 1
+ act_layer = self.act_layers[idx]
+ if act_layer is not None:
+ x = act_layer(x)
+ return x
diff --git a/direct/nn/conv/modulated/__init__.py b/direct/nn/conv/modulated/__init__.py
new file mode 100644
index 00000000..4fe70f64
--- /dev/null
+++ b/direct/nn/conv/modulated/__init__.py
@@ -0,0 +1,62 @@
+# Copyright 2026 AI for Oncology Research Group. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Modulated convolution layers and auxiliary conditioning utilities."""
+
+from direct.nn.conv.modulated.auxiliary_data import (
+ AUXILIARY_FEATURE_REGISTRY,
+ DEFAULT_AUXILIARY_FEATURE_NAMES,
+ AuxiliaryFeature,
+ ModulationConfig,
+ prepare_auxiliary_data,
+ register_auxiliary_feature,
+ resolve_auxiliary_features,
+)
+from direct.nn.conv.modulated.factory import (
+ ModulationParams,
+ mod_conv2d,
+ mod_conv3d,
+ mod_conv_transpose2d,
+ mod_conv_transpose3d,
+)
+from direct.nn.conv.modulated.modulated_conv import (
+ ModConv2d,
+ ModConv2dBias,
+ ModConv3d,
+ ModConvActivation,
+ ModConvTranspose2d,
+ ModConvTranspose3d,
+ ModConvType,
+)
+
+__all__ = [
+ "ModulationParams",
+ "mod_conv2d",
+ "mod_conv3d",
+ "mod_conv_transpose2d",
+ "mod_conv_transpose3d",
+ "AUXILIARY_FEATURE_REGISTRY",
+ "AuxiliaryFeature",
+ "DEFAULT_AUXILIARY_FEATURE_NAMES",
+ "ModConv2d",
+ "ModConv2dBias",
+ "ModConv3d",
+ "ModConvActivation",
+ "ModConvTranspose2d",
+ "ModConvTranspose3d",
+ "ModConvType",
+ "ModulationConfig",
+ "prepare_auxiliary_data",
+ "register_auxiliary_feature",
+ "resolve_auxiliary_features",
+]
diff --git a/direct/nn/conv/modulated/auxiliary_data.py b/direct/nn/conv/modulated/auxiliary_data.py
new file mode 100644
index 00000000..9c943220
--- /dev/null
+++ b/direct/nn/conv/modulated/auxiliary_data.py
@@ -0,0 +1,231 @@
+# Copyright 2026 AI for Oncology Research Group. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Utilities for building auxiliary conditioning vectors for modulated convolutions."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Mapping, Optional, Protocol, Sequence
+
+import torch
+
+from direct.nn.adain.adain import NormType
+from direct.nn.conv.modulated.modulated_conv import ModConvType
+
+__all__ = [
+ "AUXILIARY_FEATURE_REGISTRY",
+ "AuxiliaryFeature",
+ "DEFAULT_AUXILIARY_FEATURE_NAMES",
+ "ModulationConfig",
+ "prepare_auxiliary_data",
+ "register_auxiliary_feature",
+ "resolve_auxiliary_features",
+]
+
+
+@dataclass(frozen=True)
+class AuxiliaryFeature:
+ """Metadata for one auxiliary conditioning channel.
+
+ Parameters
+ ----------
+ key : str
+ Key in the batch dictionary.
+ log_scale : float
+ Multiplier applied to the feature before ``log`` when ``log_aux`` is enabled.
+ """
+
+ key: str
+ log_scale: float = 1.0
+
+
+AUXILIARY_FEATURE_REGISTRY: dict[str, AuxiliaryFeature] = {
+ "acceleration": AuxiliaryFeature("acceleration"),
+ "center_fraction": AuxiliaryFeature("center_fraction", log_scale=100.0),
+ "field_strength": AuxiliaryFeature("field_strength"),
+}
+
+DEFAULT_AUXILIARY_FEATURE_NAMES: tuple[str, ...] = tuple(AUXILIARY_FEATURE_REGISTRY.keys())
+
+
+class ModulationConfig(Protocol):
+ """Minimal model configuration required to build auxiliary data."""
+
+ conv_modulation: ModConvType
+ aux_in_features: int
+ log_aux: bool
+ auxiliary_features: Optional[tuple[str, ...]]
+
+
+def register_auxiliary_feature(feature: AuxiliaryFeature) -> None:
+ """Register a custom auxiliary feature for use in ``auxiliary_features`` configs."""
+ AUXILIARY_FEATURE_REGISTRY[feature.key] = feature
+
+
+def resolve_auxiliary_features(
+ feature_names: Optional[Sequence[str]],
+ aux_in_features: int,
+) -> tuple[AuxiliaryFeature, ...]:
+ """Resolve auxiliary feature names from config into feature metadata.
+
+ Parameters
+ ----------
+ feature_names : Sequence[str], optional
+ Explicit ordered list of feature keys. When ``None``, the first ``aux_in_features`` entries
+ from :data:`DEFAULT_AUXILIARY_FEATURE_NAMES` are used.
+ aux_in_features : int
+ Expected number of auxiliary channels. Must match the resolved list length.
+
+ Returns
+ -------
+ tuple[AuxiliaryFeature, ...]
+ Resolved feature metadata in request order.
+
+ Raises
+ ------
+ ValueError
+ If feature names are unknown, or the list length does not match ``aux_in_features``.
+ """
+ if aux_in_features <= 0:
+ raise ValueError(f"aux_in_features must be positive, got {aux_in_features}.")
+
+ if feature_names is None:
+ if aux_in_features > len(DEFAULT_AUXILIARY_FEATURE_NAMES):
+ raise ValueError(
+ f"aux_in_features={aux_in_features} exceeds the number of default auxiliary "
+ f"features ({len(DEFAULT_AUXILIARY_FEATURE_NAMES)}): "
+ f"{list(DEFAULT_AUXILIARY_FEATURE_NAMES)}. "
+ f"Set auxiliary_features explicitly in the model config."
+ )
+ names = DEFAULT_AUXILIARY_FEATURE_NAMES[:aux_in_features]
+ else:
+ names = tuple(feature_names)
+ if len(names) != aux_in_features:
+ raise ValueError(
+ f"auxiliary_features has length {len(names)} ({list(names)}) but aux_in_features={aux_in_features}."
+ )
+
+ unknown = sorted(set(names) - AUXILIARY_FEATURE_REGISTRY.keys())
+ if unknown:
+ raise ValueError(
+ f"Unknown auxiliary feature(s): {unknown}. Known features: {sorted(AUXILIARY_FEATURE_REGISTRY)}."
+ )
+
+ return tuple(AUXILIARY_FEATURE_REGISTRY[name] for name in names)
+
+
+def _needs_auxiliary_data(cfg: Any) -> bool:
+ """Return whether the model config requires auxiliary conditioning vectors."""
+ if not hasattr(cfg, "conv_modulation"):
+ return False
+
+ conv_modulation = cfg.conv_modulation
+ if isinstance(conv_modulation, str):
+ conv_modulation = ModConvType.from_str(conv_modulation) or ModConvType.NONE
+
+ if conv_modulation != ModConvType.NONE:
+ return True
+
+ for attr in ("image_unet_norm_type", "unet_norm_type"):
+ norm_type = getattr(cfg, attr, None)
+ if norm_type is None:
+ continue
+ if isinstance(norm_type, str) and norm_type.upper() == "ADAIN":
+ return True
+ if getattr(norm_type, "name", "").upper() == "ADAIN":
+ return True
+ if norm_type == NormType.ADAIN:
+ return True
+
+ return False
+
+
+def prepare_auxiliary_data(
+ data: Mapping[str, Any],
+ cfg: Optional[ModulationConfig],
+ *,
+ features: Optional[Sequence[AuxiliaryFeature]] = None,
+) -> Optional[torch.Tensor]:
+ """Build an auxiliary conditioning vector for modulated models.
+
+ Parameters
+ ----------
+ data : Mapping[str, Any]
+ Batch dictionary containing the auxiliary feature tensors.
+ cfg : ModulationConfig
+ Model configuration with modulation settings. Uses ``cfg.auxiliary_features`` when
+ ``features`` is not provided.
+ features : Sequence[AuxiliaryFeature], optional
+ Explicit feature list, mainly for testing. Overrides ``cfg.auxiliary_features``.
+
+ Returns
+ -------
+ torch.Tensor or None
+ Tensor of shape ``(batch_size, aux_in_features)``, or ``None`` when modulation is disabled.
+
+ Raises
+ ------
+ ValueError
+ If auxiliary configuration is invalid or a feature tensor has an unexpected shape.
+ KeyError
+ If a required auxiliary feature is missing from ``data``.
+ """
+ if cfg is None or not _needs_auxiliary_data(cfg):
+ return None
+
+ log_aux = getattr(cfg, "log_aux", False)
+ auxiliary_features = getattr(cfg, "auxiliary_features", None)
+ aux_in_features = getattr(cfg, "aux_in_features", None)
+ if aux_in_features is None:
+ return None
+
+ selected_features = (
+ tuple(features) if features is not None else resolve_auxiliary_features(auxiliary_features, aux_in_features)
+ )
+
+ components = [_prepare_feature(data, feature, log_aux=log_aux) for feature in selected_features]
+ auxiliary_data = torch.cat(components, dim=1)
+
+ if log_aux:
+ auxiliary_data = auxiliary_data.log()
+
+ return auxiliary_data
+
+
+def _prepare_feature(data: Mapping[str, Any], feature: AuxiliaryFeature, *, log_aux: bool) -> torch.Tensor:
+ if feature.key not in data:
+ raise KeyError(
+ f"Missing auxiliary feature '{feature.key}' required for modulation. Available keys: {sorted(data.keys())}."
+ )
+
+ tensor = _to_aux_column(data[feature.key])
+ if log_aux and feature.log_scale != 1.0:
+ tensor = tensor * feature.log_scale
+ return tensor
+
+
+def _to_aux_column(tensor: torch.Tensor) -> torch.Tensor:
+ """Convert a scalar auxiliary value to shape ``(batch_size, 1)``."""
+ tensor = tensor.float()
+
+ if tensor.ndim == 0:
+ return tensor.reshape(1, 1)
+ if tensor.ndim == 1:
+ return tensor.unsqueeze(-1)
+ if tensor.ndim == 2 and tensor.shape[-1] == 1:
+ return tensor
+
+ raise ValueError(
+ f"Expected auxiliary feature with shape (batch_size,) or (batch_size, 1), got {tuple(tensor.shape)}."
+ )
diff --git a/direct/nn/conv/modulated/factory.py b/direct/nn/conv/modulated/factory.py
new file mode 100644
index 00000000..75f4c216
--- /dev/null
+++ b/direct/nn/conv/modulated/factory.py
@@ -0,0 +1,175 @@
+# Copyright 2026 AI for Oncology Research Group. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Typed helpers for constructing modulated convolution layers."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Optional
+
+from direct.nn.conv.modulated.modulated_conv import (
+ ModConv2d,
+ ModConv2dBias,
+ ModConv3d,
+ ModConvActivation,
+ ModConvTranspose2d,
+ ModConvTranspose3d,
+ ModConvType,
+)
+from direct.types import IntOrTuple
+
+__all__ = [
+ "ModulationParams",
+ "mod_conv2d",
+ "mod_conv3d",
+ "mod_conv_transpose2d",
+ "mod_conv_transpose3d",
+]
+
+
+def mod_conv2d(
+ in_channels: int,
+ out_channels: int,
+ *,
+ kernel_size: IntOrTuple,
+ modulation_params: Optional[ModulationParams] = None,
+ stride: IntOrTuple = 1,
+ padding: IntOrTuple = 0,
+ dilation: IntOrTuple = 1,
+ bias: ModConv2dBias = ModConv2dBias.PARAM,
+) -> ModConv2d:
+ """See :class:`ModConv2d`."""
+ params = modulation_params or ModulationParams()
+ return ModConv2d(
+ in_channels,
+ out_channels,
+ kernel_size=kernel_size,
+ stride=stride,
+ padding=padding,
+ dilation=dilation,
+ bias=bias,
+ modulation=params.modulation,
+ aux_in_features=params.aux_in_features,
+ fc_hidden_features=params.fc_hidden_features,
+ fc_bias=params.fc_bias,
+ fc_groups=params.fc_groups,
+ fc_activation=params.fc_activation,
+ num_weights=params.num_weights,
+ )
+
+
+def mod_conv_transpose2d(
+ in_channels: int,
+ out_channels: int,
+ *,
+ kernel_size: IntOrTuple,
+ modulation_params: Optional[ModulationParams] = None,
+ stride: IntOrTuple = 1,
+ padding: IntOrTuple = 0,
+ dilation: IntOrTuple = 1,
+ bias: ModConv2dBias = ModConv2dBias.PARAM,
+) -> ModConvTranspose2d:
+ """See :class:`ModConvTranspose2d`."""
+ params = modulation_params or ModulationParams()
+ return ModConvTranspose2d(
+ in_channels,
+ out_channels,
+ kernel_size=kernel_size,
+ stride=stride,
+ padding=padding,
+ dilation=dilation,
+ bias=bias,
+ modulation=params.modulation,
+ aux_in_features=params.aux_in_features,
+ fc_hidden_features=params.fc_hidden_features,
+ fc_bias=params.fc_bias,
+ fc_groups=params.fc_groups,
+ fc_activation=params.fc_activation,
+ num_weights=params.num_weights,
+ )
+
+
+def mod_conv3d(
+ in_channels: int,
+ out_channels: int,
+ *,
+ kernel_size: IntOrTuple,
+ modulation_params: Optional[ModulationParams] = None,
+ stride: IntOrTuple = 1,
+ padding: IntOrTuple = 0,
+ dilation: IntOrTuple = 1,
+ bias: ModConv2dBias = ModConv2dBias.PARAM,
+) -> ModConv3d:
+ """See :class:`ModConv3d`."""
+ params = modulation_params or ModulationParams()
+ return ModConv3d(
+ in_channels,
+ out_channels,
+ kernel_size=kernel_size,
+ stride=stride,
+ padding=padding,
+ dilation=dilation,
+ bias=bias,
+ modulation=params.modulation,
+ aux_in_features=params.aux_in_features,
+ fc_hidden_features=params.fc_hidden_features,
+ fc_bias=params.fc_bias,
+ fc_groups=params.fc_groups,
+ fc_activation=params.fc_activation,
+ num_weights=params.num_weights,
+ )
+
+
+def mod_conv_transpose3d(
+ in_channels: int,
+ out_channels: int,
+ *,
+ kernel_size: IntOrTuple,
+ modulation_params: Optional[ModulationParams] = None,
+ stride: IntOrTuple = 1,
+ padding: IntOrTuple = 0,
+ dilation: IntOrTuple = 1,
+ bias: ModConv2dBias = ModConv2dBias.PARAM,
+) -> ModConvTranspose3d:
+ """See :class:`ModConvTranspose3d`."""
+ params = modulation_params or ModulationParams()
+ return ModConvTranspose3d(
+ in_channels,
+ out_channels,
+ kernel_size=kernel_size,
+ stride=stride,
+ padding=padding,
+ dilation=dilation,
+ bias=bias,
+ modulation=params.modulation,
+ aux_in_features=params.aux_in_features,
+ fc_hidden_features=params.fc_hidden_features,
+ fc_bias=params.fc_bias,
+ fc_groups=params.fc_groups,
+ fc_activation=params.fc_activation,
+ num_weights=params.num_weights,
+ )
+
+
+@dataclass(frozen=True)
+class ModulationParams:
+ """Shared modulation settings for modulated convolution layers."""
+
+ modulation: ModConvType = ModConvType.NONE
+ aux_in_features: Optional[int] = None
+ fc_hidden_features: Optional[int | tuple[int, ...]] = None
+ fc_groups: int = 1
+ fc_activation: ModConvActivation = ModConvActivation.SIGMOID
+ num_weights: Optional[int] = None
+ fc_bias: Optional[bool] = True
diff --git a/direct/nn/conv/modulated/modulated_conv.py b/direct/nn/conv/modulated/modulated_conv.py
new file mode 100644
index 00000000..af26a45d
--- /dev/null
+++ b/direct/nn/conv/modulated/modulated_conv.py
@@ -0,0 +1,1182 @@
+# Copyright 2026 AI for Oncology Research Group. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Modulated convolution layers based on [1]_.
+
+These layers extend standard convolutions with input-dependent weight modulation,
+allowing the network to dynamically adjust its convolutional filters based on an
+auxiliary signal (e.g., acceleration factor, coil information).
+
+References
+----------
+
+.. [1] Moriakov, N., Yiasemis, G., Sonke, J.-J. & Teuwen, J. (2026). Conditional Learned Reconstruction for
+ Medical Imaging. Proceedings of The 9th International Conference on Medical Imaging with Deep Learning,
+ PMLR 315:754-780. https://proceedings.mlr.press/v315/moriakov26a.html
+"""
+
+from __future__ import annotations
+
+import math
+from typing import Optional
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from direct.types import DirectEnum, IntOrTuple
+
+__all__ = [
+ "ModConv2d",
+ "ModConv2dBias",
+ "ModConvActivation",
+ "ModConvType",
+ "ModConvTranspose2d",
+ "ModConv3d",
+ "ModConvTranspose3d",
+]
+
+
+class ModConv2dBias(DirectEnum):
+ LEARNED = "learned"
+ PARAM = "param"
+ NONE = "none"
+
+
+class ModConvActivation(DirectEnum):
+ SIGMOID = "sigmoid"
+ SOFTPLUS = "softplus"
+
+
+class ModConvType(DirectEnum):
+ FEATURES = "features"
+ PARTIAL_IN = "partial_in"
+ PARTIAL_OUT = "partial_out"
+ FULL = "full"
+ SUM = "sum"
+ NONE = "none"
+
+
+def _as_hidden_tuple(features: int | tuple[int, ...]) -> tuple[int, ...]:
+ return (features,) if isinstance(features, int) else features
+
+
+def _resolved_fc_bias(fc_bias: Optional[bool]) -> bool:
+ return True if fc_bias is None else fc_bias
+
+
+def _build_modulation_mlp(
+ aux_in_features: int,
+ hidden_features: tuple[int, ...],
+ fc_activation: ModConvActivation,
+ fc_bias: bool,
+) -> nn.Sequential:
+ layers: list[nn.Module] = [nn.Linear(aux_in_features, hidden_features[0], bias=fc_bias)]
+ for i in range(len(hidden_features) - 1):
+ layers.append(nn.PReLU())
+ layers.append(nn.Linear(hidden_features[i], hidden_features[i + 1]))
+ if fc_activation == ModConvActivation.SIGMOID:
+ layers.append(nn.Sigmoid())
+ elif fc_activation == ModConvActivation.SOFTPLUS:
+ layers.append(nn.Softplus())
+ return nn.Sequential(*layers)
+
+
+def _build_learned_bias_mlp(
+ aux_in_features: int,
+ hidden_features: tuple[int, ...],
+ out_channels: int,
+ fc_bias: bool,
+) -> nn.Sequential:
+ bias_layers: list[nn.Module] = [nn.Linear(aux_in_features, hidden_features[0], bias=fc_bias)]
+ for i in range(len(hidden_features) - 1):
+ bias_layers.append(nn.PReLU())
+ bias_layers.append(
+ nn.Linear(
+ hidden_features[i],
+ hidden_features[i + 1] if i != (len(hidden_features) - 2) else out_channels,
+ )
+ )
+ return nn.Sequential(*bias_layers)
+
+
+class ModConv2d(nn.Module):
+ """Modulated 2D convolution based on [1]_.
+
+ When ``modulation`` is :attr:`ModConvType.NONE` and ``bias`` is :attr:`ModConv2dBias.PARAM`,
+ this behaves identically to :class:`torch.nn.Conv2d`.
+
+ When modulation is enabled, the convolutional weights are element-wise scaled by
+ an MLP-derived signal conditioned on an auxiliary input ``y``.
+
+ References
+ ----------
+
+ .. [1] Moriakov, N., Yiasemis, G., Sonke, J.-J. & Teuwen, J. (2026). Conditional Learned Reconstruction for
+ Medical Imaging. Proceedings of The 9th International Conference on Medical Imaging with Deep Learning,
+ PMLR 315:754-780. https://proceedings.mlr.press/v315/moriakov26a.html
+ """
+
+ def __init__(
+ self,
+ in_channels: int,
+ out_channels: int,
+ kernel_size: IntOrTuple,
+ modulation: ModConvType = ModConvType.NONE,
+ stride: IntOrTuple = 1,
+ padding: IntOrTuple = 0,
+ dilation: IntOrTuple = 1,
+ bias: ModConv2dBias = ModConv2dBias.PARAM,
+ aux_in_features: Optional[int] = None,
+ fc_hidden_features: Optional[int | tuple[int, ...]] = None,
+ fc_bias: Optional[bool] = True,
+ fc_groups: int | None = 1,
+ fc_activation: ModConvActivation = ModConvActivation.SIGMOID,
+ num_weights: Optional[int] = None,
+ ):
+ """Inits :class:`ModConv2d`.
+
+ Parameters
+ ----------
+ in_channels : int
+ Number of input channels.
+ out_channels : int
+ Number of output channels.
+ kernel_size : int or tuple of int
+ Size of the convolutional kernel.
+ modulation : ModConvType
+ Modulation type. Default: ModConvType.NONE.
+ stride : int or tuple of int
+ Stride of the convolution. Default: 1.
+ padding : int or tuple of int
+ Padding added to all sides of the input. Default: 0.
+ dilation : int or tuple of int
+ Spacing between kernel elements. Default: 1.
+ bias : ModConv2dBias
+ Type of bias. Default: ModConv2dBias.PARAM.
+ aux_in_features : int, optional
+ Number of features in the auxiliary input ``y``.
+ fc_hidden_features : int or tuple of int, optional
+ Hidden features in the modulation MLP.
+ fc_bias : bool, optional
+ Whether the modulation MLP uses bias. Default: True.
+ fc_groups : int or None
+ If > 1, the MLP output is divided by fc_groups^2 and expanded via nearest interpolation.
+ Default: 1.
+ fc_activation : ModConvActivation
+ Activation after the MLP. Default: ModConvActivation.SIGMOID.
+ num_weights : int, optional
+ Number of weight bases for ModConvType.SUM.
+ """
+ super().__init__()
+
+ self.kernel_size = (kernel_size, kernel_size) if isinstance(kernel_size, int) else kernel_size
+ self.stride = stride
+ self.padding = padding
+ self.dilation = dilation
+
+ self.in_channels = in_channels
+ self.out_channels = out_channels
+
+ self.modulation = modulation
+ self.aux_in_features = aux_in_features
+ self.fc_hidden_features = fc_hidden_features
+ self.fc_bias = fc_bias
+ self.fc_groups = 1 if fc_groups is None else fc_groups
+ self.fc_activation = fc_activation
+ self.num_weights = num_weights
+
+ mod_hidden: tuple[int, ...] | None = None
+ if modulation != ModConvType.NONE:
+ if aux_in_features is None:
+ raise ValueError("aux_in_features cannot be None when modulation is enabled.")
+ if fc_hidden_features is None:
+ raise ValueError("fc_hidden_features cannot be None when modulation is enabled.")
+ if fc_groups is None:
+ raise ValueError("fc_groups cannot be None when modulation is enabled.")
+ if fc_groups < 1:
+ raise ValueError("fc_groups must be >= 1.")
+
+ hidden = _as_hidden_tuple(fc_hidden_features)
+
+ if modulation == ModConvType.FEATURES:
+ mod_out_features = (out_channels // fc_groups) * (in_channels // fc_groups)
+ elif modulation == ModConvType.FULL:
+ mod_out_features = (
+ (out_channels // fc_groups) * (in_channels // fc_groups) * self.kernel_size[0] * self.kernel_size[1]
+ )
+ elif modulation == ModConvType.PARTIAL_OUT:
+ mod_out_features = self.kernel_size[0] * self.kernel_size[1] * (out_channels // fc_groups)
+ elif modulation == ModConvType.PARTIAL_IN:
+ mod_out_features = self.kernel_size[0] * self.kernel_size[1] * (in_channels // fc_groups)
+ else:
+ if (num_weights is None) or (num_weights < 1):
+ raise ValueError(f"ModConvType.SUM requires num_weights >= 1, got {num_weights}.")
+ mod_out_features = num_weights
+
+ mod_hidden = hidden + (mod_out_features,)
+ self.fc = _build_modulation_mlp(
+ aux_in_features,
+ mod_hidden,
+ fc_activation,
+ _resolved_fc_bias(fc_bias),
+ )
+
+ weight_shape = (out_channels, in_channels, *self.kernel_size)
+ if modulation == ModConvType.SUM:
+ weight_shape = (num_weights,) + weight_shape
+ k = math.sqrt(1 / (in_channels * self.kernel_size[0] * self.kernel_size[1]))
+ self.weight = nn.Parameter(torch.FloatTensor(*weight_shape).uniform_(-k, k))
+
+ self.bias_type = bias
+ if bias == ModConv2dBias.PARAM:
+ self.bias = nn.Parameter(torch.FloatTensor(out_channels).uniform_(-k, k))
+ elif bias == ModConv2dBias.LEARNED:
+ if mod_hidden is None or aux_in_features is None:
+ raise ValueError("ModConv2dBias.LEARNED requires modulation to be enabled.")
+ self.bias = _build_learned_bias_mlp(
+ aux_in_features,
+ mod_hidden,
+ out_channels,
+ _resolved_fc_bias(fc_bias),
+ )
+ else:
+ self.bias = None
+
+ def __repr__(self):
+ return (
+ f"ModConv2d(in_channels={self.in_channels}, out_channels={self.out_channels}, "
+ f"kernel_size={self.kernel_size}, modulation={self.modulation}, "
+ f"stride={self.stride}, padding={self.padding}, "
+ f"dilation={self.dilation}, bias={self.bias_type})"
+ )
+
+ def forward(self, x: torch.Tensor, y: Optional[torch.Tensor] = None) -> torch.Tensor:
+ """Forward pass.
+
+ Parameters
+ ----------
+ x : torch.Tensor
+ Input of shape (N, in_channels, H, W).
+ y : torch.Tensor, optional
+ Auxiliary signal of shape (N, aux_in_features).
+
+ Returns
+ -------
+ torch.Tensor
+ Output of shape (N, out_channels, H_out, W_out).
+ """
+ if self.modulation == ModConvType.NONE:
+ out = F.conv2d(
+ x,
+ self.weight,
+ stride=self.stride,
+ padding=self.padding,
+ dilation=self.dilation,
+ )
+ else:
+ fc_out = self.fc(y)
+
+ if self.modulation == ModConvType.SUM:
+ weight = (fc_out.view(x.shape[0], -1, 1, 1, 1, 1) * self.weight).sum(1)
+ else:
+ if self.modulation == ModConvType.FEATURES:
+ if self.fc_groups > 1:
+ fc_out = fc_out.view(
+ x.shape[0],
+ 1,
+ self.out_channels // self.fc_groups,
+ self.in_channels // self.fc_groups,
+ )
+ fc_out = F.interpolate(
+ fc_out,
+ size=(self.out_channels, self.in_channels),
+ mode="nearest",
+ )
+ fc_out = fc_out.view(x.shape[0], self.out_channels, self.in_channels, 1, 1)
+
+ elif self.modulation == ModConvType.FULL:
+ if self.fc_groups > 1:
+ fc_out = fc_out.view(
+ x.shape[0],
+ self.kernel_size[0],
+ self.kernel_size[1],
+ self.out_channels // self.fc_groups,
+ self.in_channels // self.fc_groups,
+ )
+ fc_out = F.interpolate(
+ fc_out,
+ size=(
+ self.kernel_size[1],
+ self.out_channels,
+ self.in_channels,
+ ),
+ mode="nearest",
+ )
+ fc_out = fc_out.permute(0, 3, 4, 1, 2)
+ fc_out = fc_out.view(
+ x.shape[0],
+ self.out_channels,
+ self.in_channels,
+ self.kernel_size[0],
+ self.kernel_size[1],
+ )
+
+ elif self.modulation == ModConvType.PARTIAL_OUT:
+ if self.fc_groups > 1:
+ fc_out = fc_out.view(
+ x.shape[0],
+ self.kernel_size[0],
+ self.kernel_size[1],
+ self.out_channels // self.fc_groups,
+ 1,
+ )
+ fc_out = F.interpolate(
+ fc_out,
+ size=(self.kernel_size[1], self.out_channels, 1),
+ mode="nearest",
+ )
+ fc_out = fc_out.permute(0, 3, 4, 1, 2)
+ fc_out = fc_out.view(
+ x.shape[0],
+ self.out_channels,
+ 1,
+ self.kernel_size[0],
+ self.kernel_size[1],
+ )
+
+ else: # PARTIAL_IN
+ if self.fc_groups > 1:
+ fc_out = fc_out.view(
+ x.shape[0],
+ self.kernel_size[0],
+ self.kernel_size[1],
+ 1,
+ self.in_channels // self.fc_groups,
+ )
+ fc_out = F.interpolate(
+ fc_out,
+ size=(self.kernel_size[1], 1, self.in_channels),
+ mode="nearest",
+ )
+ fc_out = fc_out.permute(0, 3, 4, 1, 2)
+ fc_out = fc_out.view(
+ x.shape[0],
+ 1,
+ self.in_channels,
+ self.kernel_size[0],
+ self.kernel_size[1],
+ )
+
+ weight = fc_out * self.weight
+
+ out = torch.cat(
+ [
+ F.conv2d(
+ x[i : i + 1],
+ weight[i],
+ stride=self.stride,
+ padding=self.padding,
+ dilation=self.dilation,
+ )
+ for i in range(x.shape[0])
+ ],
+ 0,
+ )
+
+ if self.bias is not None:
+ if isinstance(self.bias, nn.parameter.Parameter):
+ bias = self.bias.view(1, -1, 1, 1)
+ else:
+ bias = self.bias(y).view(x.shape[0], -1, 1, 1)
+ out = out + bias
+
+ return out
+
+
+class ModConvTranspose2d(nn.Module):
+ """Modulated 2D transposed convolution.
+
+ Transpose variant of :class:`ModConv2d` supporting the same modulation types.
+ """
+
+ def __init__(
+ self,
+ in_channels: int,
+ out_channels: int,
+ kernel_size: IntOrTuple,
+ modulation: ModConvType = ModConvType.NONE,
+ stride: IntOrTuple = 1,
+ padding: IntOrTuple = 0,
+ dilation: IntOrTuple = 1,
+ bias: ModConv2dBias = ModConv2dBias.PARAM,
+ aux_in_features: Optional[int] = None,
+ fc_hidden_features: Optional[int | tuple[int, ...]] = None,
+ fc_bias: Optional[bool] = True,
+ fc_groups: int | None = 1,
+ fc_activation: ModConvActivation = ModConvActivation.SIGMOID,
+ num_weights: Optional[int] = None,
+ ):
+ super().__init__()
+
+ self.kernel_size = (kernel_size, kernel_size) if isinstance(kernel_size, int) else kernel_size
+ self.stride = stride
+ self.padding = padding
+ self.dilation = dilation
+
+ self.in_channels = in_channels
+ self.out_channels = out_channels
+
+ self.modulation = modulation
+ self.aux_in_features = aux_in_features
+ self.fc_hidden_features = fc_hidden_features
+ self.fc_bias = fc_bias
+ self.fc_groups = 1 if fc_groups is None else fc_groups
+ self.fc_activation = fc_activation
+ self.num_weights = num_weights
+
+ mod_hidden: tuple[int, ...] | None = None
+ if modulation != ModConvType.NONE:
+ if aux_in_features is None:
+ raise ValueError("aux_in_features cannot be None when modulation is enabled.")
+ if fc_hidden_features is None:
+ raise ValueError("fc_hidden_features cannot be None when modulation is enabled.")
+ if fc_groups is None:
+ raise ValueError("fc_groups cannot be None when modulation is enabled.")
+ if fc_groups < 1:
+ raise ValueError("fc_groups must be >= 1.")
+
+ hidden = _as_hidden_tuple(fc_hidden_features)
+
+ if modulation == ModConvType.FEATURES:
+ mod_out_features = (in_channels // fc_groups) * (out_channels // fc_groups)
+ elif modulation == ModConvType.FULL:
+ mod_out_features = (
+ (in_channels // fc_groups) * (out_channels // fc_groups) * self.kernel_size[0] * self.kernel_size[1]
+ )
+ elif modulation == ModConvType.PARTIAL_OUT:
+ mod_out_features = self.kernel_size[0] * self.kernel_size[1] * (out_channels // fc_groups)
+ elif modulation == ModConvType.PARTIAL_IN:
+ mod_out_features = self.kernel_size[0] * self.kernel_size[1] * (in_channels // fc_groups)
+ else:
+ if (num_weights is None) or (num_weights < 1):
+ raise ValueError(f"ModConvType.SUM requires num_weights >= 1, got {num_weights}.")
+ mod_out_features = num_weights
+
+ mod_hidden = hidden + (mod_out_features,)
+ self.fc = _build_modulation_mlp(
+ aux_in_features,
+ mod_hidden,
+ fc_activation,
+ _resolved_fc_bias(fc_bias),
+ )
+
+ weight_shape = (in_channels, out_channels, *self.kernel_size)
+ if modulation == ModConvType.SUM:
+ weight_shape = (num_weights,) + weight_shape
+ k = math.sqrt(1 / (out_channels * self.kernel_size[0] * self.kernel_size[1]))
+ self.weight = nn.Parameter(torch.FloatTensor(*weight_shape).uniform_(-k, k))
+
+ self.bias_type = bias
+ if bias == ModConv2dBias.PARAM:
+ self.bias = nn.Parameter(torch.FloatTensor(out_channels).uniform_(-k, k))
+ elif bias == ModConv2dBias.LEARNED:
+ if mod_hidden is None or aux_in_features is None:
+ raise ValueError("ModConv2dBias.LEARNED requires modulation to be enabled.")
+ self.bias = _build_learned_bias_mlp(
+ aux_in_features,
+ mod_hidden,
+ out_channels,
+ _resolved_fc_bias(fc_bias),
+ )
+ else:
+ self.bias = None
+
+ def __repr__(self):
+ return (
+ f"ModConvTranspose2d(in_channels={self.in_channels}, out_channels={self.out_channels}, "
+ f"kernel_size={self.kernel_size}, modulation={self.modulation}, "
+ f"stride={self.stride}, padding={self.padding}, dilation={self.dilation}, bias={self.bias_type})"
+ )
+
+ def forward(self, x: torch.Tensor, y: Optional[torch.Tensor] = None) -> torch.Tensor:
+ """Forward pass.
+
+ Parameters
+ ----------
+ x : torch.Tensor
+ Input of shape (N, in_channels, H, W).
+ y : torch.Tensor, optional
+ Auxiliary signal of shape (N, aux_in_features).
+
+ Returns
+ -------
+ torch.Tensor
+ Output of shape (N, out_channels, H_out, W_out).
+ """
+ if self.modulation == ModConvType.NONE:
+ out = F.conv_transpose2d(
+ x,
+ self.weight,
+ stride=self.stride,
+ padding=self.padding,
+ dilation=self.dilation,
+ )
+ else:
+ fc_out = self.fc(y)
+
+ if self.modulation == ModConvType.SUM:
+ weight = (fc_out.view(x.shape[0], -1, 1, 1, 1, 1) * self.weight).sum(1)
+ else:
+ if self.modulation == ModConvType.FEATURES:
+ if self.fc_groups > 1:
+ fc_out = fc_out.view(
+ x.shape[0],
+ 1,
+ self.in_channels // self.fc_groups,
+ self.out_channels // self.fc_groups,
+ )
+ fc_out = F.interpolate(
+ fc_out,
+ size=(self.in_channels, self.out_channels),
+ mode="nearest",
+ )
+ fc_out = fc_out.view(x.shape[0], self.in_channels, self.out_channels, 1, 1)
+
+ elif self.modulation == ModConvType.FULL:
+ if self.fc_groups > 1:
+ fc_out = fc_out.view(
+ x.shape[0],
+ self.kernel_size[0],
+ self.kernel_size[1],
+ self.in_channels // self.fc_groups,
+ self.out_channels // self.fc_groups,
+ )
+ fc_out = F.interpolate(
+ fc_out,
+ size=(
+ self.kernel_size[1],
+ self.in_channels,
+ self.out_channels,
+ ),
+ mode="nearest",
+ )
+ fc_out = fc_out.permute(0, 3, 4, 1, 2)
+ fc_out = fc_out.view(
+ x.shape[0],
+ self.in_channels,
+ self.out_channels,
+ self.kernel_size[0],
+ self.kernel_size[1],
+ )
+
+ elif self.modulation == ModConvType.PARTIAL_OUT:
+ if self.fc_groups > 1:
+ fc_out = fc_out.view(
+ x.shape[0],
+ self.kernel_size[0],
+ self.kernel_size[1],
+ 1,
+ self.out_channels // self.fc_groups,
+ )
+ fc_out = F.interpolate(
+ fc_out,
+ size=(self.kernel_size[1], 1, self.out_channels),
+ mode="nearest",
+ )
+ fc_out = fc_out.permute(0, 3, 4, 1, 2)
+ fc_out = fc_out.view(
+ x.shape[0],
+ 1,
+ self.out_channels,
+ self.kernel_size[0],
+ self.kernel_size[1],
+ )
+
+ else: # PARTIAL_IN
+ if self.fc_groups > 1:
+ fc_out = fc_out.view(
+ x.shape[0],
+ self.kernel_size[0],
+ self.kernel_size[1],
+ self.in_channels // self.fc_groups,
+ 1,
+ )
+ fc_out = F.interpolate(
+ fc_out,
+ size=(self.kernel_size[1], self.in_channels, 1),
+ mode="nearest",
+ )
+ fc_out = fc_out.permute(0, 3, 4, 1, 2)
+ fc_out = fc_out.view(
+ x.shape[0],
+ self.in_channels,
+ 1,
+ self.kernel_size[0],
+ self.kernel_size[1],
+ )
+
+ weight = fc_out * self.weight
+
+ out = torch.cat(
+ [
+ F.conv_transpose2d(
+ x[i : i + 1],
+ weight[i],
+ stride=self.stride,
+ padding=self.padding,
+ dilation=self.dilation,
+ )
+ for i in range(x.shape[0])
+ ],
+ 0,
+ )
+
+ if self.bias is not None:
+ if isinstance(self.bias, nn.parameter.Parameter):
+ bias = self.bias.view(1, -1, 1, 1)
+ else:
+ bias = self.bias(y).view(x.shape[0], -1, 1, 1)
+ out = out + bias
+
+ return out
+
+
+class ModConv3d(nn.Module):
+ """Modulated 3D convolution.
+
+ 3D extension of :class:`ModConv2d` supporting the same modulation types.
+ """
+
+ def __init__(
+ self,
+ in_channels: int,
+ out_channels: int,
+ kernel_size: IntOrTuple,
+ modulation: ModConvType = ModConvType.NONE,
+ stride: IntOrTuple = 1,
+ padding: IntOrTuple = 0,
+ dilation: IntOrTuple = 1,
+ bias: ModConv2dBias = ModConv2dBias.PARAM,
+ aux_in_features: Optional[int] = None,
+ fc_hidden_features: Optional[int | tuple[int, ...]] = None,
+ fc_bias: Optional[bool] = True,
+ fc_groups: int | None = 1,
+ fc_activation: ModConvActivation = ModConvActivation.SIGMOID,
+ num_weights: Optional[int] = None,
+ ):
+ super().__init__()
+
+ if isinstance(kernel_size, int):
+ self.kernel_size = (kernel_size, kernel_size, kernel_size)
+ elif len(kernel_size) == 2:
+ self.kernel_size = (kernel_size[0], kernel_size[0], kernel_size[1])
+ else:
+ self.kernel_size = kernel_size
+ self.stride = stride
+ self.padding = padding
+ self.dilation = dilation
+
+ self.in_channels = in_channels
+ self.out_channels = out_channels
+
+ self.modulation = modulation
+ self.aux_in_features = aux_in_features
+ self.fc_hidden_features = fc_hidden_features
+ self.fc_bias = fc_bias
+ self.fc_groups = 1 if fc_groups is None else fc_groups
+ self.fc_activation = fc_activation
+ self.num_weights = num_weights
+
+ mod_hidden: tuple[int, ...] | None = None
+ if modulation != ModConvType.NONE:
+ if aux_in_features is None:
+ raise ValueError("aux_in_features cannot be None when modulation is enabled.")
+ if fc_hidden_features is None:
+ raise ValueError("fc_hidden_features cannot be None when modulation is enabled.")
+ if fc_groups is None:
+ raise ValueError("fc_groups cannot be None when modulation is enabled.")
+ if fc_groups < 1:
+ raise ValueError("fc_groups must be >= 1.")
+
+ hidden = _as_hidden_tuple(fc_hidden_features)
+
+ if modulation == ModConvType.FEATURES:
+ mod_out_features = (out_channels // fc_groups) * (in_channels // fc_groups)
+ elif modulation == ModConvType.FULL:
+ mod_out_features = (
+ (out_channels // fc_groups)
+ * (in_channels // fc_groups)
+ * self.kernel_size[0]
+ * self.kernel_size[1]
+ * self.kernel_size[2]
+ )
+ elif modulation == ModConvType.PARTIAL_OUT:
+ mod_out_features = (
+ self.kernel_size[0] * self.kernel_size[1] * self.kernel_size[2] * (out_channels // fc_groups)
+ )
+ elif modulation == ModConvType.PARTIAL_IN:
+ mod_out_features = (
+ self.kernel_size[0] * self.kernel_size[1] * self.kernel_size[2] * (in_channels // fc_groups)
+ )
+ else:
+ if (num_weights is None) or (num_weights < 1):
+ raise ValueError(f"ModConvType.SUM requires num_weights >= 1, got {num_weights}.")
+ mod_out_features = num_weights
+
+ mod_hidden = hidden + (mod_out_features,)
+ self.fc = _build_modulation_mlp(
+ aux_in_features,
+ mod_hidden,
+ fc_activation,
+ _resolved_fc_bias(fc_bias),
+ )
+
+ weight_shape = (out_channels, in_channels, *self.kernel_size)
+ if modulation == ModConvType.SUM:
+ weight_shape = (num_weights,) + weight_shape
+ k = math.sqrt(1 / (in_channels * self.kernel_size[0] * self.kernel_size[1] * self.kernel_size[2]))
+ self.weight = nn.Parameter(torch.FloatTensor(*weight_shape).uniform_(-k, k))
+
+ self.bias_type = bias
+ if bias == ModConv2dBias.PARAM:
+ self.bias = nn.Parameter(torch.FloatTensor(out_channels).uniform_(-k, k))
+ elif bias == ModConv2dBias.LEARNED:
+ if mod_hidden is None or aux_in_features is None:
+ raise ValueError("ModConv2dBias.LEARNED requires modulation to be enabled.")
+ self.bias = _build_learned_bias_mlp(
+ aux_in_features,
+ mod_hidden,
+ out_channels,
+ _resolved_fc_bias(fc_bias),
+ )
+ else:
+ self.bias = None
+
+ def __repr__(self):
+ return (
+ f"ModConv3d(in_channels={self.in_channels}, out_channels={self.out_channels}, "
+ f"kernel_size={self.kernel_size}, modulation={self.modulation}, "
+ f"stride={self.stride}, padding={self.padding}, dilation={self.dilation}, bias={self.bias_type})"
+ )
+
+ def forward(self, x: torch.Tensor, y: Optional[torch.Tensor] = None) -> torch.Tensor:
+ """Forward pass.
+
+ Parameters
+ ----------
+ x : torch.Tensor
+ Input of shape (N, in_channels, D, H, W).
+ y : torch.Tensor, optional
+ Auxiliary signal of shape (N, aux_in_features).
+
+ Returns
+ -------
+ torch.Tensor
+ Output of shape (N, out_channels, D_out, H_out, W_out).
+ """
+ if self.modulation == ModConvType.NONE:
+ out = F.conv3d(
+ x,
+ self.weight,
+ stride=self.stride,
+ padding=self.padding,
+ dilation=self.dilation,
+ )
+ else:
+ fc_out = self.fc(y)
+
+ if self.modulation == ModConvType.SUM:
+ weight = (fc_out.view(x.shape[0], -1, 1, 1, 1, 1, 1) * self.weight).sum(1)
+ else:
+ if self.modulation == ModConvType.FEATURES:
+ if self.fc_groups > 1:
+ fc_out = fc_out.view(
+ x.shape[0],
+ 1,
+ self.out_channels // self.fc_groups,
+ self.in_channels // self.fc_groups,
+ )
+ fc_out = F.interpolate(
+ fc_out,
+ size=(self.out_channels, self.in_channels),
+ mode="nearest",
+ )
+ fc_out = fc_out.view(x.shape[0], self.out_channels, self.in_channels, 1, 1, 1)
+
+ elif self.modulation == ModConvType.FULL:
+ if self.fc_groups > 1:
+ fc_out = fc_out.view(
+ x.shape[0],
+ self.kernel_size[0],
+ self.kernel_size[1],
+ self.kernel_size[2],
+ self.out_channels // self.fc_groups,
+ self.in_channels // self.fc_groups,
+ )
+ fc_out = fc_out.permute(0, 4, 5, 1, 2, 3)
+ out_ch_expand = self.out_channels // self.fc_groups
+ in_ch_expand = self.in_channels // self.fc_groups
+ if out_ch_expand < self.out_channels or in_ch_expand < self.in_channels:
+ fc_out = fc_out.repeat(1, self.fc_groups, self.fc_groups, 1, 1, 1)
+ fc_out = fc_out[:, : self.out_channels, : self.in_channels, :, :, :]
+ fc_out = fc_out.view(
+ x.shape[0],
+ self.out_channels,
+ self.in_channels,
+ self.kernel_size[0],
+ self.kernel_size[1],
+ self.kernel_size[2],
+ )
+ else:
+ fc_out = fc_out.view(
+ x.shape[0],
+ self.out_channels,
+ self.in_channels,
+ self.kernel_size[0],
+ self.kernel_size[1],
+ self.kernel_size[2],
+ )
+
+ elif self.modulation == ModConvType.PARTIAL_OUT:
+ if self.fc_groups > 1:
+ fc_out = fc_out.view(
+ x.shape[0],
+ self.kernel_size[0],
+ self.kernel_size[1],
+ self.kernel_size[2],
+ self.out_channels // self.fc_groups,
+ 1,
+ )
+ fc_out = fc_out.permute(0, 4, 5, 1, 2, 3)
+ fc_out = fc_out.view(
+ x.shape[0],
+ self.out_channels,
+ 1,
+ self.kernel_size[0],
+ self.kernel_size[1],
+ self.kernel_size[2],
+ )
+
+ else: # PARTIAL_IN
+ if self.fc_groups > 1:
+ fc_out = fc_out.view(
+ x.shape[0],
+ self.kernel_size[0],
+ self.kernel_size[1],
+ self.kernel_size[2],
+ 1,
+ self.in_channels // self.fc_groups,
+ )
+ fc_out = fc_out.permute(0, 4, 5, 1, 2, 3)
+ fc_out = fc_out.view(
+ x.shape[0],
+ 1,
+ self.in_channels,
+ self.kernel_size[0],
+ self.kernel_size[1],
+ self.kernel_size[2],
+ )
+
+ weight = fc_out * self.weight
+
+ out = torch.cat(
+ [
+ F.conv3d(
+ x[i : i + 1],
+ weight[i],
+ stride=self.stride,
+ padding=self.padding,
+ dilation=self.dilation,
+ )
+ for i in range(x.shape[0])
+ ],
+ 0,
+ )
+
+ if self.bias is not None:
+ if isinstance(self.bias, nn.parameter.Parameter):
+ bias = self.bias.view(1, -1, 1, 1, 1)
+ else:
+ bias = self.bias(y).view(x.shape[0], -1, 1, 1, 1)
+ out = out + bias
+
+ return out
+
+
+class ModConvTranspose3d(nn.Module):
+ """Modulated 3D transposed convolution.
+
+ 3D transpose variant of :class:`ModConv3d`.
+ """
+
+ def __init__(
+ self,
+ in_channels: int,
+ out_channels: int,
+ kernel_size: IntOrTuple,
+ modulation: ModConvType = ModConvType.NONE,
+ stride: IntOrTuple = 1,
+ padding: IntOrTuple = 0,
+ dilation: IntOrTuple = 1,
+ bias: ModConv2dBias = ModConv2dBias.PARAM,
+ aux_in_features: Optional[int] = None,
+ fc_hidden_features: Optional[int | tuple[int, ...]] = None,
+ fc_bias: Optional[bool] = True,
+ fc_groups: int | None = 1,
+ fc_activation: ModConvActivation = ModConvActivation.SIGMOID,
+ num_weights: Optional[int] = None,
+ ):
+ super().__init__()
+
+ if isinstance(kernel_size, int):
+ self.kernel_size = (kernel_size, kernel_size, kernel_size)
+ elif len(kernel_size) == 2:
+ self.kernel_size = (kernel_size[0], kernel_size[0], kernel_size[1])
+ else:
+ self.kernel_size = kernel_size
+ self.stride = stride
+ self.padding = padding
+ self.dilation = dilation
+
+ self.in_channels = in_channels
+ self.out_channels = out_channels
+
+ self.modulation = modulation
+ self.aux_in_features = aux_in_features
+ self.fc_hidden_features = fc_hidden_features
+ self.fc_bias = fc_bias
+ self.fc_groups = 1 if fc_groups is None else fc_groups
+ self.fc_activation = fc_activation
+ self.num_weights = num_weights
+
+ mod_hidden: tuple[int, ...] | None = None
+ if modulation != ModConvType.NONE:
+ if aux_in_features is None:
+ raise ValueError("aux_in_features cannot be None when modulation is enabled.")
+ if fc_hidden_features is None:
+ raise ValueError("fc_hidden_features cannot be None when modulation is enabled.")
+ if fc_groups is None:
+ raise ValueError("fc_groups cannot be None when modulation is enabled.")
+ if fc_groups < 1:
+ raise ValueError("fc_groups must be >= 1.")
+
+ hidden = _as_hidden_tuple(fc_hidden_features)
+
+ if modulation == ModConvType.FEATURES:
+ mod_out_features = (in_channels // fc_groups) * (out_channels // fc_groups)
+ elif modulation == ModConvType.FULL:
+ mod_out_features = (
+ (in_channels // fc_groups)
+ * (out_channels // fc_groups)
+ * self.kernel_size[0]
+ * self.kernel_size[1]
+ * self.kernel_size[2]
+ )
+ elif modulation == ModConvType.PARTIAL_OUT:
+ mod_out_features = (
+ self.kernel_size[0] * self.kernel_size[1] * self.kernel_size[2] * (out_channels // fc_groups)
+ )
+ elif modulation == ModConvType.PARTIAL_IN:
+ mod_out_features = (
+ self.kernel_size[0] * self.kernel_size[1] * self.kernel_size[2] * (in_channels // fc_groups)
+ )
+ else:
+ if (num_weights is None) or (num_weights < 1):
+ raise ValueError(f"ModConvType.SUM requires num_weights >= 1, got {num_weights}.")
+ mod_out_features = num_weights
+
+ mod_hidden = hidden + (mod_out_features,)
+ self.fc = _build_modulation_mlp(
+ aux_in_features,
+ mod_hidden,
+ fc_activation,
+ _resolved_fc_bias(fc_bias),
+ )
+
+ weight_shape = (in_channels, out_channels, *self.kernel_size)
+ if modulation == ModConvType.SUM:
+ weight_shape = (num_weights,) + weight_shape
+ k = math.sqrt(1 / (out_channels * self.kernel_size[0] * self.kernel_size[1] * self.kernel_size[2]))
+ self.weight = nn.Parameter(torch.FloatTensor(*weight_shape).uniform_(-k, k))
+
+ self.bias_type = bias
+ if bias == ModConv2dBias.PARAM:
+ self.bias = nn.Parameter(torch.FloatTensor(out_channels).uniform_(-k, k))
+ elif bias == ModConv2dBias.LEARNED:
+ if mod_hidden is None or aux_in_features is None:
+ raise ValueError("ModConv2dBias.LEARNED requires modulation to be enabled.")
+ self.bias = _build_learned_bias_mlp(
+ aux_in_features,
+ mod_hidden,
+ out_channels,
+ _resolved_fc_bias(fc_bias),
+ )
+ else:
+ self.bias = None
+
+ def __repr__(self):
+ return (
+ f"ModConvTranspose3d(in_channels={self.in_channels}, out_channels={self.out_channels}, "
+ f"kernel_size={self.kernel_size}, modulation={self.modulation}, "
+ f"stride={self.stride}, padding={self.padding}, dilation={self.dilation}, bias={self.bias_type})"
+ )
+
+ def forward(self, x: torch.Tensor, y: Optional[torch.Tensor] = None) -> torch.Tensor:
+ """Forward pass.
+
+ Parameters
+ ----------
+ x : torch.Tensor
+ Input of shape (N, in_channels, D, H, W).
+ y : torch.Tensor, optional
+ Auxiliary signal of shape (N, aux_in_features).
+
+ Returns
+ -------
+ torch.Tensor
+ Output of shape (N, out_channels, D_out, H_out, W_out).
+ """
+ if self.modulation == ModConvType.NONE:
+ out = F.conv_transpose3d(
+ x,
+ self.weight,
+ stride=self.stride,
+ padding=self.padding,
+ dilation=self.dilation,
+ )
+ else:
+ fc_out = self.fc(y)
+
+ if self.modulation == ModConvType.SUM:
+ weight = (fc_out.view(x.shape[0], -1, 1, 1, 1, 1, 1) * self.weight).sum(1)
+ else:
+ if self.modulation == ModConvType.FEATURES:
+ if self.fc_groups > 1:
+ fc_out = fc_out.view(
+ x.shape[0],
+ 1,
+ self.in_channels // self.fc_groups,
+ self.out_channels // self.fc_groups,
+ )
+ fc_out = F.interpolate(
+ fc_out,
+ size=(self.in_channels, self.out_channels),
+ mode="nearest",
+ )
+ fc_out = fc_out.view(x.shape[0], self.in_channels, self.out_channels, 1, 1, 1)
+
+ elif self.modulation == ModConvType.FULL:
+ if self.fc_groups > 1:
+ fc_out = fc_out.view(
+ x.shape[0],
+ self.kernel_size[0],
+ self.kernel_size[1],
+ self.kernel_size[2],
+ self.in_channels // self.fc_groups,
+ self.out_channels // self.fc_groups,
+ )
+ fc_out = fc_out.permute(0, 4, 5, 1, 2, 3)
+ in_ch_expand = self.in_channels // self.fc_groups
+ out_ch_expand = self.out_channels // self.fc_groups
+ if out_ch_expand < self.out_channels or in_ch_expand < self.in_channels:
+ fc_out = fc_out.repeat(1, self.fc_groups, self.fc_groups, 1, 1, 1)
+ fc_out = fc_out[:, : self.in_channels, : self.out_channels, :, :, :]
+ fc_out = fc_out.view(
+ x.shape[0],
+ self.in_channels,
+ self.out_channels,
+ self.kernel_size[0],
+ self.kernel_size[1],
+ self.kernel_size[2],
+ )
+ else:
+ fc_out = fc_out.view(
+ x.shape[0],
+ self.in_channels,
+ self.out_channels,
+ self.kernel_size[0],
+ self.kernel_size[1],
+ self.kernel_size[2],
+ )
+
+ elif self.modulation == ModConvType.PARTIAL_OUT:
+ if self.fc_groups > 1:
+ fc_out = fc_out.view(
+ x.shape[0],
+ self.kernel_size[0],
+ self.kernel_size[1],
+ self.kernel_size[2],
+ 1,
+ self.out_channels // self.fc_groups,
+ )
+ fc_out = fc_out.permute(0, 4, 5, 1, 2, 3)
+ if self.out_channels // self.fc_groups < self.out_channels:
+ fc_out = fc_out.repeat(1, 1, self.fc_groups, 1, 1, 1)
+ fc_out = fc_out[:, :, : self.out_channels, :, :, :]
+ fc_out = fc_out.view(
+ x.shape[0],
+ 1,
+ self.out_channels,
+ self.kernel_size[0],
+ self.kernel_size[1],
+ self.kernel_size[2],
+ )
+
+ else: # PARTIAL_IN
+ if self.fc_groups > 1:
+ fc_out = fc_out.view(
+ x.shape[0],
+ self.kernel_size[0],
+ self.kernel_size[1],
+ self.kernel_size[2],
+ self.in_channels // self.fc_groups,
+ 1,
+ )
+ fc_out = fc_out.permute(0, 4, 5, 1, 2, 3)
+ if self.in_channels // self.fc_groups < self.in_channels:
+ fc_out = fc_out.repeat(1, self.fc_groups, 1, 1, 1, 1)
+ fc_out = fc_out[:, : self.in_channels, :, :, :, :]
+ fc_out = fc_out.view(
+ x.shape[0],
+ self.in_channels,
+ 1,
+ self.kernel_size[0],
+ self.kernel_size[1],
+ self.kernel_size[2],
+ )
+
+ weight = fc_out * self.weight
+
+ out = torch.cat(
+ [
+ F.conv_transpose3d(
+ x[i : i + 1],
+ weight[i],
+ stride=self.stride,
+ padding=self.padding,
+ dilation=self.dilation,
+ )
+ for i in range(x.shape[0])
+ ],
+ 0,
+ )
+
+ if self.bias is not None:
+ if isinstance(self.bias, nn.parameter.Parameter):
+ bias = self.bias.view(1, -1, 1, 1, 1)
+ else:
+ bias = self.bias(y).view(x.shape[0], -1, 1, 1, 1)
+ out = out + bias
+
+ return out
diff --git a/direct/nn/crossdomain/crossdomain.py b/direct/nn/crossdomain/crossdomain.py
index 0ca0fd5d..f977ff4f 100644
--- a/direct/nn/crossdomain/crossdomain.py
+++ b/direct/nn/crossdomain/crossdomain.py
@@ -17,6 +17,7 @@
import torch.nn as nn
import direct.data.transforms as T
+from direct.nn.conv.modulated import ModConvType
from direct.types import FFTOperator
@@ -33,6 +34,7 @@ def __init__(
image_buffer_size: int = 1,
kspace_buffer_size: int = 1,
normalize_image: bool = False,
+ conv_modulation: ModConvType = ModConvType.NONE,
**kwargs,
):
"""Inits CrossDomainNetwork.
@@ -81,6 +83,7 @@ def __init__(
self.image_buffer_size = image_buffer_size
self.normalize_image = normalize_image
+ self.conv_modulation = conv_modulation
self._coil_dim = 1
self._complex_dim = -1
@@ -94,6 +97,7 @@ def kspace_correction(
sampling_mask: torch.Tensor,
sensitivity_map: torch.Tensor,
masked_kspace: torch.Tensor,
+ auxiliary_data: Optional[torch.Tensor] = None,
) -> torch.Tensor:
forward_buffer = torch.cat(
[
@@ -109,9 +113,11 @@ def kspace_correction(
kspace_buffer = torch.cat([kspace_buffer, forward_buffer, masked_kspace], self._complex_dim)
if self.kspace_model_list is not None:
- kspace_buffer = self.kspace_model_list[block_idx](kspace_buffer.permute(0, 1, 4, 2, 3)).permute(
- 0, 1, 3, 4, 2
- )
+ kspace_input = kspace_buffer.permute(0, 1, 4, 2, 3)
+ if self.conv_modulation != ModConvType.NONE:
+ kspace_buffer = self.kspace_model_list[block_idx](kspace_input, auxiliary_data).permute(0, 1, 3, 4, 2)
+ else:
+ kspace_buffer = self.kspace_model_list[block_idx](kspace_input).permute(0, 1, 3, 4, 2)
else:
kspace_buffer = kspace_buffer[..., :2] - kspace_buffer[..., 2:4]
@@ -124,6 +130,7 @@ def image_correction(
kspace_buffer: torch.Tensor,
sampling_mask: torch.Tensor,
sensitivity_map: torch.Tensor,
+ auxiliary_data: Optional[torch.Tensor] = None,
) -> torch.Tensor:
backward_buffer = torch.cat(
[
@@ -134,22 +141,34 @@ def image_correction(
)
image_buffer = torch.cat([image_buffer, backward_buffer], self._complex_dim).permute(0, 3, 1, 2)
- image_buffer = self.image_model_list[block_idx](image_buffer).permute(0, 2, 3, 1)
+ if self.conv_modulation != ModConvType.NONE:
+ image_buffer = self.image_model_list[block_idx](image_buffer, auxiliary_data).permute(0, 2, 3, 1)
+ else:
+ image_buffer = self.image_model_list[block_idx](image_buffer).permute(0, 2, 3, 1)
return image_buffer
def _forward_operator(
- self, image: torch.Tensor, sampling_mask: torch.Tensor, sensitivity_map: torch.Tensor
+ self,
+ image: torch.Tensor,
+ sampling_mask: torch.Tensor,
+ sensitivity_map: torch.Tensor,
) -> torch.Tensor:
forward = torch.where(
sampling_mask == 0,
torch.tensor([0.0], dtype=image.dtype).to(image.device),
- self.forward_operator(T.expand_operator(image, sensitivity_map, self._coil_dim), dim=self._spatial_dims),
+ self.forward_operator(
+ T.expand_operator(image, sensitivity_map, self._coil_dim),
+ dim=self._spatial_dims,
+ ),
)
return forward
def _backward_operator(
- self, kspace: torch.Tensor, sampling_mask: torch.Tensor, sensitivity_map: torch.Tensor
+ self,
+ kspace: torch.Tensor,
+ sampling_mask: torch.Tensor,
+ sensitivity_map: torch.Tensor,
) -> torch.Tensor:
backward = T.reduce_operator(
self.backward_operator(
@@ -171,6 +190,7 @@ def forward(
sampling_mask: torch.Tensor,
sensitivity_map: torch.Tensor,
scaling_factor: Optional[torch.Tensor] = None,
+ auxiliary_data: Optional[torch.Tensor] = None,
) -> torch.Tensor:
"""Computes the forward pass of :class:`CrossDomainNetwork`.
@@ -204,12 +224,23 @@ def forward(
for block_domain in self.domain_sequence:
if block_domain == "K":
kspace_buffer = self.kspace_correction(
- kspace_block_idx, image_buffer, kspace_buffer, sampling_mask, sensitivity_map, masked_kspace
+ kspace_block_idx,
+ image_buffer,
+ kspace_buffer,
+ sampling_mask,
+ sensitivity_map,
+ masked_kspace,
+ auxiliary_data,
)
kspace_block_idx += 1
else:
image_buffer = self.image_correction(
- image_block_idx, image_buffer, kspace_buffer, sampling_mask, sensitivity_map
+ image_block_idx,
+ image_buffer,
+ kspace_buffer,
+ sampling_mask,
+ sensitivity_map,
+ auxiliary_data,
)
image_block_idx += 1
diff --git a/direct/nn/crossdomain/multicoil.py b/direct/nn/crossdomain/multicoil.py
index dc86e37b..bd6cdc9a 100644
--- a/direct/nn/crossdomain/multicoil.py
+++ b/direct/nn/crossdomain/multicoil.py
@@ -11,6 +11,8 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+from typing import Optional
+
import torch
import torch.nn as nn
@@ -41,16 +43,19 @@ def __init__(self, model: nn.Module, coil_dim: int = 1, coil_to_batch: bool = Fa
self.coil_to_batch = coil_to_batch
self._coil_dim = coil_dim
- def _compute_model_per_coil(self, data: torch.Tensor) -> torch.Tensor:
+ def _compute_model_per_coil(self, data: torch.Tensor, y: Optional[torch.Tensor] = None) -> torch.Tensor:
output = []
for idx in range(data.size(self._coil_dim)):
subselected_data = data.select(self._coil_dim, idx)
- output.append(self.model(subselected_data))
+ if y is not None:
+ output.append(self.model(subselected_data, y))
+ else:
+ output.append(self.model(subselected_data))
return torch.stack(output, dim=self._coil_dim)
- def forward(self, x: torch.Tensor) -> torch.Tensor:
+ def forward(self, x: torch.Tensor, y: Optional[torch.Tensor] = None) -> torch.Tensor:
"""Performs the forward pass of MultiCoil.
Parameters
@@ -68,9 +73,12 @@ def forward(self, x: torch.Tensor) -> torch.Tensor:
batch, coil, height, width, channels = x.size()
x = x.reshape(batch * coil, height, width, channels).permute(0, 3, 1, 2).contiguous()
- x = self.model(x).permute(0, 2, 3, 1)
+ if y is not None:
+ x = self.model(x, y).permute(0, 2, 3, 1)
+ else:
+ x = self.model(x).permute(0, 2, 3, 1)
x = x.reshape(batch, coil, height, width, -1)
else:
- x = self._compute_model_per_coil(x).contiguous()
+ x = self._compute_model_per_coil(x, y).contiguous()
return x
diff --git a/direct/nn/didn/didn.py b/direct/nn/didn/didn.py
index 6d6495af..5f894598 100644
--- a/direct/nn/didn/didn.py
+++ b/direct/nn/didn/didn.py
@@ -11,12 +11,23 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-from typing import Tuple, Union
+from __future__ import annotations
+
+from typing import Optional, Tuple, Union
import torch
import torch.nn as nn
import torch.nn.functional as F
+from direct.nn.conv.modulated import (
+ ModConv2d,
+ ModConv2dBias,
+ ModConvActivation,
+ ModConvType,
+ ModulationParams,
+ mod_conv2d,
+)
+
class Subpixel(nn.Module):
"""Subpixel convolution layer for up-scaling of low resolution features at super-resolution as implemented in [1]_.
@@ -24,7 +35,8 @@ class Subpixel(nn.Module):
References
----------
- .. [1] Yu, Songhyun, et al. “Deep Iterative Down-Up CNN for Image Denoising.” 2019 IEEE/CVF Conference on Computer Vision and Pattern Recognition Workshops (CVPRW), 2019, pp. 2095–103. IEEE Xplore, https://doi.org/10.1109/CVPRW.2019.00262.
+ .. [1] Yu, Songhyun, et al. "Deep Iterative Down-Up CNN for Image Denoising."
+ CVPRW, 2019. https://doi.org/10.1109/CVPRW.2019.00262.
"""
def __init__(
@@ -34,6 +46,13 @@ def __init__(
upscale_factor: int,
kernel_size: Union[int, Tuple[int, int]],
padding: int = 0,
+ modulation: ModConvType = ModConvType.NONE,
+ modulation_params: Optional[ModulationParams] = None,
+ aux_in_features: Optional[int] = None,
+ fc_hidden_features: Optional[tuple[int] | int] = None,
+ fc_groups: int = 1,
+ fc_activation: ModConvActivation = ModConvActivation.SIGMOID,
+ num_weights: Optional[int] = None,
):
"""Inits :class:`Subpixel`.
@@ -49,19 +68,52 @@ def __init__(
Convolution kernel size.
padding: int
Padding size. Default: 0.
+ modulation : ModConvType
+ Modulation type. Default: ModConvType.NONE.
+ aux_in_features : int, optional
+ Auxiliary input features for modulation.
+ fc_hidden_features : int or tuple of int, optional
+ Hidden features for modulation MLP.
+ fc_groups : int
+ Groups for modulation MLP. Default: 1.
+ fc_activation : ModConvActivation
+ Activation for modulation MLP. Default: ModConvActivation.SIGMOID.
+ num_weights : int, optional
+ Number of weight bases for ModConvType.SUM.
"""
super().__init__()
- self.conv = nn.Conv2d(in_channels, out_channels * upscale_factor**2, kernel_size=kernel_size, padding=padding)
+ if modulation_params is None:
+ modulation_params = ModulationParams(
+ modulation=modulation,
+ aux_in_features=aux_in_features,
+ fc_hidden_features=fc_hidden_features,
+ fc_groups=fc_groups,
+ fc_activation=fc_activation,
+ num_weights=num_weights,
+ )
+ self.modulation = modulation_params.modulation
+ self.conv = mod_conv2d(
+ in_channels,
+ out_channels * upscale_factor**2,
+ kernel_size=kernel_size,
+ padding=padding,
+ bias=ModConv2dBias.PARAM,
+ modulation_params=modulation_params,
+ )
self.pixelshuffle = nn.PixelShuffle(upscale_factor)
- def forward(self, x: torch.Tensor) -> torch.Tensor:
+ def forward(self, x: torch.Tensor, y: Optional[torch.Tensor] = None) -> torch.Tensor:
"""Computes :class:`Subpixel` convolution on input torch.Tensor ``x``.
Parameters
----------
x: torch.Tensor
Input tensor.
+ y: torch.Tensor, optional
+ Auxiliary signal for modulation.
"""
+ if self.modulation != ModConvType.NONE:
+ return self.pixelshuffle(self.conv(x, y))
return self.pixelshuffle(self.conv(x))
@@ -71,10 +123,22 @@ class ReconBlock(nn.Module):
References
----------
- .. [1] Yu, Songhyun, et al. “Deep Iterative Down-Up CNN for Image Denoising.” 2019 IEEE/CVF Conference on Computer Vision and Pattern Recognition Workshops (CVPRW), 2019, pp. 2095–103. IEEE Xplore, https://doi.org/10.1109/CVPRW.2019.00262.
+ .. [1] Yu, Songhyun, et al. "Deep Iterative Down-Up CNN for Image Denoising."
+ CVPRW, 2019. https://doi.org/10.1109/CVPRW.2019.00262.
"""
- def __init__(self, in_channels: int, num_convs: int):
+ def __init__(
+ self,
+ in_channels: int,
+ num_convs: int,
+ modulation: ModConvType = ModConvType.NONE,
+ modulation_params: Optional[ModulationParams] = None,
+ aux_in_features: Optional[int] = None,
+ fc_hidden_features: Optional[tuple[int] | int] = None,
+ fc_groups: int = 1,
+ fc_activation: ModConvActivation = ModConvActivation.SIGMOID,
+ num_weights: Optional[int] = None,
+ ):
"""Inits :class:`ReconBlock`.
Parameters
@@ -83,34 +147,64 @@ def __init__(self, in_channels: int, num_convs: int):
Number of input channels.
num_convs: int
Number of convolution blocks.
+ modulation : ModConvType
+ Modulation type. Default: ModConvType.NONE.
+ aux_in_features : int, optional
+ Auxiliary input features for modulation.
+ fc_hidden_features : int or tuple of int, optional
+ Hidden features for modulation MLP.
+ fc_groups : int
+ Groups for modulation MLP. Default: 1.
+ fc_activation : ModConvActivation
+ Activation for modulation MLP. Default: ModConvActivation.SIGMOID.
+ num_weights : int, optional
+ Number of weight bases for ModConvType.SUM.
"""
super().__init__()
- self.convs = nn.ModuleList(
- [
- nn.Sequential(
- *[
- nn.Conv2d(in_channels=in_channels, out_channels=in_channels, kernel_size=3, padding=1),
- nn.PReLU(),
- ]
- )
- for _ in range(num_convs - 1)
- ]
- )
- self.convs.append(nn.Conv2d(in_channels=in_channels, out_channels=in_channels, kernel_size=3, padding=1))
+ if modulation_params is None:
+ modulation_params = ModulationParams(
+ modulation=modulation,
+ aux_in_features=aux_in_features,
+ fc_hidden_features=fc_hidden_features,
+ fc_groups=fc_groups,
+ fc_activation=fc_activation,
+ num_weights=num_weights,
+ )
+ self.modulation = modulation_params.modulation
self.num_convs = num_convs
+ self.activations = nn.ModuleList([nn.PReLU() for _ in range(num_convs - 1)])
+
+ self.convs = nn.ModuleList()
+ for _ in range(num_convs):
+ self.convs.append(
+ mod_conv2d(
+ in_channels=in_channels,
+ out_channels=in_channels,
+ kernel_size=3,
+ padding=1,
+ bias=ModConv2dBias.PARAM,
+ modulation_params=modulation_params,
+ )
+ )
- def forward(self, input_data: torch.Tensor) -> torch.Tensor:
+ def forward(self, input_data: torch.Tensor, y: Optional[torch.Tensor] = None) -> torch.Tensor:
"""Computes num_convs convolutions followed by PReLU activation on `input_data`.
Parameters
----------
input_data: torch.Tensor
Input tensor.
+ y: torch.Tensor, optional
+ Auxiliary signal for modulation.
"""
-
output = input_data.clone()
for idx in range(self.num_convs):
- output = self.convs[idx](output)
+ if self.modulation != ModConvType.NONE:
+ output = self.convs[idx](output, y)
+ else:
+ output = self.convs[idx](output)
+ if idx < self.num_convs - 1:
+ output = self.activations[idx](output)
return input_data + output
@@ -121,13 +215,21 @@ class DUB(nn.Module):
References
----------
- .. [1] Yu, Songhyun, et al. “Deep Iterative Down-Up CNN for Image Denoising.” 2019 IEEE/CVF Conference on Computer Vision and Pattern Recognition Workshops (CVPRW), 2019, pp. 2095–103. IEEE Xplore, https://doi.org/10.1109/CVPRW.2019.00262.
+ .. [1] Yu, Songhyun, et al. "Deep Iterative Down-Up CNN for Image Denoising."
+ CVPRW, 2019. https://doi.org/10.1109/CVPRW.2019.00262.
"""
def __init__(
self,
in_channels: int,
out_channels: int,
+ modulation: ModConvType = ModConvType.NONE,
+ modulation_params: Optional[ModulationParams] = None,
+ aux_in_features: Optional[int] = None,
+ fc_hidden_features: Optional[tuple[int] | int] = None,
+ fc_groups: int = 1,
+ fc_activation: ModConvActivation = ModConvActivation.SIGMOID,
+ num_weights: Optional[int] = None,
):
"""Inits :class:`DUB`.
@@ -137,72 +239,129 @@ def __init__(
Number of input channels.
out_channels: int
Number of output channels.
+ modulation : ModConvType
+ Modulation type. Default: ModConvType.NONE.
+ aux_in_features : int, optional
+ Auxiliary input features for modulation.
+ fc_hidden_features : int or tuple of int, optional
+ Hidden features for modulation MLP.
+ fc_groups : int
+ Groups for modulation MLP. Default: 1.
+ fc_activation : ModConvActivation
+ Activation for modulation MLP. Default: ModConvActivation.SIGMOID.
+ num_weights : int, optional
+ Number of weight bases for ModConvType.SUM.
"""
super().__init__()
self.in_channels = in_channels
self.out_channels = out_channels
+ if modulation_params is None:
+ modulation_params = ModulationParams(
+ modulation=modulation,
+ aux_in_features=aux_in_features,
+ fc_hidden_features=fc_hidden_features,
+ fc_groups=fc_groups,
+ fc_activation=fc_activation,
+ num_weights=num_weights,
+ )
+ self.modulation = modulation_params.modulation
+
+ # Scale 1 - down path
+ self.conv1_1_a = mod_conv2d(
+ in_channels, in_channels, kernel_size=3, padding=1, modulation_params=modulation_params
+ )
+ self.conv1_1_a_act = nn.PReLU()
+ self.conv1_1_b = mod_conv2d(
+ in_channels, in_channels, kernel_size=3, padding=1, modulation_params=modulation_params
+ )
+ self.conv1_1_b_act = nn.PReLU()
+ self.down1 = mod_conv2d(
+ in_channels,
+ in_channels * 2,
+ kernel_size=3,
+ stride=2,
+ padding=1,
+ modulation_params=modulation_params,
+ )
- # Scale 1
- self.conv1_1 = nn.Sequential(*[nn.Conv2d(in_channels, in_channels, kernel_size=3, padding=1), nn.PReLU()] * 2)
- self.down1 = nn.Conv2d(in_channels, in_channels * 2, kernel_size=3, stride=2, padding=1)
- # Scale 2
- self.conv2_1 = nn.Sequential(
- *[nn.Conv2d(in_channels * 2, in_channels * 2, kernel_size=3, padding=1), nn.PReLU()]
+ # Scale 2 - down path
+ self.conv2_1 = mod_conv2d(
+ in_channels * 2,
+ in_channels * 2,
+ kernel_size=3,
+ padding=1,
+ modulation_params=modulation_params,
)
- self.down2 = nn.Conv2d(in_channels * 2, in_channels * 4, kernel_size=3, stride=2, padding=1)
- # Scale 3
- self.conv3_1 = nn.Sequential(
- *[
- nn.Conv2d(in_channels * 4, in_channels * 4, kernel_size=3, padding=1),
- nn.PReLU(),
- ]
+ self.conv2_1_act = nn.PReLU()
+ self.down2 = mod_conv2d(
+ in_channels * 2,
+ in_channels * 4,
+ kernel_size=3,
+ stride=2,
+ padding=1,
+ modulation_params=modulation_params,
)
- self.up1 = nn.Sequential(
- *[
- # nn.Conv2d(in_channels * 4, in_channels * 8, kernel_size=1),
- Subpixel(in_channels * 4, in_channels * 2, 2, 1, 0)
- ]
+
+ # Scale 3 - bottom
+ self.conv3_1 = mod_conv2d(
+ in_channels * 4,
+ in_channels * 4,
+ kernel_size=3,
+ padding=1,
+ modulation_params=modulation_params,
)
- # Scale 2
- self.conv_agg_1 = nn.Conv2d(in_channels * 4, in_channels * 2, kernel_size=1)
- self.conv2_2 = nn.Sequential(
- *[
- nn.Conv2d(in_channels * 2, in_channels * 2, kernel_size=3, padding=1),
- nn.PReLU(),
- ]
+ self.conv3_1_act = nn.PReLU()
+ self.up1 = Subpixel(in_channels * 4, in_channels * 2, 2, 1, 0, modulation_params=modulation_params)
+
+ # Scale 2 - up path
+ self.conv_agg_1 = mod_conv2d(
+ in_channels * 4,
+ in_channels * 2,
+ kernel_size=1,
+ padding=0,
+ modulation_params=modulation_params,
)
- self.up2 = nn.Sequential(
- *[
- # nn.Conv2d(in_channels * 2, in_channels * 4, kernel_size=1),
- Subpixel(in_channels * 2, in_channels, 2, 1, 0)
- ]
+ self.conv2_2 = mod_conv2d(
+ in_channels * 2,
+ in_channels * 2,
+ kernel_size=3,
+ padding=1,
+ modulation_params=modulation_params,
)
- # Scale 1
- self.conv_agg_2 = nn.Conv2d(in_channels * 2, in_channels, kernel_size=1)
- self.conv1_2 = nn.Sequential(*[nn.Conv2d(in_channels, in_channels, kernel_size=3, padding=1), nn.PReLU()] * 2)
- self.conv_out = nn.Sequential(*[nn.Conv2d(in_channels, in_channels, kernel_size=3, padding=1), nn.PReLU()])
+ self.conv2_2_act = nn.PReLU()
+ self.up2 = Subpixel(in_channels * 2, in_channels, 2, 1, 0, modulation_params=modulation_params)
- @staticmethod
- def pad(x: torch.Tensor) -> torch.Tensor:
- """Pads input to height and width dimensions if odd.
+ # Scale 1 - up path
+ self.conv_agg_2 = mod_conv2d(
+ in_channels * 2, in_channels, kernel_size=1, padding=0, modulation_params=modulation_params
+ )
+ self.conv1_2_a = mod_conv2d(
+ in_channels, in_channels, kernel_size=3, padding=1, modulation_params=modulation_params
+ )
+ self.conv1_2_a_act = nn.PReLU()
+ self.conv1_2_b = mod_conv2d(
+ in_channels, in_channels, kernel_size=3, padding=1, modulation_params=modulation_params
+ )
+ self.conv1_2_b_act = nn.PReLU()
+ self.conv_out = mod_conv2d(
+ in_channels, in_channels, kernel_size=3, padding=1, modulation_params=modulation_params
+ )
+ self.conv_out_act = nn.PReLU()
- Parameters
- ----------
- x: torch.Tensor
- Input to pad.
+ def _conv(self, layer: ModConv2d, x: torch.Tensor, y: Optional[torch.Tensor]) -> torch.Tensor:
+ if self.modulation != ModConvType.NONE:
+ return layer(x, y)
+ return layer(x)
- Returns
- -------
- x: torch.Tensor
- Padded tensor.
- """
+ @staticmethod
+ def pad(x: torch.Tensor) -> torch.Tensor:
+ """Pads input to height and width dimensions if odd."""
padding = [0, 0, 0, 0]
-
if x.shape[-2] % 2 != 0:
- padding[3] = 1 # Padding right - width
+ padding[3] = 1
if x.shape[-1] % 2 != 0:
- padding[1] = 1 # Padding bottom - height
+ padding[1] = 1
if sum(padding) != 0:
x = F.pad(x, padding, "reflect")
return x
@@ -214,7 +373,7 @@ def crop_to_shape(x: torch.Tensor, shape: Tuple[int, int]) -> torch.Tensor:
Parameters
----------
x: torch.Tensor
- Input tensor with shape (\*, H, W).
+ Input tensor with shape (\\*, H, W).
shape: Tuple(int, int)
Crop shape corresponding to H, W.
@@ -224,19 +383,21 @@ def crop_to_shape(x: torch.Tensor, shape: Tuple[int, int]) -> torch.Tensor:
Cropped tensor.
"""
h, w = x.shape[-2:]
-
if h > shape[0]:
x = x[:, :, : shape[0], :]
if w > shape[1]:
x = x[:, :, :, : shape[1]]
return x
- def forward(self, x: torch.Tensor) -> torch.Tensor:
- """
+ def forward(self, x: torch.Tensor, y: Optional[torch.Tensor] = None) -> torch.Tensor:
+ """Forward pass.
+
Parameters
----------
x: torch.Tensor
Input tensor.
+ y: torch.Tensor, optional
+ Auxiliary signal for modulation.
Returns
-------
@@ -244,20 +405,41 @@ def forward(self, x: torch.Tensor) -> torch.Tensor:
DUB output.
"""
x1 = self.pad(x.clone())
- x1 = x1 + self.conv1_1(x1)
- x2 = self.down1(x1)
- x2 = x2 + self.conv2_1(x2)
- out = self.down2(x2)
- out = out + self.conv3_1(out)
- out = self.up1(out)
+ x1 = x1 + self.conv1_1_b_act(
+ self._conv(self.conv1_1_b, self.conv1_1_a_act(self._conv(self.conv1_1_a, x1, y)), y)
+ )
+ x2 = self._conv(self.down1, x1, y)
+ x2 = x2 + self.conv2_1_act(self._conv(self.conv2_1, x2, y))
+ out = self._conv(self.down2, x2, y)
+ out = out + self.conv3_1_act(self._conv(self.conv3_1, out, y))
+
+ if self.modulation != ModConvType.NONE:
+ out = self.up1(out, y)
+ else:
+ out = self.up1(out)
+
out = torch.cat([x2, self.crop_to_shape(out, (x2.shape[-2], x2.shape[-1]))], dim=1)
- out = self.conv_agg_1(out)
- out = out + self.conv2_2(out)
- out = self.up2(out)
+ out = self._conv(self.conv_agg_1, out, y)
+ out = out + self.conv2_2_act(self._conv(self.conv2_2, out, y))
+
+ if self.modulation != ModConvType.NONE:
+ out = self.up2(out, y)
+ else:
+ out = self.up2(out)
+
out = torch.cat([x1, self.crop_to_shape(out, (x1.shape[-2], x1.shape[-1]))], dim=1)
- out = self.conv_agg_2(out)
- out = out + self.conv1_2(out)
- out = x + self.crop_to_shape(self.conv_out(out), (x.shape[-2], x.shape[-1]))
+ out = self._conv(self.conv_agg_2, out, y)
+ out = out + self.conv1_2_b_act(
+ self._conv(
+ self.conv1_2_b,
+ self.conv1_2_a_act(self._conv(self.conv1_2_a, out, y)),
+ y,
+ )
+ )
+ out = x + self.crop_to_shape(
+ self.conv_out_act(self._conv(self.conv_out, out, y)),
+ (x.shape[-2], x.shape[-1]),
+ )
return out
@@ -267,7 +449,8 @@ class DIDN(nn.Module):
References
----------
- .. [1] Yu, Songhyun, et al. “Deep Iterative Down-Up CNN for Image Denoising.” 2019 IEEE/CVF Conference on Computer Vision and Pattern Recognition Workshops (CVPRW), 2019, pp. 2095–103. IEEE Xplore, https://doi.org/10.1109/CVPRW.2019.00262.
+ .. [1] Yu, Songhyun, et al. "Deep Iterative Down-Up CNN for Image Denoising."
+ CVPRW, 2019. https://doi.org/10.1109/CVPRW.2019.00262.
"""
def __init__(
@@ -278,6 +461,13 @@ def __init__(
num_dubs: int = 6,
num_convs_recon: int = 9,
skip_connection: bool = False,
+ modulation: ModConvType = ModConvType.NONE,
+ modulation_params: Optional[ModulationParams] = None,
+ aux_in_features: Optional[int] = None,
+ fc_hidden_features: Optional[tuple[int] | int] = None,
+ fc_groups: int = 1,
+ fc_activation: ModConvActivation = ModConvActivation.SIGMOID,
+ num_weights: Optional[int] = None,
):
"""Inits :class:`DIDN`.
@@ -295,44 +485,94 @@ def __init__(
Number of ReconBlock convolutions. Default: 9.
skip_connection: bool
Use skip connection. Default: False.
+ modulation : ModConvType
+ Modulation type. Default: ModConvType.NONE.
+ aux_in_features : int, optional
+ Auxiliary input features for modulation.
+ fc_hidden_features : int or tuple of int, optional
+ Hidden features for modulation MLP.
+ fc_groups : int
+ Groups for modulation MLP. Default: 1.
+ fc_activation : ModConvActivation
+ Activation for modulation MLP. Default: ModConvActivation.SIGMOID.
+ num_weights : int, optional
+ Number of weight bases for ModConvType.SUM.
"""
super().__init__()
- self.conv_in = nn.Sequential(
- *[nn.Conv2d(in_channels=in_channels, out_channels=hidden_channels, kernel_size=3, padding=1), nn.PReLU()]
+ if modulation_params is None:
+ modulation_params = ModulationParams(
+ modulation=modulation,
+ aux_in_features=aux_in_features,
+ fc_hidden_features=fc_hidden_features,
+ fc_groups=fc_groups,
+ fc_activation=fc_activation,
+ num_weights=num_weights,
+ )
+ self.modulation = modulation_params.modulation
+
+ self.conv_in = mod_conv2d(
+ in_channels=in_channels,
+ out_channels=hidden_channels,
+ kernel_size=3,
+ padding=1,
+ modulation_params=modulation_params,
)
- self.down = nn.Conv2d(
+ self.conv_in_act = nn.PReLU()
+ self.down = mod_conv2d(
in_channels=hidden_channels,
out_channels=hidden_channels,
kernel_size=3,
stride=2,
padding=1,
+ modulation_params=modulation_params,
)
self.dubs = nn.ModuleList(
- [DUB(in_channels=hidden_channels, out_channels=hidden_channels) for _ in range(num_dubs)]
- )
- self.recon_block = ReconBlock(in_channels=hidden_channels, num_convs=num_convs_recon)
- self.recon_agg = nn.Conv2d(in_channels=hidden_channels * num_dubs, out_channels=hidden_channels, kernel_size=1)
- self.conv = nn.Sequential(
- *[
- nn.Conv2d(
+ [
+ DUB(
in_channels=hidden_channels,
out_channels=hidden_channels,
- kernel_size=3,
- padding=1,
- ),
- nn.PReLU(),
+ modulation_params=modulation_params,
+ )
+ for _ in range(num_dubs)
]
)
- self.up2 = Subpixel(hidden_channels, hidden_channels, 2, 1)
- self.conv_out = nn.Conv2d(
+ self.recon_blocks = nn.ModuleList(
+ [
+ ReconBlock(in_channels=hidden_channels, num_convs=num_convs_recon, modulation_params=modulation_params)
+ for _ in range(num_dubs)
+ ]
+ )
+ self.recon_agg = mod_conv2d(
+ in_channels=hidden_channels * num_dubs,
+ out_channels=hidden_channels,
+ kernel_size=1,
+ padding=0,
+ modulation_params=modulation_params,
+ )
+ self.conv = mod_conv2d(
+ in_channels=hidden_channels,
+ out_channels=hidden_channels,
+ kernel_size=3,
+ padding=1,
+ modulation_params=modulation_params,
+ )
+ self.conv_act = nn.PReLU()
+ self.up2 = Subpixel(hidden_channels, hidden_channels, 2, 1, modulation_params=modulation_params)
+ self.conv_out = mod_conv2d(
in_channels=hidden_channels,
out_channels=out_channels,
kernel_size=3,
padding=1,
+ modulation_params=modulation_params,
)
self.num_dubs = num_dubs
self.skip_connection = (in_channels == out_channels) and skip_connection
+ def _conv(self, layer: ModConv2d, x: torch.Tensor, y: Optional[torch.Tensor]) -> torch.Tensor:
+ if self.modulation != ModConvType.NONE:
+ return layer(x, y)
+ return layer(x)
+
@staticmethod
def crop_to_shape(x: torch.Tensor, shape: Tuple[int, int]) -> torch.Tensor:
"""Crops ``x`` to specified shape.
@@ -340,7 +580,7 @@ def crop_to_shape(x: torch.Tensor, shape: Tuple[int, int]) -> torch.Tensor:
Parameters
----------
x: torch.Tensor
- Input tensor with shape (\*, H, W).
+ Input tensor with shape (\\*, H, W).
shape: Tuple(int, int)
Crop shape corresponding to H, W.
@@ -350,20 +590,21 @@ def crop_to_shape(x: torch.Tensor, shape: Tuple[int, int]) -> torch.Tensor:
Cropped tensor.
"""
h, w = x.shape[-2:]
-
if h > shape[0]:
x = x[:, :, : shape[0], :]
if w > shape[1]:
x = x[:, :, :, : shape[1]]
return x
- def forward(self, x: torch.Tensor, channel_dim: int = 1) -> torch.Tensor:
+ def forward(self, x: torch.Tensor, y: Optional[torch.Tensor] = None, channel_dim: int = 1) -> torch.Tensor:
"""Takes as input a torch.Tensor `x` and computes DIDN(x).
Parameters
----------
x: torch.Tensor
Input tensor.
+ y: torch.Tensor, optional
+ Auxiliary signal for modulation of shape (N, aux_in_features).
channel_dim: int
Channel dimension. Default: 1.
@@ -372,19 +613,24 @@ def forward(self, x: torch.Tensor, channel_dim: int = 1) -> torch.Tensor:
out: torch.Tensor
DIDN output tensor.
"""
- out = self.conv_in(x)
- out = self.down(out)
+ out = self.conv_in_act(self._conv(self.conv_in, x, y))
+ out = self._conv(self.down, out, y)
dub_outs = []
for dub in self.dubs:
- out = dub(out)
+ out = dub(out, y)
dub_outs.append(out)
- out = [self.recon_block(dub_out) for dub_out in dub_outs]
- out = self.recon_agg(torch.cat(out, dim=channel_dim))
- out = self.conv(out)
- out = self.up2(out)
- out = self.conv_out(out)
+ out = [self.recon_blocks[i](dub_outs[i], y) for i in range(self.num_dubs)]
+ out = self._conv(self.recon_agg, torch.cat(out, dim=channel_dim), y)
+ out = self.conv_act(self._conv(self.conv, out, y))
+
+ if self.modulation != ModConvType.NONE:
+ out = self.up2(out, y)
+ else:
+ out = self.up2(out)
+
+ out = self._conv(self.conv_out, out, y)
out = self.crop_to_shape(out, (x.shape[-2], x.shape[-1]))
if self.skip_connection:
diff --git a/direct/nn/get_nn_model_config.py b/direct/nn/get_nn_model_config.py
index 286ba227..79f25c41 100644
--- a/direct/nn/get_nn_model_config.py
+++ b/direct/nn/get_nn_model_config.py
@@ -16,7 +16,9 @@
from torch import nn
from direct.constants import COMPLEX_SIZE
+from direct.nn.adain.adain import NormType
from direct.nn.conv.conv import Conv2d
+from direct.nn.conv.modulated import ModConvActivation, ModConvType
from direct.nn.didn.didn import DIDN
from direct.nn.resnet.resnet import ResNet
from direct.nn.types import ActivationType, ModelName
@@ -42,9 +44,15 @@ def _get_relu_activation(activation: ActivationType = ActivationType.RELU, **kwa
def _get_model_config(
- model_architecture_name: ModelName, in_channels: int = COMPLEX_SIZE, out_channels: int = COMPLEX_SIZE, **kwargs
+ model_architecture_name: ModelName,
+ in_channels: int = COMPLEX_SIZE,
+ out_channels: int = COMPLEX_SIZE,
+ **kwargs,
) -> tuple[type[nn.Module], dict[str, object]]:
- model_kwargs: dict[str, object] = {"in_channels": in_channels, "out_channels": out_channels}
+ model_kwargs: dict[str, object] = {
+ "in_channels": in_channels,
+ "out_channels": out_channels,
+ }
if model_architecture_name in ["unet", "normunet"]:
model_architecture = UnetModel2d if model_architecture_name == "unet" else NormUnetModel2d
model_kwargs.update(
@@ -52,6 +60,15 @@ def _get_model_config(
"num_filters": kwargs.get("unet_num_filters", 32),
"num_pool_layers": kwargs.get("unet_num_pool_layers", 4),
"dropout_probability": kwargs.get("unet_dropout", 0.0),
+ "modulation": kwargs.get("modulation", ModConvType.NONE),
+ "aux_in_features": kwargs.get("aux_in_features"),
+ "fc_hidden_features": kwargs.get("fc_hidden_features"),
+ "fc_groups": kwargs.get("fc_groups", 1),
+ "fc_activation": kwargs.get("fc_activation", ModConvActivation.SIGMOID),
+ "num_weights": kwargs.get("num_weights"),
+ "modulation_at_input": kwargs.get("modulation_at_input", False),
+ "norm_type": kwargs.get("unet_norm_type", NormType.INSTANCE),
+ "adain_hidden_features": kwargs.get("unet_adain_hidden_features"),
}
)
elif model_architecture_name == "resnet":
diff --git a/direct/nn/iterdualnet/config.py b/direct/nn/iterdualnet/config.py
index 2ec2d697..a4914bcd 100644
--- a/direct/nn/iterdualnet/config.py
+++ b/direct/nn/iterdualnet/config.py
@@ -12,8 +12,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from dataclasses import dataclass
+from typing import Optional
from direct.config.defaults import ModelConfig
+from direct.nn.conv.modulated import ModConvActivation, ModConvType
@dataclass
@@ -30,3 +32,11 @@ class IterDualNetConfig(ModelConfig):
image_no_parameter_sharing: bool = True
kspace_no_parameter_sharing: bool = False
compute_per_coil: bool = True
+ conv_modulation: ModConvType = ModConvType.NONE
+ aux_in_features: Optional[int] = None
+ auxiliary_features: Optional[tuple[str, ...]] = None
+ log_aux: bool = False
+ fc_hidden_features: Optional[tuple[int, ...]] = None
+ fc_groups: int = 1
+ fc_activation: ModConvActivation = ModConvActivation.SIGMOID
+ num_weights: Optional[int] = None
diff --git a/direct/nn/iterdualnet/iterdualnet.py b/direct/nn/iterdualnet/iterdualnet.py
index 93da7b31..761bcd86 100644
--- a/direct/nn/iterdualnet/iterdualnet.py
+++ b/direct/nn/iterdualnet/iterdualnet.py
@@ -12,11 +12,16 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import annotations
+
+from typing import Optional
+
import torch
from torch import nn
import direct.data.transforms as T
from direct.constants import COMPLEX_SIZE
+from direct.nn.conv.modulated import ModConvActivation, ModConvType, ModulationParams
from direct.nn.unet.unet_2d import NormUnetModel2d, UnetModel2d
from direct.types import FFTOperator
@@ -33,6 +38,15 @@ class IterDualNet(nn.Module):
by unrolling a gradient descent scheme where :math:`\mathcal{E}` and :math:`\mathcal{R}` are the expand and
reduce operators which use the sensitivity maps. :math:`D_I` and :math:`D_F` are trainable U-Nets operating
in the image and k-space domain.
+
+ Supports conditional weight modulation as proposed in [1]_.
+
+ References
+ ----------
+
+ .. [1] Moriakov, N., Yiasemis, G., Sonke, J.-J. & Teuwen, J. (2026). Conditional Learned Reconstruction for
+ Medical Imaging. Proceedings of The 9th International Conference on Medical Imaging with Deep Learning,
+ PMLR 315:754-780. https://proceedings.mlr.press/v315/moriakov26a.html
"""
def __init__(
@@ -45,6 +59,12 @@ def __init__(
image_no_parameter_sharing: bool = True,
kspace_no_parameter_sharing: bool = True,
compute_per_coil: bool = True,
+ conv_modulation: ModConvType = ModConvType.NONE,
+ aux_in_features: Optional[int] = None,
+ fc_hidden_features: Optional[tuple[int] | int] = None,
+ fc_groups: int = 1,
+ fc_activation: ModConvActivation = ModConvActivation.SIGMOID,
+ num_weights: Optional[int] = None,
**kwargs,
):
"""Inits :class:`IterDualNet`.
@@ -67,6 +87,18 @@ def __init__(
If False, a single kspace model will be shared across all iterations. Default: True.
compute_per_coil : bool
If True :math:`f` will be transformed into a multi-coil kspace.
+ conv_modulation : ModConvType
+ Modulation type for convolutional layers. Default: ModConvType.NONE.
+ aux_in_features : int, optional
+ Number of features in the auxiliary input for modulation.
+ fc_hidden_features : int or tuple of int, optional
+ Hidden features in the modulation MLP.
+ fc_groups : int
+ Groups for modulation MLP output. Default: 1.
+ fc_activation : ModConvActivation
+ Activation after modulation MLP. Default: ModConvActivation.SIGMOID.
+ num_weights : int, optional
+ Number of weight bases for ModConvType.SUM.
kwargs : dict
Kwargs for unet models.
"""
@@ -75,12 +107,22 @@ def __init__(
self.forward_operator = forward_operator
self.backward_operator = backward_operator
self.num_iter = num_iter
+ self.conv_modulation = conv_modulation
self.image_no_parameter_sharing = image_no_parameter_sharing
self.kspace_no_parameter_sharing = kspace_no_parameter_sharing
image_unet_architecture = NormUnetModel2d if image_normunet else UnetModel2d
kspace_unet_architecture = NormUnetModel2d if kspace_normunet else UnetModel2d
+ modulation_params = ModulationParams(
+ modulation=conv_modulation,
+ aux_in_features=aux_in_features,
+ fc_hidden_features=fc_hidden_features,
+ fc_groups=fc_groups,
+ fc_activation=fc_activation,
+ num_weights=num_weights,
+ )
+
self.image_block_list = nn.ModuleList()
self.kspace_block_list = nn.ModuleList()
@@ -92,6 +134,7 @@ def __init__(
num_filters=kwargs.get("image_unet_num_filters", 8),
num_pool_layers=kwargs.get("image_unet_num_pool_layers", 4),
dropout_probability=kwargs.get("image_unet_dropout", 0.0),
+ modulation_params=modulation_params,
)
)
for _ in range(self.num_iter if self.kspace_no_parameter_sharing else 1):
@@ -102,6 +145,7 @@ def __init__(
num_filters=kwargs.get("kspace_unet_num_filters", 8),
num_pool_layers=kwargs.get("kspace_unet_num_pool_layers", 4),
dropout_probability=kwargs.get("kspace_unet_dropout", 0.0),
+ modulation_params=modulation_params,
)
)
self.compute_per_coil = compute_per_coil
@@ -114,44 +158,87 @@ def __init__(
self._complex_dim = -1
self._spatial_dims = (2, 3)
- def _image_model(self, image: torch.Tensor, step: int) -> torch.Tensor:
+ def _image_model(
+ self,
+ image: torch.Tensor,
+ step: int,
+ auxiliary_data: Optional[torch.Tensor] = None,
+ ) -> torch.Tensor:
image = image.permute(0, 3, 1, 2)
block_idx = step if self.image_no_parameter_sharing else 0
+ if self.conv_modulation != ModConvType.NONE:
+ return self.image_block_list[block_idx](image, auxiliary_data).permute(0, 2, 3, 1).contiguous()
return self.image_block_list[block_idx](image).permute(0, 2, 3, 1).contiguous()
- def _kspace_model(self, kspace: torch.Tensor, step: int) -> torch.Tensor:
+ def _kspace_model(
+ self,
+ kspace: torch.Tensor,
+ step: int,
+ auxiliary_data: Optional[torch.Tensor] = None,
+ ) -> torch.Tensor:
block_idx = step if self.kspace_no_parameter_sharing else 0
if self.compute_per_coil:
kspace = (
- self._compute_model_per_coil(self.kspace_block_list[block_idx], kspace.permute(0, 1, 4, 2, 3))
+ self._compute_model_per_coil(
+ self.kspace_block_list[block_idx],
+ kspace.permute(0, 1, 4, 2, 3),
+ auxiliary_data,
+ )
.permute(0, 1, 3, 4, 2)
.contiguous()
)
else:
- kspace = self.kspace_block_list[block_idx](kspace.permute(0, 3, 1, 2)).permute(0, 2, 3, 1).contiguous()
+ if self.conv_modulation != ModConvType.NONE:
+ kspace = (
+ self.kspace_block_list[block_idx](kspace.permute(0, 3, 1, 2), auxiliary_data)
+ .permute(0, 2, 3, 1)
+ .contiguous()
+ )
+ else:
+ kspace = self.kspace_block_list[block_idx](kspace.permute(0, 3, 1, 2)).permute(0, 2, 3, 1).contiguous()
return kspace
- def _compute_model_per_coil(self, model: nn.Module, data: torch.Tensor) -> torch.Tensor:
+ def _compute_model_per_coil(
+ self,
+ model: nn.Module,
+ data: torch.Tensor,
+ auxiliary_data: Optional[torch.Tensor] = None,
+ ) -> torch.Tensor:
output = []
for idx in range(data.size(self._coil_dim)):
subselected_data = data.select(self._coil_dim, idx)
- output.append(model(subselected_data))
+ if self.conv_modulation != ModConvType.NONE:
+ output.append(model(subselected_data, auxiliary_data))
+ else:
+ output.append(model(subselected_data))
return torch.stack(output, dim=self._coil_dim)
def _forward_operator(
- self, image: torch.Tensor, sampling_mask: torch.Tensor, sensitivity_map: torch.Tensor
+ self,
+ image: torch.Tensor,
+ sampling_mask: torch.Tensor,
+ sensitivity_map: torch.Tensor,
) -> torch.Tensor:
return T.apply_mask(
- self.forward_operator(T.expand_operator(image, sensitivity_map, self._coil_dim), dim=self._spatial_dims),
+ self.forward_operator(
+ T.expand_operator(image, sensitivity_map, self._coil_dim),
+ dim=self._spatial_dims,
+ ),
sampling_mask,
return_mask=False,
)
def _backward_operator(
- self, kspace: torch.Tensor, sampling_mask: torch.Tensor, sensitivity_map: torch.Tensor
+ self,
+ kspace: torch.Tensor,
+ sampling_mask: torch.Tensor,
+ sensitivity_map: torch.Tensor,
) -> torch.Tensor:
return T.reduce_operator(
- self.backward_operator(T.apply_mask(kspace, sampling_mask, return_mask=False), self._spatial_dims),
+ self.backward_operator(
+ T.apply_mask(kspace, sampling_mask, return_mask=False),
+ self._spatial_dims,
+ ),
sensitivity_map,
self._coil_dim,
)
@@ -161,6 +248,7 @@ def forward(
masked_kspace: torch.Tensor,
sampling_mask: torch.Tensor,
sensitivity_map: torch.Tensor,
+ auxiliary_data: Optional[torch.Tensor] = None,
) -> torch.Tensor:
"""Computes forward pass of :class:`IterDualNet`.
@@ -172,6 +260,8 @@ def forward(
Sampling mask of shape (N, 1, height, width, 1).
sensitivity_map: torch.Tensor
Sensitivity map of shape (N, coil, height, width, complex=2).
+ auxiliary_data: torch.Tensor, optional
+ Auxiliary data for modulation of shape (N, aux_in_features).
Returns
-------
@@ -179,16 +269,21 @@ def forward(
Output image of shape (N, height, width, complex=2).
"""
x = T.reduce_operator(
- self.backward_operator(masked_kspace, self._spatial_dims), sensitivity_map, self._coil_dim
+ self.backward_operator(masked_kspace, self._spatial_dims),
+ sensitivity_map,
+ self._coil_dim,
)
for step in range(self.num_iter):
f = (
- self.forward_operator(T.expand_operator(x, sensitivity_map, self._coil_dim), dim=self._spatial_dims)
+ self.forward_operator(
+ T.expand_operator(x, sensitivity_map, self._coil_dim),
+ dim=self._spatial_dims,
+ )
if self.compute_per_coil
else self.forward_operator(x, dim=[d - 1 for d in self._spatial_dims])
)
- kspace_model_out = self._kspace_model(f, step)
+ kspace_model_out = self._kspace_model(f, step, auxiliary_data)
kspace_model_out = (
T.reduce_operator(
self.backward_operator(kspace_model_out, self._spatial_dims),
@@ -199,7 +294,7 @@ def forward(
else self.backward_operator(kspace_model_out, dim=[d - 1 for d in self._spatial_dims])
)
- img_model_out = self._image_model(x, step)
+ img_model_out = self._image_model(x, step, auxiliary_data)
dc_out = self._backward_operator(
self._forward_operator(x, sampling_mask, sensitivity_map) - masked_kspace,
diff --git a/direct/nn/iterdualnet/iterdualnet_engine.py b/direct/nn/iterdualnet/iterdualnet_engine.py
index 23dda7c5..6590c6ac 100644
--- a/direct/nn/iterdualnet/iterdualnet_engine.py
+++ b/direct/nn/iterdualnet/iterdualnet_engine.py
@@ -66,6 +66,7 @@ def forward_function(self, data: Dict[str, Any]) -> Tuple[torch.Tensor, None]:
masked_kspace=data["masked_kspace"],
sampling_mask=data["sampling_mask"],
sensitivity_map=data["sensitivity_map"],
+ auxiliary_data=self.auxiliary_data_from(data),
) # shape (batch, height, width)
output_kspace = None
diff --git a/direct/nn/jointicnet/config.py b/direct/nn/jointicnet/config.py
index 961769ed..392646e9 100644
--- a/direct/nn/jointicnet/config.py
+++ b/direct/nn/jointicnet/config.py
@@ -12,8 +12,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from dataclasses import dataclass
+from typing import Optional
from direct.config.defaults import ModelConfig
+from direct.nn.conv.modulated import ModConvActivation, ModConvType
@dataclass
@@ -29,3 +31,11 @@ class JointICNetConfig(ModelConfig):
sens_unet_num_filters: int = 8
sens_unet_num_pool_layers: int = 4
sens_unet_dropout: float = 0.0
+ conv_modulation: ModConvType = ModConvType.NONE
+ aux_in_features: Optional[int] = None
+ auxiliary_features: Optional[tuple[str, ...]] = None
+ log_aux: bool = False
+ fc_hidden_features: Optional[tuple[int, ...]] = None
+ fc_groups: int = 1
+ fc_activation: ModConvActivation = ModConvActivation.SIGMOID
+ num_weights: Optional[int] = None
diff --git a/direct/nn/jointicnet/jointicnet.py b/direct/nn/jointicnet/jointicnet.py
index ec20a4dd..d72a22ea 100644
--- a/direct/nn/jointicnet/jointicnet.py
+++ b/direct/nn/jointicnet/jointicnet.py
@@ -12,10 +12,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import annotations
+
+from typing import Optional
+
import torch
import torch.nn as nn
import direct.data.transforms as T
+from direct.nn.conv.modulated import ModConvActivation, ModConvType, ModulationParams
from direct.nn.unet.unet_2d import NormUnetModel2d, UnetModel2d
from direct.types import FFTOperator
@@ -24,9 +29,17 @@ class JointICNet(nn.Module):
"""Joint Deep Model-Based MR Image and Coil Sensitivity Reconstruction Network (Joint-ICNet) implementation as
presented in [1]_.
+ Supports conditional weight modulation as proposed in [2]_.
+
References
----------
- .. [1] Jun, Yohan, et al. “Joint Deep Model-Based MR Image and Coil Sensitivity Reconstruction Network (Joint-ICNet) for Fast MRI.” 2021 IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR), IEEE, 2021, pp. 5266–75. DOI.org (Crossref), https://doi.org/10.1109/CVPR46437.2021.00523.
+ .. [1] Jun, Yohan, et al. "Joint Deep Model-Based MR Image and Coil Sensitivity Reconstruction Network
+ (Joint-ICNet) for Fast MRI." 2021 IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR),
+ IEEE, 2021, pp. 5266-75. https://doi.org/10.1109/CVPR46437.2021.00523.
+
+ .. [2] Moriakov, N., Yiasemis, G., Sonke, J.-J. & Teuwen, J. (2026). Conditional Learned Reconstruction for
+ Medical Imaging. Proceedings of The 9th International Conference on Medical Imaging with Deep Learning,
+ PMLR 315:754-780. https://proceedings.mlr.press/v315/moriakov26a.html
"""
def __init__(
@@ -35,6 +48,12 @@ def __init__(
backward_operator: FFTOperator,
num_iter: int = 10,
use_norm_unet: bool = False,
+ conv_modulation: ModConvType = ModConvType.NONE,
+ aux_in_features: Optional[int] = None,
+ fc_hidden_features: Optional[tuple[int] | int] = None,
+ fc_groups: int = 1,
+ fc_activation: ModConvActivation = ModConvActivation.SIGMOID,
+ num_weights: Optional[int] = None,
**kwargs,
):
"""Inits :class:`JointICNet`.
@@ -49,6 +68,18 @@ def __init__(
Number of unrolled iterations. Default: 10.
use_norm_unet: bool
If True, a Normalized U-Net is used. Default: False.
+ conv_modulation : ModConvType
+ Modulation type for convolutional layers. Default: ModConvType.NONE.
+ aux_in_features : int, optional
+ Number of features in the auxiliary input for modulation.
+ fc_hidden_features : int or tuple of int, optional
+ Hidden features in the modulation MLP.
+ fc_groups : int
+ Groups for modulation MLP output. Default: 1.
+ fc_activation : ModConvActivation
+ Activation after modulation MLP. Default: ModConvActivation.SIGMOID.
+ num_weights : int, optional
+ Number of weight bases for ModConvType.SUM.
kwargs: dict
Image, k-space and sensitivity-map U-Net models keyword-arguments.
"""
@@ -57,15 +88,26 @@ def __init__(
self.forward_operator = forward_operator
self.backward_operator = backward_operator
self.num_iter = num_iter
+ self.conv_modulation = conv_modulation
unet_architecture = NormUnetModel2d if use_norm_unet else UnetModel2d
+ modulation_params = ModulationParams(
+ modulation=conv_modulation,
+ aux_in_features=aux_in_features,
+ fc_hidden_features=fc_hidden_features,
+ fc_groups=fc_groups,
+ fc_activation=fc_activation,
+ num_weights=num_weights,
+ )
+
self.image_model = unet_architecture(
in_channels=2,
out_channels=2,
num_filters=kwargs.get("image_unet_num_filters", 8),
num_pool_layers=kwargs.get("image_unet_num_pool_layers", 4),
dropout_probability=kwargs.get("image_unet_dropout", 0.0),
+ modulation_params=modulation_params,
)
self.kspace_model = unet_architecture(
in_channels=2,
@@ -73,6 +115,7 @@ def __init__(
num_filters=kwargs.get("kspace_unet_num_filters", 8),
num_pool_layers=kwargs.get("kspace_unet_num_pool_layers", 4),
dropout_probability=kwargs.get("kspace_unet_dropout", 0.0),
+ modulation_params=modulation_params,
)
self.sens_model = unet_architecture(
in_channels=2,
@@ -80,6 +123,7 @@ def __init__(
num_filters=kwargs.get("sens_unet_num_filters", 8),
num_pool_layers=kwargs.get("sens_unet_num_pool_layers", 4),
dropout_probability=kwargs.get("sens_unet_dropout", 0.0),
+ modulation_params=modulation_params,
)
self.conv_out = nn.Conv2d(in_channels=2, out_channels=2, kernel_size=1)
@@ -94,40 +138,65 @@ def __init__(
self._complex_dim = -1
self._spatial_dims = (2, 3)
- def _image_model(self, image: torch.Tensor) -> torch.Tensor:
+ def _image_model(self, image: torch.Tensor, auxiliary_data: Optional[torch.Tensor] = None) -> torch.Tensor:
image = image.permute(0, 3, 1, 2)
+ if self.conv_modulation != ModConvType.NONE:
+ return self.image_model(image, auxiliary_data).permute(0, 2, 3, 1).contiguous()
return self.image_model(image).permute(0, 2, 3, 1).contiguous()
- def _kspace_model(self, kspace: torch.Tensor) -> torch.Tensor:
+ def _kspace_model(self, kspace: torch.Tensor, auxiliary_data: Optional[torch.Tensor] = None) -> torch.Tensor:
kspace = kspace.permute(0, 3, 1, 2)
+ if self.conv_modulation != ModConvType.NONE:
+ return self.kspace_model(kspace, auxiliary_data).permute(0, 2, 3, 1).contiguous()
return self.kspace_model(kspace).permute(0, 2, 3, 1).contiguous()
- def _sens_model(self, sensitivity_map: torch.Tensor) -> torch.Tensor:
+ def _sens_model(
+ self,
+ sensitivity_map: torch.Tensor,
+ auxiliary_data: Optional[torch.Tensor] = None,
+ ) -> torch.Tensor:
return (
- self._compute_model_per_coil(self.sens_model, sensitivity_map.permute(0, 1, 4, 2, 3))
+ self._compute_model_per_coil(self.sens_model, sensitivity_map.permute(0, 1, 4, 2, 3), auxiliary_data)
.permute(0, 1, 3, 4, 2)
.contiguous()
)
- def _compute_model_per_coil(self, model: nn.Module, data: torch.Tensor) -> torch.Tensor:
+ def _compute_model_per_coil(
+ self,
+ model: nn.Module,
+ data: torch.Tensor,
+ auxiliary_data: Optional[torch.Tensor] = None,
+ ) -> torch.Tensor:
output = []
for idx in range(data.size(self._coil_dim)):
subselected_data = data.select(self._coil_dim, idx)
- output.append(model(subselected_data))
+ if self.conv_modulation != ModConvType.NONE:
+ output.append(model(subselected_data, auxiliary_data))
+ else:
+ output.append(model(subselected_data))
return torch.stack(output, dim=self._coil_dim)
def _forward_operator(
- self, image: torch.Tensor, sampling_mask: torch.Tensor, sensitivity_map: torch.Tensor
+ self,
+ image: torch.Tensor,
+ sampling_mask: torch.Tensor,
+ sensitivity_map: torch.Tensor,
) -> torch.Tensor:
forward = torch.where(
sampling_mask == 0,
torch.tensor([0.0], dtype=image.dtype).to(image.device),
- self.forward_operator(T.expand_operator(image, sensitivity_map, self._coil_dim), dim=self._spatial_dims),
+ self.forward_operator(
+ T.expand_operator(image, sensitivity_map, self._coil_dim),
+ dim=self._spatial_dims,
+ ),
)
return forward
def _backward_operator(
- self, kspace: torch.Tensor, sampling_mask: torch.Tensor, sensitivity_map: torch.Tensor
+ self,
+ kspace: torch.Tensor,
+ sampling_mask: torch.Tensor,
+ sensitivity_map: torch.Tensor,
) -> torch.Tensor:
backward = T.reduce_operator(
self.backward_operator(
@@ -148,6 +217,7 @@ def forward(
masked_kspace: torch.Tensor,
sampling_mask: torch.Tensor,
sensitivity_map: torch.Tensor,
+ auxiliary_data: Optional[torch.Tensor] = None,
) -> torch.Tensor:
"""Computes forward pass of :class:`JointICNet`.
@@ -159,6 +229,8 @@ def forward(
Sampling mask of shape (N, 1, height, width, 1).
sensitivity_map: torch.Tensor
Sensitivity map of shape (N, coil, height, width, complex=2).
+ auxiliary_data: torch.Tensor, optional
+ Auxiliary data for modulation of shape (N, aux_in_features).
Returns
-------
@@ -190,7 +262,10 @@ def forward(
+ self.reg_param_C[curr_iter]
* (
sensitivity_map
- - self._sens_model(self.backward_operator(masked_kspace, dim=self._spatial_dims))
+ - self._sens_model(
+ self.backward_operator(masked_kspace, dim=self._spatial_dims),
+ auxiliary_data,
+ )
)
)
)
@@ -209,12 +284,13 @@ def forward(
sampling_mask,
sensitivity_map,
)
- + self.reg_param_I[curr_iter] * (input_image - self._image_model(input_image))
+ + self.reg_param_I[curr_iter] * (input_image - self._image_model(input_image, auxiliary_data))
+ self.reg_param_F[curr_iter]
* (
input_image
- self.backward_operator(
- self._kspace_model(input_kspace), dim=tuple(d - 1 for d in self._spatial_dims)
+ self._kspace_model(input_kspace, auxiliary_data),
+ dim=tuple(d - 1 for d in self._spatial_dims),
)
)
)
diff --git a/direct/nn/jointicnet/jointicnet_engine.py b/direct/nn/jointicnet/jointicnet_engine.py
index 8b40c611..504f134e 100644
--- a/direct/nn/jointicnet/jointicnet_engine.py
+++ b/direct/nn/jointicnet/jointicnet_engine.py
@@ -51,6 +51,7 @@ def forward_function(self, data: Dict[str, Any]) -> Tuple[torch.Tensor, None]:
masked_kspace=data["masked_kspace"],
sampling_mask=data["sampling_mask"],
sensitivity_map=data["sensitivity_map"],
+ auxiliary_data=self.auxiliary_data_from(data),
) # shape (batch, height, width)
output_kspace = None
diff --git a/direct/nn/kikinet/config.py b/direct/nn/kikinet/config.py
index 826c8b7a..888a0604 100644
--- a/direct/nn/kikinet/config.py
+++ b/direct/nn/kikinet/config.py
@@ -12,8 +12,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from dataclasses import dataclass
+from typing import Optional
from direct.config.defaults import ModelConfig
+from direct.nn.conv.modulated import ModConvActivation, ModConvType
@dataclass
@@ -38,3 +40,11 @@ class KIKINetConfig(ModelConfig):
kspace_unet_num_pool_layers: int = 4
kspace_unet_dropout_probability: float = 0.0
normalize: bool = False
+ conv_modulation: ModConvType = ModConvType.NONE
+ aux_in_features: Optional[int] = None
+ auxiliary_features: Optional[tuple[str, ...]] = None
+ log_aux: bool = False
+ fc_hidden_features: Optional[tuple[int, ...]] = None
+ fc_groups: int = 1
+ fc_activation: ModConvActivation = ModConvActivation.SIGMOID
+ num_weights: Optional[int] = None
diff --git a/direct/nn/kikinet/kikinet.py b/direct/nn/kikinet/kikinet.py
index d2b9d563..3a6de51e 100644
--- a/direct/nn/kikinet/kikinet.py
+++ b/direct/nn/kikinet/kikinet.py
@@ -11,6 +11,8 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import annotations
+
from typing import Optional
import torch
@@ -18,6 +20,7 @@
import direct.data.transforms as T
from direct.nn.conv.conv import Conv2d
+from direct.nn.conv.modulated import ModConvActivation, ModConvType, ModulationParams
from direct.nn.crossdomain.multicoil import MultiCoil
from direct.nn.didn.didn import DIDN
from direct.nn.mwcnn.mwcnn import MWCNN
@@ -28,10 +31,18 @@
class KIKINet(nn.Module):
"""Based on KIKINet implementation [1]_. Modified to work with multi-coil k-space data.
+ Supports conditional weight modulation as proposed in [2]_.
+
References
----------
- .. [1] Eo, Taejoon, et al. “KIKI-Net: Cross-Domain Convolutional Neural Networks for Reconstructing Undersampled Magnetic Resonance Images.” Magnetic Resonance in Medicine, vol. 80, no. 5, Nov. 2018, pp. 2188–201. PubMed, https://doi.org/10.1002/mrm.27201.
+ .. [1] Eo, Taejoon, et al. "KIKI-Net: Cross-Domain Convolutional Neural Networks for Reconstructing Undersampled
+ Magnetic Resonance Images." Magnetic Resonance in Medicine, vol. 80, no. 5, Nov. 2018, pp. 2188-201.
+ https://doi.org/10.1002/mrm.27201.
+
+ .. [2] Moriakov, N., Yiasemis, G., Sonke, J.-J. & Teuwen, J. (2026). Conditional Learned Reconstruction for
+ Medical Imaging. Proceedings of The 9th International Conference on Medical Imaging with Deep Learning,
+ PMLR 315:754-780. https://proceedings.mlr.press/v315/moriakov26a.html
"""
def __init__(
@@ -42,6 +53,12 @@ def __init__(
kspace_model_architecture: str = "DIDN",
num_iter: int = 2,
normalize: bool = False,
+ conv_modulation: ModConvType = ModConvType.NONE,
+ aux_in_features: Optional[int] = None,
+ fc_hidden_features: Optional[tuple[int] | int] = None,
+ fc_groups: int = 1,
+ fc_activation: ModConvActivation = ModConvActivation.SIGMOID,
+ num_weights: Optional[int] = None,
**kwargs,
):
"""Inits :class:`KIKINet`.
@@ -60,10 +77,32 @@ def __init__(
Number of unrolled iterations.
normalize: bool
If true, input is normalised based on input scaling_factor.
+ conv_modulation : ModConvType
+ Modulation type for convolutional layers. Default: ModConvType.NONE.
+ aux_in_features : int, optional
+ Number of features in the auxiliary input for modulation.
+ fc_hidden_features : int or tuple of int, optional
+ Hidden features in the modulation MLP.
+ fc_groups : int
+ Groups for modulation MLP output. Default: 1.
+ fc_activation : ModConvActivation
+ Activation after modulation MLP. Default: ModConvActivation.SIGMOID.
+ num_weights : int, optional
+ Number of weight bases for ModConvType.SUM.
kwargs: dict
Keyword arguments for model architectures.
"""
super().__init__()
+
+ modulation_params = ModulationParams(
+ modulation=conv_modulation,
+ aux_in_features=aux_in_features,
+ fc_hidden_features=fc_hidden_features,
+ fc_groups=fc_groups,
+ fc_activation=fc_activation,
+ num_weights=num_weights,
+ )
+
image_model: nn.Module
if image_model_architecture == "MWCNN":
image_model = MWCNN(
@@ -72,6 +111,7 @@ def __init__(
num_scales=kwargs.get("image_mwcnn_num_scales", 4),
bias=kwargs.get("image_mwcnn_bias", False),
batchnorm=kwargs.get("image_mwcnn_batchnorm", False),
+ modulation_params=modulation_params,
)
elif image_model_architecture in ["UNET", "NORMUNET"]:
unet = UnetModel2d if image_model_architecture == "UNET" else NormUnetModel2d
@@ -81,11 +121,12 @@ def __init__(
num_filters=kwargs.get("image_unet_num_filters", 8),
num_pool_layers=kwargs.get("image_unet_num_pool_layers", 4),
dropout_probability=kwargs.get("image_unet_dropout_probability", 0.0),
+ modulation_params=modulation_params,
)
else:
raise NotImplementedError(
- f"XPDNet is currently implemented only with image_model_architecture == 'MWCNN', 'UNET' or 'NORMUNET."
- f"Got {image_model_architecture}."
+ f"KIKINet is currently implemented only with image_model_architecture == 'MWCNN', 'UNET' or 'NORMUNET'."
+ f" Got {image_model_architecture}."
)
kspace_model: nn.Module
@@ -96,6 +137,7 @@ def __init__(
hidden_channels=kwargs.get("kspace_conv_hidden_channels", 16),
n_convs=kwargs.get("kspace_conv_n_convs", 4),
batchnorm=kwargs.get("kspace_conv_batchnorm", False),
+ modulation_params=modulation_params,
)
elif kspace_model_architecture == "DIDN":
kspace_model = DIDN(
@@ -104,6 +146,7 @@ def __init__(
hidden_channels=kwargs.get("kspace_didn_hidden_channels", 16),
num_dubs=kwargs.get("kspace_didn_num_dubs", 6),
num_convs_recon=kwargs.get("kspace_didn_num_convs_recon", 9),
+ modulation_params=modulation_params,
)
elif kspace_model_architecture in ["UNET", "NORMUNET"]:
unet = UnetModel2d if kspace_model_architecture == "UNET" else NormUnetModel2d
@@ -113,10 +156,11 @@ def __init__(
num_filters=kwargs.get("kspace_unet_num_filters", 8),
num_pool_layers=kwargs.get("kspace_unet_num_pool_layers", 4),
dropout_probability=kwargs.get("kspace_unet_dropout_probability", 0.0),
+ modulation_params=modulation_params,
)
else:
raise NotImplementedError(
- f"XPDNet is currently implemented for kspace_model_architecture == 'CONV', 'DIDN',"
+ f"KIKINet is currently implemented for kspace_model_architecture == 'CONV', 'DIDN',"
f" 'UNET' or 'NORMUNET'. Got kspace_model_architecture == {kspace_model_architecture}."
)
@@ -131,6 +175,7 @@ def __init__(
self.backward_operator = backward_operator
self.num_iter = num_iter
self.normalize = normalize
+ self.conv_modulation = conv_modulation
def forward(
self,
@@ -138,6 +183,7 @@ def forward(
sampling_mask: torch.Tensor,
sensitivity_map: torch.Tensor,
scaling_factor: Optional[torch.Tensor] = None,
+ auxiliary_data: Optional[torch.Tensor] = None,
) -> torch.Tensor:
"""Computes forward pass of :class:`KIKINet`.
@@ -151,6 +197,8 @@ def forward(
Sensitivity map of shape (N, coil, height, width, complex=2).
scaling_factor: Optional[torch.Tensor]
Scaling factor of shape (N,). If None, no scaling is applied. Default: None.
+ auxiliary_data: torch.Tensor, optional
+ Auxiliary data for modulation of shape (N, aux_in_features).
Returns
-------
@@ -163,7 +211,11 @@ def forward(
kspace = kspace / (scaling_factor**2).view(-1, 1, 1, 1, 1)
for idx in range(self.num_iter):
- kspace = self.kspace_model_list[idx](kspace.permute(0, 1, 4, 2, 3)).permute(0, 1, 3, 4, 2)
+ kspace_permuted = kspace.permute(0, 1, 4, 2, 3)
+ if self.conv_modulation != ModConvType.NONE:
+ kspace = self.kspace_model_list[idx](kspace_permuted, auxiliary_data).permute(0, 1, 3, 4, 2)
+ else:
+ kspace = self.kspace_model_list[idx](kspace_permuted).permute(0, 1, 3, 4, 2)
image = T.reduce_operator(
self.backward_operator(
@@ -178,14 +230,18 @@ def forward(
self._coil_dim,
)
- image = self.image_model_list[idx](image.permute(0, 3, 1, 2)).permute(0, 2, 3, 1)
+ if self.conv_modulation != ModConvType.NONE:
+ image = self.image_model_list[idx](image.permute(0, 3, 1, 2), auxiliary_data).permute(0, 2, 3, 1)
+ else:
+ image = self.image_model_list[idx](image.permute(0, 3, 1, 2)).permute(0, 2, 3, 1)
if idx < self.num_iter - 1:
kspace = torch.where(
sampling_mask == 0,
torch.tensor([0.0], dtype=image.dtype).to(image.device),
self.forward_operator(
- T.expand_operator(image, sensitivity_map, self._coil_dim), dim=self._spatial_dims
+ T.expand_operator(image, sensitivity_map, self._coil_dim),
+ dim=self._spatial_dims,
),
)
diff --git a/direct/nn/kikinet/kikinet_engine.py b/direct/nn/kikinet/kikinet_engine.py
index f4159077..3dd3ceac 100644
--- a/direct/nn/kikinet/kikinet_engine.py
+++ b/direct/nn/kikinet/kikinet_engine.py
@@ -52,6 +52,7 @@ def forward_function(self, data: Dict[str, Any]) -> Tuple[torch.Tensor, None]:
sampling_mask=data["sampling_mask"],
sensitivity_map=data["sensitivity_map"],
scaling_factor=data["scaling_factor"],
+ auxiliary_data=self.auxiliary_data_from(data),
) # shape (batch, height, width, complex[=2])
output_kspace = None
diff --git a/direct/nn/lpd/config.py b/direct/nn/lpd/config.py
index 0171c384..bbc3fa80 100644
--- a/direct/nn/lpd/config.py
+++ b/direct/nn/lpd/config.py
@@ -1,8 +1,21 @@
-# coding=utf-8
-# Copyright (c) DIRECT Contributors
+# Copyright 2025 AI for Oncology Research Group. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
from dataclasses import dataclass
+from typing import Optional
from direct.config.defaults import ModelConfig
+from direct.nn.conv.modulated import ModConvActivation, ModConvType
@dataclass
@@ -28,3 +41,11 @@ class LPDNetConfig(ModelConfig):
dual_unet_num_filters: int = 8
dual_unet_num_pool_layers: int = 4
dual_unet_dropout_probability: float = 0.0
+ conv_modulation: ModConvType = ModConvType.NONE
+ aux_in_features: Optional[int] = None
+ auxiliary_features: Optional[tuple[str, ...]] = None
+ log_aux: bool = False
+ fc_hidden_features: Optional[tuple[int, ...]] = None
+ fc_groups: int = 1
+ fc_activation: ModConvActivation = ModConvActivation.SIGMOID
+ num_weights: Optional[int] = None
diff --git a/direct/nn/lpd/lpd.py b/direct/nn/lpd/lpd.py
index 197dd298..7d021709 100644
--- a/direct/nn/lpd/lpd.py
+++ b/direct/nn/lpd/lpd.py
@@ -12,11 +12,16 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import annotations
+
+from typing import Optional
+
import torch
import torch.nn as nn
import direct.data.transforms as T
from direct.nn.conv.conv import Conv2d
+from direct.nn.conv.modulated import ModConvActivation, ModConvType, ModulationParams
from direct.nn.didn.didn import DIDN
from direct.nn.mwcnn.mwcnn import MWCNN
from direct.nn.unet.unet_2d import NormUnetModel2d, UnetModel2d
@@ -26,16 +31,19 @@
class DualNet(nn.Module):
"""Dual Network for Learned Primal Dual Network."""
- def __init__(self, num_dual: int, **kwargs):
+ def __init__(self, num_dual: int, conv_modulation: ModConvType = ModConvType.NONE, **kwargs):
"""Inits :class:`DualNet`.
Parameters
----------
num_dual: int
Number of dual for LPD algorithm.
+ conv_modulation: ModConvType
+ Modulation type. Default: ModConvType.NONE.
kwargs: dict
"""
super().__init__()
+ self.conv_modulation = conv_modulation
if kwargs.get("dual_architectue") is None:
n_hidden = kwargs.get("n_hidden")
@@ -54,7 +62,12 @@ def __init__(self, num_dual: int, **kwargs):
self.dual_block = kwargs.get("dual_architectue") # type: ignore
@staticmethod
- def compute_model_per_coil(model: nn.Module, data: torch.Tensor) -> torch.Tensor:
+ def compute_model_per_coil(
+ model: nn.Module,
+ data: torch.Tensor,
+ conv_modulation: ModConvType = ModConvType.NONE,
+ auxiliary_data: Optional[torch.Tensor] = None,
+ ) -> torch.Tensor:
"""Computes model per coil.
Parameters
@@ -63,6 +76,10 @@ def compute_model_per_coil(model: nn.Module, data: torch.Tensor) -> torch.Tensor
Model to compute.
data: torch.Tensor
Multi-coil input.
+ conv_modulation: ModConvType
+ Modulation type.
+ auxiliary_data: torch.Tensor, optional
+ Auxiliary data for modulation.
Returns
-------
@@ -72,28 +89,42 @@ def compute_model_per_coil(model: nn.Module, data: torch.Tensor) -> torch.Tensor
output = []
for idx in range(data.size(1)):
subselected_data = data.select(1, idx)
- output.append(model(subselected_data))
+ if conv_modulation != ModConvType.NONE:
+ output.append(model(subselected_data, auxiliary_data))
+ else:
+ output.append(model(subselected_data))
return torch.stack(output, dim=1)
- def forward(self, h: torch.Tensor, forward_f: torch.Tensor, g: torch.Tensor) -> torch.Tensor:
+ def forward(
+ self,
+ h: torch.Tensor,
+ forward_f: torch.Tensor,
+ g: torch.Tensor,
+ auxiliary_data: Optional[torch.Tensor] = None,
+ ) -> torch.Tensor:
inp = torch.cat([h, forward_f, g], dim=-1).permute(0, 1, 4, 2, 3)
assert self.dual_block is not None
- return self.compute_model_per_coil(self.dual_block, inp).permute(0, 1, 3, 4, 2)
+ return self.compute_model_per_coil(self.dual_block, inp, self.conv_modulation, auxiliary_data).permute(
+ 0, 1, 3, 4, 2
+ )
class PrimalNet(nn.Module):
"""Primal Network for Learned Primal Dual Network."""
- def __init__(self, num_primal: int, **kwargs):
+ def __init__(self, num_primal: int, conv_modulation: ModConvType = ModConvType.NONE, **kwargs):
"""Inits :class:`PrimalNet`.
Parameters
----------
num_primal: int
Number of primal for LPD algorithm.
+ conv_modulation: ModConvType
+ Modulation type. Default: ModConvType.NONE.
"""
super().__init__()
+ self.conv_modulation = conv_modulation
if kwargs.get("primal_architectue") is None:
n_hidden = kwargs.get("n_hidden")
@@ -111,19 +142,33 @@ def __init__(self, num_primal: int, **kwargs):
else:
self.primal_block = kwargs.get("primal_architectue") # type: ignore
- def forward(self, f: torch.Tensor, backward_h: torch.Tensor) -> torch.Tensor:
+ def forward(
+ self,
+ f: torch.Tensor,
+ backward_h: torch.Tensor,
+ auxiliary_data: Optional[torch.Tensor] = None,
+ ) -> torch.Tensor:
inp = torch.cat([f, backward_h], dim=-1).permute(0, 3, 1, 2)
assert self.primal_block is not None
+ if self.conv_modulation != ModConvType.NONE:
+ return self.primal_block(inp, auxiliary_data).permute(0, 2, 3, 1)
return self.primal_block(inp).permute(0, 2, 3, 1)
class LPDNet(nn.Module):
"""Learned Primal Dual network implementation inspired by [1]_.
+ Supports conditional weight modulation as proposed in [2]_.
+
References
----------
- .. [1] Adler, Jonas, and Ozan Öktem. “Learned Primal-Dual Reconstruction.” IEEE Transactions on Medical Imaging, vol. 37, no. 6, June 2018, pp. 1322–32. arXiv.org, https://doi.org/10.1109/TMI.2018.2799231.
+ .. [1] Adler, Jonas, and Ozan Oktem. "Learned Primal-Dual Reconstruction." IEEE Transactions on Medical Imaging,
+ vol. 37, no. 6, June 2018, pp. 1322-32. https://doi.org/10.1109/TMI.2018.2799231.
+
+ .. [2] Moriakov, N., Yiasemis, G., Sonke, J.-J. & Teuwen, J. (2026). Conditional Learned Reconstruction for
+ Medical Imaging. Proceedings of The 9th International Conference on Medical Imaging with Deep Learning,
+ PMLR 315:754-780. https://proceedings.mlr.press/v315/moriakov26a.html
"""
def __init__(
@@ -135,6 +180,12 @@ def __init__(
num_dual: int,
primal_model_architecture: str = "MWCNN",
dual_model_architecture: str = "DIDN",
+ conv_modulation: ModConvType = ModConvType.NONE,
+ aux_in_features: Optional[int] = None,
+ fc_hidden_features: Optional[tuple[int] | int] = None,
+ fc_groups: int = 1,
+ fc_activation: ModConvActivation = ModConvActivation.SIGMOID,
+ num_weights: Optional[int] = None,
**kwargs,
):
"""Inits :class:`LPDNet`.
@@ -155,6 +206,18 @@ def __init__(
Primal model architecture. Currently only implemented for MWCNN and (NORM)UNET. Default: 'MWCNN'.
dual_model_architecture: str
Dual model architecture. Currently only implemented for CONV and DIDN and (NORM)UNET. Default: 'DIDN'.
+ conv_modulation : ModConvType
+ Modulation type for convolutional layers. Default: ModConvType.NONE.
+ aux_in_features : int, optional
+ Number of features in the auxiliary input for modulation.
+ fc_hidden_features : int or tuple of int, optional
+ Hidden features in the modulation MLP.
+ fc_groups : int
+ Groups for modulation MLP output. Default: 1.
+ fc_activation : ModConvActivation
+ Activation after modulation MLP. Default: ModConvActivation.SIGMOID.
+ num_weights : int, optional
+ Number of weight bases for ModConvType.SUM.
kwargs: dict
Keyword arguments for model architectures.
"""
@@ -165,6 +228,16 @@ def __init__(
self.num_iter = num_iter
self.num_primal = num_primal
self.num_dual = num_dual
+ self.conv_modulation = conv_modulation
+
+ modulation_params = ModulationParams(
+ modulation=conv_modulation,
+ aux_in_features=aux_in_features,
+ fc_hidden_features=fc_hidden_features,
+ fc_groups=fc_groups,
+ fc_activation=fc_activation,
+ num_weights=num_weights,
+ )
primal_model: nn.Module
if primal_model_architecture == "MWCNN":
@@ -176,6 +249,7 @@ def __init__(
num_scales=kwargs.get("primal_mwcnn_num_scales", 4),
bias=kwargs.get("primal_mwcnn_bias", False),
batchnorm=kwargs.get("primal_mwcnn_batchnorm", False),
+ modulation_params=modulation_params,
),
nn.Conv2d(2 * (num_primal + 1), 2 * num_primal, kernel_size=1),
]
@@ -188,11 +262,12 @@ def __init__(
num_filters=kwargs.get("primal_unet_num_filters", 8),
num_pool_layers=kwargs.get("primal_unet_num_pool_layers", 4),
dropout_probability=kwargs.get("primal_unet_dropout_probability", 0.0),
+ modulation_params=modulation_params,
)
else:
raise NotImplementedError(
- f"XPDNet is currently implemented only with primal_model_architecture == 'MWCNN', 'UNET' or 'NORMUNET."
- f"Got {primal_model_architecture}."
+ f"LPDNet is currently implemented only with primal_model_architecture == 'MWCNN', 'UNET' or 'NORMUNET'."
+ f" Got {primal_model_architecture}."
)
dual_model: nn.Module
if dual_model_architecture == "CONV":
@@ -202,6 +277,7 @@ def __init__(
hidden_channels=kwargs.get("dual_conv_hidden_channels", 16),
n_convs=kwargs.get("dual_conv_n_convs", 4),
batchnorm=kwargs.get("dual_conv_batchnorm", False),
+ modulation_params=modulation_params,
)
elif dual_model_architecture == "DIDN":
dual_model = DIDN(
@@ -210,6 +286,7 @@ def __init__(
hidden_channels=kwargs.get("dual_didn_hidden_channels", 16),
num_dubs=kwargs.get("dual_didn_num_dubs", 6),
num_convs_recon=kwargs.get("dual_didn_num_convs_recon", 9),
+ modulation_params=modulation_params,
)
elif dual_model_architecture in ["UNET", "NORMUNET"]:
unet = UnetModel2d if dual_model_architecture == "UNET" else NormUnetModel2d
@@ -219,10 +296,11 @@ def __init__(
num_filters=kwargs.get("dual_unet_num_filters", 8),
num_pool_layers=kwargs.get("dual_unet_num_pool_layers", 4),
dropout_probability=kwargs.get("dual_unet_dropout_probability", 0.0),
+ modulation_params=modulation_params,
)
else:
raise NotImplementedError(
- f"XPDNet is currently implemented for dual_model_architecture == 'CONV', 'DIDN',"
+ f"LPDNet is currently implemented for dual_model_architecture == 'CONV', 'DIDN',"
f" 'UNET' or 'NORMUNET'. Got dual_model_architecture == {dual_model_architecture}."
)
@@ -231,22 +309,47 @@ def __init__(
self._spatial_dims = (2, 3)
self.primal_net = nn.ModuleList(
- [PrimalNet(num_primal, primal_architectue=primal_model) for _ in range(num_iter)]
+ [
+ PrimalNet(
+ num_primal,
+ conv_modulation=conv_modulation,
+ primal_architectue=primal_model,
+ )
+ for _ in range(num_iter)
+ ]
+ )
+ self.dual_net = nn.ModuleList(
+ [
+ DualNet(
+ num_dual,
+ conv_modulation=conv_modulation,
+ dual_architectue=dual_model,
+ )
+ for _ in range(num_iter)
+ ]
)
- self.dual_net = nn.ModuleList([DualNet(num_dual, dual_architectue=dual_model) for _ in range(num_iter)])
def _forward_operator(
- self, image: torch.Tensor, sampling_mask: torch.Tensor, sensitivity_map: torch.Tensor
+ self,
+ image: torch.Tensor,
+ sampling_mask: torch.Tensor,
+ sensitivity_map: torch.Tensor,
) -> torch.Tensor:
forward = torch.where(
sampling_mask == 0,
torch.tensor([0.0], dtype=image.dtype).to(image.device),
- self.forward_operator(T.expand_operator(image, sensitivity_map, self._coil_dim), dim=self._spatial_dims),
+ self.forward_operator(
+ T.expand_operator(image, sensitivity_map, self._coil_dim),
+ dim=self._spatial_dims,
+ ),
)
return forward
def _backward_operator(
- self, kspace: torch.Tensor, sampling_mask: torch.Tensor, sensitivity_map: torch.Tensor
+ self,
+ kspace: torch.Tensor,
+ sampling_mask: torch.Tensor,
+ sensitivity_map: torch.Tensor,
) -> torch.Tensor:
backward = T.reduce_operator(
self.backward_operator(
@@ -267,6 +370,7 @@ def forward(
masked_kspace: torch.Tensor,
sensitivity_map: torch.Tensor,
sampling_mask: torch.Tensor,
+ auxiliary_data: Optional[torch.Tensor] = None,
) -> torch.Tensor:
"""Computes forward pass of :class:`LPDNet`.
@@ -278,6 +382,8 @@ def forward(
Sensitivity map of shape (N, coil, height, width, complex=2).
sampling_mask: torch.Tensor
Sampling mask of shape (N, 1, height, width, 1).
+ auxiliary_data: torch.Tensor, optional
+ Auxiliary data for modulation of shape (N, aux_in_features).
Returns
-------
@@ -292,13 +398,18 @@ def forward(
# Dual
f_2 = primal_buffer[..., 2:4].clone()
dual_buffer = self.dual_net[curr_iter](
- dual_buffer, self._forward_operator(f_2, sampling_mask, sensitivity_map), masked_kspace
+ dual_buffer,
+ self._forward_operator(f_2, sampling_mask, sensitivity_map),
+ masked_kspace,
+ auxiliary_data,
)
# Primal
h_1 = dual_buffer[..., 0:2].clone()
primal_buffer = self.primal_net[curr_iter](
- primal_buffer, self._backward_operator(h_1, sampling_mask, sensitivity_map)
+ primal_buffer,
+ self._backward_operator(h_1, sampling_mask, sensitivity_map),
+ auxiliary_data,
)
output = primal_buffer[..., 0:2]
diff --git a/direct/nn/lpd/lpd_engine.py b/direct/nn/lpd/lpd_engine.py
index 6c77ab45..a169a156 100644
--- a/direct/nn/lpd/lpd_engine.py
+++ b/direct/nn/lpd/lpd_engine.py
@@ -52,5 +52,6 @@ def forward_function(self, data: Dict[str, Any]) -> Tuple[torch.Tensor, None]:
masked_kspace=data["masked_kspace"],
sampling_mask=data["sampling_mask"],
sensitivity_map=data["sensitivity_map"],
+ auxiliary_data=self.auxiliary_data_from(data),
) # shape (batch, height, width)
return output_image, None
diff --git a/direct/nn/mri_models.py b/direct/nn/mri_models.py
index f6806547..d5d9491d 100644
--- a/direct/nn/mri_models.py
+++ b/direct/nn/mri_models.py
@@ -100,6 +100,17 @@ def forward_function(self, data: Dict[str, Any]) -> Tuple[TensorOrNone, TensorOr
"""
raise NotImplementedError("Must be implemented by child class.")
+ @staticmethod
+ def auxiliary_data_from(data: Dict[str, Any]) -> TensorOrNone:
+ """Return auxiliary conditioning from a batch dict, if present."""
+ return data.get("auxiliary_data")
+
+ def _attach_auxiliary_data(self, data: Dict[str, Any]) -> None:
+ """Populate ``auxiliary_data`` when modulated convolutions are enabled."""
+ from direct.nn.conv.modulated import prepare_auxiliary_data
+
+ data["auxiliary_data"] = prepare_auxiliary_data(data, getattr(self.cfg, "model", None))
+
def _do_iteration(
self,
data: Dict[str, Any],
@@ -131,6 +142,7 @@ def _do_iteration(
regularizer_fns = {}
data = dict_to_device(data, self.device)
+ self._attach_auxiliary_data(data)
output_image: TensorOrNone
output_kspace: TensorOrNone
@@ -864,7 +876,12 @@ def reconstruct_volumes( # type: ignore
# Maybe not needed.
del data
yield (
- (curr_volume, curr_target, reduce_list_of_dicts(loss_dict_list), filename)
+ (
+ curr_volume,
+ curr_target,
+ reduce_list_of_dicts(loss_dict_list),
+ filename,
+ )
if add_target
else (
curr_volume,
@@ -994,7 +1011,11 @@ def compute_loss_on_data(
for key, value in loss_dict.items():
if "kspace" in key:
if output_kspace is not None:
- output, target, reconstruction_size = output_kspace, data["kspace"], None
+ output, target, reconstruction_size = (
+ output_kspace,
+ data["kspace"],
+ None,
+ )
else:
continue
else:
@@ -1021,14 +1042,19 @@ def _forward_operator(self, image, sensitivity_map, sampling_mask):
def _backward_operator(self, kspace, sensitivity_map, sampling_mask):
return T.reduce_operator(
- self.backward_operator(T.apply_mask(kspace, sampling_mask, return_mask=False), dim=self._spatial_dims),
+ self.backward_operator(
+ T.apply_mask(kspace, sampling_mask, return_mask=False),
+ dim=self._spatial_dims,
+ ),
sensitivity_map,
dim=self._coil_dim,
)
def _crop_volume(
- source: torch.Tensor, target: torch.Tensor, resolution: Union[List[int], Tuple[int, ...]]
+ source: torch.Tensor,
+ target: torch.Tensor,
+ resolution: Union[List[int], Tuple[int, ...]],
) -> Tuple[torch.Tensor, torch.Tensor]:
"""2D source/target cropper.
@@ -1118,7 +1144,8 @@ def _process_output(
def _compute_resolution(
- key: Optional[str], reconstruction_size: Optional[Union[List[int], Tuple[int]]] = None
+ key: Optional[str],
+ reconstruction_size: Optional[Union[List[int], Tuple[int]]] = None,
) -> Union[List[int], None]:
"""Computes resolution.
diff --git a/direct/nn/mwcnn/mwcnn.py b/direct/nn/mwcnn/mwcnn.py
index 1df89b64..3a737357 100644
--- a/direct/nn/mwcnn/mwcnn.py
+++ b/direct/nn/mwcnn/mwcnn.py
@@ -11,13 +11,22 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-from collections import OrderedDict
-from typing import List, Optional, Tuple
+from __future__ import annotations
+
+from typing import Optional, Tuple, cast
import torch
import torch.nn as nn
import torch.nn.functional as F
+from direct.nn.conv.modulated import (
+ ModConv2dBias,
+ ModConvActivation,
+ ModConvType,
+ ModulationParams,
+ mod_conv2d,
+)
+
class DWT(nn.Module):
"""2D Discrete Wavelet Transform as implemented in [1]_.
@@ -25,7 +34,7 @@ class DWT(nn.Module):
References
----------
- .. [1] Liu, Pengju, et al. “Multi-Level Wavelet-CNN for Image Restoration.” ArXiv:1805.07071 [Cs], May 2018. arXiv.org, http://arxiv.org/abs/1805.07071.
+ .. [1] Liu, Pengju, et al. "Multi-Level Wavelet-CNN for Image Restoration." ArXiv:1805.07071 [Cs], May 2018. arXiv.org, http://arxiv.org/abs/1805.07071.
"""
def __init__(self):
@@ -66,7 +75,7 @@ class IWT(nn.Module):
References
----------
- .. [1] Liu, Pengju, et al. “Multi-Level Wavelet-CNN for Image Restoration.” ArXiv:1805.07071 [Cs], May 2018. arXiv.org, http://arxiv.org/abs/1805.07071.
+ .. [1] Liu, Pengju, et al. "Multi-Level Wavelet-CNN for Image Restoration." ArXiv:1805.07071 [Cs], May 2018. arXiv.org, http://arxiv.org/abs/1805.07071.
"""
def __init__(self):
@@ -89,7 +98,11 @@ def forward(self, x: torch.Tensor) -> torch.Tensor:
IWT of `x`.
"""
batch, in_channel, in_height, in_width = x.size()
- out_channel, out_height, out_width = int(in_channel / (self._r**2)), self._r * in_height, self._r * in_width
+ out_channel, out_height, out_width = (
+ int(in_channel / (self._r**2)),
+ self._r * in_height,
+ self._r * in_width,
+ )
x1 = x[:, 0:out_channel, :, :] / 2
x2 = x[:, out_channel : out_channel * 2, :, :] / 2
@@ -112,7 +125,7 @@ class ConvBlock(nn.Module):
References
----------
- .. [1] Liu, Pengju, et al. “Multi-Level Wavelet-CNN for Image Restoration.” ArXiv:1805.07071 [Cs], May 2018. arXiv.org, http://arxiv.org/abs/1805.07071.
+ .. [1] Liu, Pengju, et al. "Multi-Level Wavelet-CNN for Image Restoration." ArXiv:1805.07071 [Cs], May 2018. arXiv.org, http://arxiv.org/abs/1805.07071.
"""
def __init__(
@@ -124,6 +137,13 @@ def __init__(
batchnorm: bool = False,
activation: nn.Module = nn.ReLU(True),
scale: Optional[float] = 1.0,
+ modulation: ModConvType = ModConvType.NONE,
+ modulation_params: Optional[ModulationParams] = None,
+ aux_in_features: Optional[int] = None,
+ fc_hidden_features: Optional[tuple[int] | int] = None,
+ fc_groups: int = 1,
+ fc_activation: ModConvActivation = ModConvActivation.SIGMOID,
+ num_weights: Optional[int] = None,
):
"""Inits :class:`ConvBlock`.
@@ -143,50 +163,75 @@ def __init__(
Activation function. Default: nn.ReLU(True).
scale: float, optional
Scale. Default: 1.0.
+ modulation : ModConvType
+ Modulation type. Default: ModConvType.NONE.
+ aux_in_features : int, optional
+ Auxiliary input features for modulation.
+ fc_hidden_features : int or tuple of int, optional
+ Hidden features for modulation MLP.
+ fc_groups : int
+ Groups for modulation MLP. Default: 1.
+ fc_activation : ModConvActivation
+ Activation for modulation MLP. Default: ModConvActivation.SIGMOID.
+ num_weights : int, optional
+ Number of weight bases for ModConvType.SUM.
"""
super().__init__()
- net: List[nn.Module] = []
- net.append(
- nn.Conv2d(
- in_channels=in_channels,
- out_channels=out_channels,
- kernel_size=kernel_size,
- bias=bias,
- padding=kernel_size // 2,
+ if modulation_params is None:
+ modulation_params = ModulationParams(
+ modulation=modulation,
+ aux_in_features=aux_in_features,
+ fc_hidden_features=fc_hidden_features,
+ fc_groups=fc_groups,
+ fc_activation=fc_activation,
+ num_weights=num_weights,
)
+ self.modulation = modulation_params.modulation
+ self.conv = mod_conv2d(
+ in_channels=in_channels,
+ out_channels=out_channels,
+ kernel_size=kernel_size,
+ padding=kernel_size // 2,
+ modulation_params=modulation_params,
+ bias=ModConv2dBias.PARAM if bias else ModConv2dBias.NONE,
)
- if batchnorm:
- net.append(nn.BatchNorm2d(num_features=out_channels, eps=1e-4, momentum=0.95))
- net.append(activation)
-
- self.net = nn.Sequential(*net)
+ self.batchnorm = nn.BatchNorm2d(num_features=out_channels, eps=1e-4, momentum=0.95) if batchnorm else None
+ self.activation = activation
self.scale = scale
- def forward(self, x: torch.Tensor) -> torch.Tensor:
+ def forward(self, x: torch.Tensor, y: Optional[torch.Tensor] = None) -> torch.Tensor:
"""Performs forward pass of :class:`ConvBlock`.
Parameters
----------
x: torch.Tensor
Input with shape (N, C, H, W).
+ y: torch.Tensor, optional
+ Auxiliary signal for modulation.
Returns
-------
output: torch.Tensor
Output with shape (N, C', H', W').
"""
- output = self.net(x) * self.scale
+ if self.modulation != ModConvType.NONE:
+ output = self.conv(x, y)
+ else:
+ output = self.conv(x)
+ if self.batchnorm is not None:
+ output = self.batchnorm(output)
+ output = self.activation(output) * self.scale
return output
class DilatedConvBlock(nn.Module):
- """Double dilated Convolution Block fpr :class:`MWCNN` as implemented in [1]_.
+ """Double dilated Convolution Block for :class:`MWCNN` as implemented in [1]_.
References
----------
- .. [1] Liu, Pengju, et al. “Multi-Level Wavelet-CNN for Image Restoration.” ArXiv:1805.07071 [Cs], May 2018. arXiv.org, http://arxiv.org/abs/1805.07071.
+ .. [1] Liu, Pengju, et al. "Multi-Level Wavelet-CNN for Image Restoration." ArXiv:1805.07071 [Cs], May 2018. arXiv.org, http://arxiv.org/abs/1805.07071.
"""
def __init__(
@@ -199,6 +244,13 @@ def __init__(
batchnorm: bool = False,
activation: nn.Module = nn.ReLU(True),
scale: Optional[float] = 1.0,
+ modulation: ModConvType = ModConvType.NONE,
+ modulation_params: Optional[ModulationParams] = None,
+ aux_in_features: Optional[int] = None,
+ fc_hidden_features: Optional[tuple[int] | int] = None,
+ fc_groups: int = 1,
+ fc_activation: ModConvActivation = ModConvActivation.SIGMOID,
+ num_weights: Optional[int] = None,
):
"""Inits :class:`DilatedConvBlock`.
@@ -220,55 +272,90 @@ def __init__(
Activation function. Default: nn.ReLU(True).
scale: float, optional
Scale. Default: 1.0.
+ modulation : ModConvType
+ Modulation type. Default: ModConvType.NONE.
+ aux_in_features : int, optional
+ Auxiliary input features for modulation.
+ fc_hidden_features : int or tuple of int, optional
+ Hidden features for modulation MLP.
+ fc_groups : int
+ Groups for modulation MLP. Default: 1.
+ fc_activation : ModConvActivation
+ Activation for modulation MLP. Default: ModConvActivation.SIGMOID.
+ num_weights : int, optional
+ Number of weight bases for ModConvType.SUM.
"""
super().__init__()
- net: List[nn.Module] = []
- net.append(
- nn.Conv2d(
- in_channels=in_channels,
- out_channels=in_channels,
- kernel_size=kernel_size,
- bias=bias,
- dilation=dilations[0],
- padding=kernel_size // 2 + dilations[0] - 1,
- )
- )
- if batchnorm:
- net.append(nn.BatchNorm2d(num_features=in_channels, eps=1e-4, momentum=0.95))
- net.append(activation)
if out_channels is None:
out_channels = in_channels
- net.append(
- nn.Conv2d(
- in_channels=in_channels,
- out_channels=out_channels,
- kernel_size=kernel_size,
- bias=bias,
- dilation=dilations[1],
- padding=kernel_size // 2 + dilations[1] - 1,
+
+ if modulation_params is None:
+ modulation_params = ModulationParams(
+ modulation=modulation,
+ aux_in_features=aux_in_features,
+ fc_hidden_features=fc_hidden_features,
+ fc_groups=fc_groups,
+ fc_activation=fc_activation,
+ num_weights=num_weights,
)
+ self.modulation = modulation_params.modulation
+ bias_type = ModConv2dBias.PARAM if bias else ModConv2dBias.NONE
+ self.conv1 = mod_conv2d(
+ in_channels=in_channels,
+ out_channels=in_channels,
+ kernel_size=kernel_size,
+ dilation=dilations[0],
+ padding=kernel_size // 2 + dilations[0] - 1,
+ modulation_params=modulation_params,
+ bias=bias_type,
)
- if batchnorm:
- net.append(nn.BatchNorm2d(num_features=in_channels, eps=1e-4, momentum=0.95))
- net.append(activation)
+ self.bn1 = nn.BatchNorm2d(num_features=in_channels, eps=1e-4, momentum=0.95) if batchnorm else None
+ self.act1 = activation
+
+ self.conv2 = mod_conv2d(
+ in_channels=in_channels,
+ out_channels=out_channels,
+ kernel_size=kernel_size,
+ dilation=dilations[1],
+ padding=kernel_size // 2 + dilations[1] - 1,
+ modulation_params=modulation_params,
+ bias=bias_type,
+ )
+ self.bn2 = nn.BatchNorm2d(num_features=out_channels, eps=1e-4, momentum=0.95) if batchnorm else None
+ self.act2 = activation
- self.net = nn.Sequential(*net)
self.scale = scale
- def forward(self, x: torch.Tensor) -> torch.Tensor:
+ def forward(self, x: torch.Tensor, y: Optional[torch.Tensor] = None) -> torch.Tensor:
"""Performs forward pass of :class:`DilatedConvBlock`.
Parameters
----------
x: torch.Tensor
Input with shape (N, C, H, W).
+ y: torch.Tensor, optional
+ Auxiliary signal for modulation.
Returns
-------
output: torch.Tensor
Output with shape (N, C', H', W').
"""
- output = self.net(x) * self.scale
+ if self.modulation != ModConvType.NONE:
+ output = self.conv1(x, y)
+ else:
+ output = self.conv1(x)
+ if self.bn1 is not None:
+ output = self.bn1(output)
+ output = self.act1(output)
+
+ if self.modulation != ModConvType.NONE:
+ output = self.conv2(output, y)
+ else:
+ output = self.conv2(output)
+ if self.bn2 is not None:
+ output = self.bn2(output)
+ output = self.act2(output) * self.scale
return output
@@ -278,7 +365,7 @@ class MWCNN(nn.Module):
References
----------
- .. [1] Liu, Pengju, et al. “Multi-Level Wavelet-CNN for Image Restoration.” ArXiv:1805.07071 [Cs], May 2018. arXiv.org, http://arxiv.org/abs/1805.07071.
+ .. [1] Liu, Pengju, et al. "Multi-Level Wavelet-CNN for Image Restoration." ArXiv:1805.07071 [Cs], May 2018. arXiv.org, http://arxiv.org/abs/1805.07071.
"""
def __init__(
@@ -289,6 +376,13 @@ def __init__(
bias: bool = True,
batchnorm: bool = False,
activation: nn.Module = nn.ReLU(True),
+ modulation: ModConvType = ModConvType.NONE,
+ modulation_params: Optional[ModulationParams] = None,
+ aux_in_features: Optional[int] = None,
+ fc_hidden_features: Optional[tuple[int] | int] = None,
+ fc_groups: int = 1,
+ fc_activation: ModConvActivation = ModConvActivation.SIGMOID,
+ num_weights: Optional[int] = None,
):
"""Inits :class:`MWCNN`.
@@ -306,45 +400,70 @@ def __init__(
If True, a batchnorm layer is added after each convolution. Default: False.
activation: nn.Module
Activation function applied after each convolution. Default: nn.ReLU().
+ modulation : ModConvType
+ Modulation type. Default: ModConvType.NONE.
+ aux_in_features : int, optional
+ Auxiliary input features for modulation.
+ fc_hidden_features : int or tuple of int, optional
+ Hidden features for modulation MLP.
+ fc_groups : int
+ Groups for modulation MLP. Default: 1.
+ fc_activation : ModConvActivation
+ Activation for modulation MLP. Default: ModConvActivation.SIGMOID.
+ num_weights : int, optional
+ Number of weight bases for ModConvType.SUM.
"""
super().__init__()
self._kernel_size = 3
+ if modulation_params is None:
+ modulation_params = ModulationParams(
+ modulation=modulation,
+ aux_in_features=aux_in_features,
+ fc_hidden_features=fc_hidden_features,
+ fc_groups=fc_groups,
+ fc_activation=fc_activation,
+ num_weights=num_weights,
+ )
+ self.modulation = modulation_params.modulation
self.DWT = DWT()
self.IWT = IWT()
+ modulation_params = ModulationParams(
+ modulation=modulation,
+ aux_in_features=aux_in_features,
+ fc_hidden_features=fc_hidden_features,
+ fc_groups=fc_groups,
+ fc_activation=fc_activation,
+ num_weights=num_weights,
+ )
+
self.down = nn.ModuleList()
for idx in range(0, num_scales):
in_channels = input_channels if idx == 0 else first_conv_hidden_channels * 2 ** (idx + 1)
out_channels = first_conv_hidden_channels * 2**idx
dilations = (2, 1) if idx != num_scales - 1 else (2, 3)
self.down.append(
- nn.Sequential(
- OrderedDict(
- [
- (
- f"convblock{idx}",
- ConvBlock(
- in_channels=in_channels,
- out_channels=out_channels,
- kernel_size=self._kernel_size,
- bias=bias,
- batchnorm=batchnorm,
- activation=activation,
- ),
- ),
- (
- f"dilconvblock{idx}",
- DilatedConvBlock(
- in_channels=out_channels,
- dilations=dilations,
- kernel_size=self._kernel_size,
- bias=bias,
- batchnorm=batchnorm,
- activation=activation,
- ),
- ),
- ]
- )
+ nn.ModuleDict(
+ {
+ "convblock": ConvBlock(
+ in_channels=in_channels,
+ out_channels=out_channels,
+ kernel_size=self._kernel_size,
+ bias=bias,
+ batchnorm=batchnorm,
+ activation=activation,
+ modulation_params=modulation_params,
+ ),
+ "dilconvblock": DilatedConvBlock(
+ in_channels=out_channels,
+ dilations=dilations,
+ kernel_size=self._kernel_size,
+ bias=bias,
+ batchnorm=batchnorm,
+ activation=activation,
+ modulation_params=modulation_params,
+ ),
+ }
)
)
self.up = nn.ModuleList()
@@ -353,33 +472,27 @@ def __init__(
out_channels = input_channels if idx == 0 else first_conv_hidden_channels * 2 ** (idx + 1)
dilations = (2, 1) if idx != num_scales - 1 else (3, 2)
self.up.append(
- nn.Sequential(
- OrderedDict(
- [
- (
- f"invdilconvblock{num_scales - 2 - idx}",
- DilatedConvBlock(
- in_channels=in_channels,
- dilations=dilations,
- kernel_size=self._kernel_size,
- bias=bias,
- batchnorm=batchnorm,
- activation=activation,
- ),
- ),
- (
- f"invconvblock{num_scales - 2 - idx}",
- ConvBlock(
- in_channels=in_channels,
- out_channels=out_channels,
- kernel_size=self._kernel_size,
- bias=bias,
- batchnorm=batchnorm,
- activation=activation,
- ),
- ),
- ]
- )
+ nn.ModuleDict(
+ {
+ "dilconvblock": DilatedConvBlock(
+ in_channels=in_channels,
+ dilations=dilations,
+ kernel_size=self._kernel_size,
+ bias=bias,
+ batchnorm=batchnorm,
+ activation=activation,
+ modulation_params=modulation_params,
+ ),
+ "convblock": ConvBlock(
+ in_channels=in_channels,
+ out_channels=out_channels,
+ kernel_size=self._kernel_size,
+ bias=bias,
+ batchnorm=batchnorm,
+ activation=activation,
+ modulation_params=modulation_params,
+ ),
+ }
)
)
self.num_scales = num_scales
@@ -387,11 +500,10 @@ def __init__(
@staticmethod
def pad(x: torch.Tensor) -> torch.Tensor:
padding = [0, 0, 0, 0]
-
if x.shape[-2] % 2 != 0:
- padding[3] = 1 # Padding right - width
+ padding[3] = 1
if x.shape[-1] % 2 != 0:
- padding[1] = 1 # Padding bottom - height
+ padding[1] = 1
if sum(padding) != 0:
x = F.pad(x, padding, "reflect")
return x
@@ -399,20 +511,26 @@ def pad(x: torch.Tensor) -> torch.Tensor:
@staticmethod
def crop_to_shape(x: torch.Tensor, shape: tuple) -> torch.Tensor:
h, w = x.shape[-2:]
-
if h > shape[0]:
x = x[:, :, : shape[0], :]
if w > shape[1]:
x = x[:, :, :, : shape[1]]
return x
- def forward(self, input_tensor: torch.Tensor, res: bool = False) -> torch.Tensor:
+ def forward(
+ self,
+ input_tensor: torch.Tensor,
+ y: Optional[torch.Tensor] = None,
+ res: bool = False,
+ ) -> torch.Tensor:
"""Computes forward pass of :class:`MWCNN`.
Parameters
----------
input_tensor: torch.Tensor
Input tensor.
+ y: torch.Tensor, optional
+ Auxiliary signal for modulation of shape (N, aux_in_features).
res: bool
If True, residual connection is applied to the output. Default: False.
@@ -424,23 +542,34 @@ def forward(self, input_tensor: torch.Tensor, res: bool = False) -> torch.Tensor
res_values = []
x = self.pad(input_tensor.clone())
for idx in range(self.num_scales):
+ down_block = cast(nn.ModuleDict, self.down[idx])
if idx == 0:
- x = self.pad(self.down[idx](x))
+ x = down_block["convblock"](x, y)
+ x = down_block["dilconvblock"](x, y)
+ x = self.pad(x)
res_values.append(x)
elif idx == self.num_scales - 1:
- x = self.down[idx](self.DWT(x))
+ x = down_block["convblock"](self.DWT(x), y)
+ x = down_block["dilconvblock"](x, y)
else:
- x = self.pad(self.down[idx](self.DWT(x)))
+ x = down_block["convblock"](self.DWT(x), y)
+ x = down_block["dilconvblock"](x, y)
+ x = self.pad(x)
res_values.append(x)
for idx in range(self.num_scales):
+ up_block = cast(nn.ModuleDict, self.up[idx])
if idx != self.num_scales - 1:
+ x = up_block["dilconvblock"](x, y)
+ x = up_block["convblock"](x, y)
x = (
- self.crop_to_shape(self.IWT(self.up[idx](x)), res_values[self.num_scales - 2 - idx].shape[-2:])
+ self.crop_to_shape(self.IWT(x), res_values[self.num_scales - 2 - idx].shape[-2:])
+ res_values[self.num_scales - 2 - idx]
)
else:
- x = self.crop_to_shape(self.up[idx](x), input_tensor.shape[-2:])
+ x = up_block["dilconvblock"](x, y)
+ x = up_block["convblock"](x, y)
+ x = self.crop_to_shape(x, input_tensor.shape[-2:])
if res:
x += input_tensor
return x
diff --git a/direct/nn/recurrent/recurrent.py b/direct/nn/recurrent/recurrent.py
index fb33720e..49efdb16 100644
--- a/direct/nn/recurrent/recurrent.py
+++ b/direct/nn/recurrent/recurrent.py
@@ -142,7 +142,13 @@ def forward(
conv_skip: List[torch.Tensor] = []
if previous_state is None:
- batch_size, spatial_size = cell_input.size(0), (cell_input.size(2), cell_input.size(3))
+ batch_size, spatial_size = (
+ cell_input.size(0),
+ (
+ cell_input.size(2),
+ cell_input.size(3),
+ ),
+ )
state_size = [batch_size, self.hidden_channels] + list(spatial_size) + [self.num_layers]
previous_state = torch.zeros(*state_size, dtype=cell_input.dtype).to(cell_input.device)
diff --git a/direct/nn/ssl/mri_models.py b/direct/nn/ssl/mri_models.py
index 35725be0..f4f08d5e 100644
--- a/direct/nn/ssl/mri_models.py
+++ b/direct/nn/ssl/mri_models.py
@@ -117,7 +117,11 @@ def log_first_training_example_and_model(self, data: dict[str, Any]) -> None:
"""
storage = get_event_storage()
- self.logger.info("First case: slice_no: %s, filename: %s.", data["slice_no"][0], data["filename"][0])
+ self.logger.info(
+ "First case: slice_no: %s, filename: %s.",
+ data["slice_no"][0],
+ data["filename"][0],
+ )
if "input_sampling_mask" in data:
first_input_sampling_mask = data["input_sampling_mask"][0][0]
@@ -209,6 +213,7 @@ def _do_iteration(
regularizer_fns = {}
data = dict_to_device(data, self.device)
+ self._attach_auxiliary_data(data)
# Get the k-space and mask which differ during training and inference for SSL
kspace = data["input_kspace"] if self.model.training else data["masked_kspace"]
@@ -387,6 +392,7 @@ def _do_iteration(
regularizer_fns = {}
data = dict_to_device(data, self.device)
+ self._attach_auxiliary_data(data)
# Get a boolean indicating if the sample is for SSL training
# This will expect the input data to contain the keys "input_kspace" and "input_sampling_mask" if SSL training
diff --git a/direct/nn/transformers/uformer.py b/direct/nn/transformers/uformer.py
index 4c7e8211..74fe2766 100644
--- a/direct/nn/transformers/uformer.py
+++ b/direct/nn/transformers/uformer.py
@@ -37,7 +37,12 @@
from direct.nn.transformers.utils import DropoutPath, init_weights, norm, pad_to_square, unnorm, unpad_to_original
from direct.types import DirectEnum
-__all__ = ["AttentionTokenProjectionType", "LeWinTransformerMLPTokenType", "UFormer", "UFormerModel"]
+__all__ = [
+ "AttentionTokenProjectionType",
+ "LeWinTransformerMLPTokenType",
+ "UFormer",
+ "UFormerModel",
+]
class ECALayer1d(nn.Module):
@@ -250,13 +255,28 @@ def __init__(
self.heads = heads
pad = (kernel_size - q_stride) // 2
self.to_q = SepConv2d(
- in_channels=dim, out_channels=inner_dim, kernel_size=kernel_size, stride=q_stride, padding=pad, bias=bias
+ in_channels=dim,
+ out_channels=inner_dim,
+ kernel_size=kernel_size,
+ stride=q_stride,
+ padding=pad,
+ bias=bias,
)
self.to_k = SepConv2d(
- in_channels=dim, out_channels=inner_dim, kernel_size=kernel_size, stride=k_stride, padding=pad, bias=bias
+ in_channels=dim,
+ out_channels=inner_dim,
+ kernel_size=kernel_size,
+ stride=k_stride,
+ padding=pad,
+ bias=bias,
)
self.to_v = SepConv2d(
- in_channels=dim, out_channels=inner_dim, kernel_size=kernel_size, stride=v_stride, padding=pad, bias=bias
+ in_channels=dim,
+ out_channels=inner_dim,
+ kernel_size=kernel_size,
+ stride=v_stride,
+ padding=pad,
+ bias=bias,
)
def forward(
@@ -471,7 +491,10 @@ def __init__(
self.softmax = nn.Softmax(dim=-1)
def forward(
- self, x: torch.Tensor, attn_kv: Optional[torch.Tensor] = None, mask: Optional[torch.Tensor] = None
+ self,
+ x: torch.Tensor,
+ attn_kv: Optional[torch.Tensor] = None,
+ mask: Optional[torch.Tensor] = None,
) -> torch.Tensor:
"""Performs forward pass of :class:`WindowAttentionModule`.
@@ -586,7 +609,10 @@ def __init__(
self.softmax = nn.Softmax(dim=-1)
def forward(
- self, x: torch.Tensor, attn_kv: Optional[torch.Tensor] = None, mask: Optional[torch.Tensor] = None
+ self,
+ x: torch.Tensor,
+ attn_kv: Optional[torch.Tensor] = None,
+ mask: Optional[torch.Tensor] = None,
) -> torch.Tensor:
"""Performs the forward pass of :class:`AttentionModule`.
@@ -713,7 +739,11 @@ class LeFF(nn.Module):
"""
def __init__(
- self, dim: int = 32, hidden_dim: int = 128, act_layer: type[nn.Module] = nn.GELU, use_eca: bool = False
+ self,
+ dim: int = 32,
+ hidden_dim: int = 128,
+ act_layer: type[nn.Module] = nn.GELU,
+ use_eca: bool = False,
) -> None:
"""Inits :class:`LeFF`.
@@ -731,7 +761,15 @@ def __init__(
super().__init__()
self.linear1 = nn.Sequential(nn.Linear(dim, hidden_dim), act_layer())
self.dwconv = nn.Sequential(
- nn.Conv2d(hidden_dim, hidden_dim, groups=hidden_dim, kernel_size=3, stride=1, padding=1), act_layer()
+ nn.Conv2d(
+ hidden_dim,
+ hidden_dim,
+ groups=hidden_dim,
+ kernel_size=3,
+ stride=1,
+ padding=1,
+ ),
+ act_layer(),
)
self.linear2 = nn.Sequential(nn.Linear(hidden_dim, dim))
self.dim = dim
@@ -792,7 +830,11 @@ def window_partition(x: torch.Tensor, win_size: int, dilation_rate: int = 1) ->
if dilation_rate != 1:
x = x.permute(0, 3, 1, 2) # B, C, H, W
x = F.unfold(
- x, kernel_size=win_size, dilation=dilation_rate, padding=4 * (dilation_rate - 1), stride=win_size
+ x,
+ kernel_size=win_size,
+ dilation=dilation_rate,
+ padding=4 * (dilation_rate - 1),
+ stride=win_size,
) # B, C*Wh*Ww, H/Wh*W/Ww
windows = x.permute(0, 2, 1).contiguous().view(-1, C, win_size, win_size) # B' ,C ,Wh ,Ww
windows = windows.permute(0, 2, 3, 1).contiguous() # B' ,Wh ,Ww ,C
@@ -830,7 +872,12 @@ def window_reverse(windows: torch.Tensor, win_size: int, H: int, W: int, dilatio
if dilation_rate != 1:
x = windows.permute(0, 5, 3, 4, 1, 2).contiguous() # B, C*Wh*Ww, H/Wh*W/Ww
x = F.fold(
- x, (H, W), kernel_size=win_size, dilation=dilation_rate, padding=4 * (dilation_rate - 1), stride=win_size
+ x,
+ (H, W),
+ kernel_size=win_size,
+ dilation=dilation_rate,
+ padding=4 * (dilation_rate - 1),
+ stride=win_size,
)
else:
x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, H, W, -1)
@@ -983,7 +1030,13 @@ def __init__(
super().__init__()
kernel_size_int = kernel_size if isinstance(kernel_size, int) else kernel_size[0]
self.proj = nn.Sequential(
- nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=stride, padding=kernel_size_int // 2),
+ nn.Conv2d(
+ in_channels,
+ out_channels,
+ kernel_size=3,
+ stride=stride,
+ padding=kernel_size_int // 2,
+ ),
act_layer(inplace=True),
)
if norm_layer is not None:
@@ -1058,7 +1111,13 @@ def __init__(
super().__init__()
kernel_size_int = kernel_size if isinstance(kernel_size, int) else kernel_size[0]
self.proj = nn.Sequential(
- nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=stride, padding=kernel_size_int // 2),
+ nn.Conv2d(
+ in_channels,
+ out_channels,
+ kernel_size=3,
+ stride=stride,
+ padding=kernel_size_int // 2,
+ ),
)
if act_layer is not None:
self.proj.add_module("activation", act_layer(inplace=True))
@@ -1117,7 +1176,7 @@ class LeWinTransformerBlock(nn.Module):
Whether to use bias in the query, key, and value projections of the attention mechanism. Default: True.
qk_scale : float, optional
Scale factor for the query and key projection vectors.
- If set to None, will use the default value of :math`1 / \sqrt(dim)`. Default: None.
+ If set to None, will use the default value of :math:`1 / \\sqrt(dim)`. Default: None.
drop : float
Dropout rate for the token-level dropout layer. Default: 0.0.
attn_drop : float
@@ -1180,7 +1239,7 @@ def __init__(
Whether to use bias in the query, key, and value projections of the attention mechanism. Default: True.
qk_scale : float, optional
Scale factor for the query and key projection vectors.
- If set to None, will use the default value of :math`1 / \sqrt(dim)`. Default: None.
+ If set to None, will use the default value of :math:`1 / \\sqrt(dim)`. Default: None.
drop : float
Dropout rate for the token-level dropout layer. Default: 0.0.
attn_drop : float
@@ -1251,7 +1310,12 @@ def __init__(
self.norm2 = norm_layer(dim)
mlp_hidden_dim = int(dim * mlp_ratio)
if token_mlp == LeWinTransformerMLPTokenType.MLP:
- self.mlp = MLP(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop)
+ self.mlp = MLP(
+ in_features=dim,
+ hidden_features=mlp_hidden_dim,
+ act_layer=act_layer,
+ drop=drop,
+ )
else:
self.mlp = LeFF(dim, mlp_hidden_dim, act_layer=act_layer)
@@ -1396,7 +1460,7 @@ class BasicUFormerLayer(nn.Module):
Whether to use bias in the query, key, and value projections of the attention mechanism. Default: True.
qk_scale : float, optional
Scale factor for the query and key projection vectors.
- If set to None, will use the default value of :math`1 / \sqrt(dim)`. Default: None.
+ If set to None, will use the default value of :math:`1 / \\sqrt(dim)`. Default: None.
drop : float
Dropout rate for the token-level dropout layer. Default: 0.0.
attn_drop : float
@@ -1457,7 +1521,7 @@ def __init__(
Whether to use bias in the query, key, and value projections of the attention mechanism. Default: True.
qk_scale : float, optional
Scale factor for the query and key projection vectors.
- If set to None, will use the default value of :math`1 / \sqrt(dim)`. Default: None.
+ If set to None, will use the default value of :math:`1 / \\sqrt(dim)`. Default: None.
drop : float
Dropout rate for the token-level dropout layer. Default: 0.0.
attn_drop : float
@@ -1493,13 +1557,13 @@ def __init__(
input_resolution=input_resolution,
num_heads=num_heads,
win_size=win_size,
- shift_size=(0 if (i % 2 == 0) else win_size // 2) if shift_flag else 0,
+ shift_size=((0 if (i % 2 == 0) else win_size // 2) if shift_flag else 0),
mlp_ratio=mlp_ratio,
qkv_bias=qkv_bias,
qk_scale=qk_scale,
drop=drop,
attn_drop=attn_drop,
- drop_path=drop_path[i] if isinstance(drop_path, list) else drop_path,
+ drop_path=(drop_path[i] if isinstance(drop_path, list) else drop_path),
norm_layer=norm_layer,
token_projection=token_projection,
token_mlp=token_mlp,
@@ -1682,7 +1746,12 @@ def __init__(
self.num_enc_layers = len(encoder_num_heads)
self.num_dec_layers = len(encoder_num_heads)
depths = (*encoder_depths, bottleneck_depth, *encoder_depths[::-1])
- num_heads = (*encoder_num_heads, bottleneck_num_heads, bottleneck_num_heads, *encoder_num_heads[::-1][:-1])
+ num_heads = (
+ *encoder_num_heads,
+ bottleneck_num_heads,
+ bottleneck_num_heads,
+ *encoder_num_heads[::-1][:-1],
+ )
self.embedding_dim = embedding_dim
self.patch_norm = patch_norm
self.mlp_ratio = mlp_ratio
@@ -1701,15 +1770,27 @@ def __init__(
# Input
self.input_proj = InputProjection(
- in_channels=in_channels, out_channels=embedding_dim, kernel_size=3, stride=1, act_layer=nn.LeakyReLU
+ in_channels=in_channels,
+ out_channels=embedding_dim,
+ kernel_size=3,
+ stride=1,
+ act_layer=nn.LeakyReLU,
)
out_channels = out_channels if out_channels else in_channels
# Output
self.output_proj = OutputProjection(
- in_channels=2 * embedding_dim, out_channels=out_channels, kernel_size=3, stride=1
+ in_channels=2 * embedding_dim,
+ out_channels=out_channels,
+ kernel_size=3,
+ stride=1,
)
if in_channels != out_channels:
- self.conv_out = nn.Conv2d(in_channels=in_channels, out_channels=out_channels, kernel_size=1, padding=0)
+ self.conv_out = nn.Conv2d(
+ in_channels=in_channels,
+ out_channels=out_channels,
+ kernel_size=1,
+ padding=0,
+ )
self.in_channels = in_channels
self.out_channels = out_channels
@@ -1747,7 +1828,10 @@ def __init__(
# Bottleneck
self.bottleneck = BasicUFormerLayer(
dim=embedding_dim * (2**self.num_enc_layers),
- input_resolution=(patch_size // (2**self.num_enc_layers), patch_size // (2**self.num_enc_layers)),
+ input_resolution=(
+ patch_size // (2**self.num_enc_layers),
+ patch_size // (2**self.num_enc_layers),
+ ),
depth=depths[self.num_enc_layers],
num_heads=num_heads[self.num_enc_layers],
win_size=win_size,
@@ -1776,7 +1860,10 @@ def __init__(
self.upsamples.add_module(upsample_layer_name, upsample_layer)
layer_name = f"decoderlayer_{self.num_dec_layers - i}"
- layer_input_resolution = (patch_size // (2 ** (i - 1)), patch_size // (2 ** (i - 1)))
+ layer_input_resolution = (
+ patch_size // (2 ** (i - 1)),
+ patch_size // (2 ** (i - 1)),
+ )
layer_dim = embedding_dim * (2**i)
layer_num = self.num_enc_layers + self.num_dec_layers - i + 1
layer_depth = depths[layer_num]
diff --git a/direct/nn/transformers/utils.py b/direct/nn/transformers/utils.py
index 173302ce..227c0b5e 100644
--- a/direct/nn/transformers/utils.py
+++ b/direct/nn/transformers/utils.py
@@ -23,7 +23,15 @@
from torch import nn
from torch.nn import init
-__all__ = ["init_weights", "norm", "pad_to_divisible", "pad_to_square", "unnorm", "unpad_to_original", "DropoutPath"]
+__all__ = [
+ "init_weights",
+ "norm",
+ "pad_to_divisible",
+ "pad_to_square",
+ "unnorm",
+ "unpad_to_original",
+ "DropoutPath",
+]
def pad_to_divisible(x: torch.Tensor, pad_size: tuple[int, ...]) -> tuple[torch.Tensor, tuple[tuple[int, int], ...]]:
@@ -85,7 +93,7 @@ def pad_to_square(
Parameters
----------
inp : torch.Tensor
- The input tensor to pad to square shape. Expected shape is (\*, height, width).
+ The input tensor to pad to square shape. Expected shape is (\\*, height, width).
factor : float
The factor to which the input tensor will be padded.
diff --git a/direct/nn/unet/config.py b/direct/nn/unet/config.py
index d9bc8f23..59f98552 100644
--- a/direct/nn/unet/config.py
+++ b/direct/nn/unet/config.py
@@ -12,8 +12,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from dataclasses import dataclass
+from typing import Optional
from direct.config.defaults import ModelConfig
+from direct.nn.conv.modulated import ModConvActivation, ModConvType
from direct.nn.types import InitType
@@ -24,6 +26,12 @@ class UnetModel2dConfig(ModelConfig):
num_filters: int = 16
num_pool_layers: int = 4
dropout_probability: float = 0.0
+ modulation: ModConvType = ModConvType.NONE
+ aux_in_features: Optional[int] = None
+ fc_hidden_features: Optional[int] = None
+ fc_groups: int = 1
+ fc_activation: ModConvActivation = ModConvActivation.SIGMOID
+ num_weights: Optional[int] = None
class NormUnetModel2dConfig(ModelConfig):
@@ -33,6 +41,12 @@ class NormUnetModel2dConfig(ModelConfig):
num_pool_layers: int = 4
dropout_probability: float = 0.0
norm_groups: int = 2
+ modulation: ModConvType = ModConvType.NONE
+ aux_in_features: Optional[int] = None
+ fc_hidden_features: Optional[int] = None
+ fc_groups: int = 1
+ fc_activation: ModConvActivation = ModConvActivation.SIGMOID
+ num_weights: Optional[int] = None
@dataclass
@@ -43,6 +57,14 @@ class Unet2dConfig(ModelConfig):
skip_connection: bool = False
normalized: bool = False
image_initialization: InitType = InitType.ZERO_FILLED
+ conv_modulation: ModConvType = ModConvType.NONE
+ aux_in_features: Optional[int] = None
+ auxiliary_features: Optional[tuple[str, ...]] = None
+ log_aux: bool = False
+ fc_hidden_features: Optional[tuple[int, ...]] = None
+ fc_groups: int = 1
+ fc_activation: ModConvActivation = ModConvActivation.SIGMOID
+ num_weights: Optional[int] = None
@dataclass
@@ -52,3 +74,12 @@ class UnetModel3dConfig(ModelConfig):
num_filters: int = 16
num_pool_layers: int = 4
dropout_probability: float = 0.0
+
+
+class NormUnetModel3dConfig(ModelConfig):
+ in_channels: int = 2
+ out_channels: int = 2
+ num_filters: int = 16
+ num_pool_layers: int = 4
+ dropout_probability: float = 0.0
+ norm_groups: int = 2
diff --git a/direct/nn/unet/unet_2d.py b/direct/nn/unet/unet_2d.py
index b83acda4..162b487c 100644
--- a/direct/nn/unet/unet_2d.py
+++ b/direct/nn/unet/unet_2d.py
@@ -2,6 +2,9 @@
# Copyright (c) DIRECT Contributors
# Code borrowed / edited from: https://github.com/facebookresearch/fastMRI/blob/
+
+from __future__ import annotations
+
import math
from typing import List, Optional, Tuple
@@ -10,18 +13,116 @@
from torch.nn import functional as F
from direct.data import transforms as T
+from direct.nn.adain.adain import AdaIN2d, NormType
+from direct.nn.conv.modulated import (
+ ModConv2dBias,
+ ModConvActivation,
+ ModConvType,
+ ModulationParams,
+ mod_conv2d,
+ mod_conv_transpose2d,
+)
from direct.nn.types import InitType
from direct.types import FFTOperator
+class ConvModule(nn.Module):
+ """Single convolution + norm + activation + dropout module supporting modulated convolutions."""
+
+ def __init__(
+ self,
+ in_channels: int,
+ out_channels: int,
+ kernel_size: int,
+ padding: int,
+ dropout_probability: float,
+ modulation: ModConvType = ModConvType.NONE,
+ modulation_params: Optional[ModulationParams] = None,
+ bias: ModConv2dBias = ModConv2dBias.PARAM,
+ aux_in_features: Optional[int] = None,
+ fc_hidden_features: Optional[tuple[int] | int] = None,
+ fc_groups: int = 1,
+ fc_activation: ModConvActivation = ModConvActivation.SIGMOID,
+ num_weights: Optional[int] = None,
+ norm_type: NormType = NormType.INSTANCE,
+ adain_hidden_features: Optional[tuple[int] | int] = None,
+ ):
+ super().__init__()
+
+ if modulation_params is None:
+ modulation_params = ModulationParams(
+ modulation=modulation,
+ aux_in_features=aux_in_features,
+ fc_hidden_features=fc_hidden_features,
+ fc_groups=fc_groups,
+ fc_activation=fc_activation,
+ num_weights=num_weights,
+ )
+ self.modulation = modulation_params.modulation
+
+ self.conv = mod_conv2d(
+ in_channels=in_channels,
+ out_channels=out_channels,
+ kernel_size=kernel_size,
+ padding=padding,
+ bias=bias,
+ modulation_params=modulation_params,
+ )
+ self.norm_type = norm_type
+ if norm_type == NormType.ADAIN:
+ if adain_hidden_features is None:
+ raise ValueError("AdaIN hidden features must be provided if norm_type is NormType.ADAIN.")
+ if modulation_params.aux_in_features is None:
+ raise ValueError("aux_in_features must be provided if norm_type is NormType.ADAIN.")
+ self.instance_norm = AdaIN2d(
+ num_channels=out_channels,
+ aux_in_features=modulation_params.aux_in_features,
+ hidden_features=adain_hidden_features,
+ )
+ else:
+ self.instance_norm = nn.InstanceNorm2d(out_channels)
+ self.leaky_relu = nn.LeakyReLU(negative_slope=0.2, inplace=True)
+ self.dropout = nn.Dropout2d(dropout_probability)
+
+ def forward(self, x: torch.Tensor, y: Optional[torch.Tensor] = None) -> torch.Tensor:
+ if self.modulation != ModConvType.NONE:
+ x = self.conv(x, y)
+ else:
+ x = self.conv(x)
+ if self.norm_type == NormType.ADAIN:
+ if y is None:
+ raise ValueError("AdaIN requires aux vector y, but got None.")
+ x = self.instance_norm(x, y)
+ else:
+ x = self.instance_norm(x)
+ x = self.leaky_relu(x)
+ x = self.dropout(x)
+ return x
+
+
class ConvBlock(nn.Module):
"""U-Net convolutional block.
It consists of two convolution layers each followed by instance normalization, LeakyReLU activation and dropout.
+ Supports modulated convolutions and AdaIN normalization.
"""
- def __init__(self, in_channels: int, out_channels: int, dropout_probability: float):
- """Inits ConvBlock.
+ def __init__(
+ self,
+ in_channels: int,
+ out_channels: int,
+ dropout_probability: float,
+ modulation: ModConvType = ModConvType.NONE,
+ modulation_params: Optional[ModulationParams] = None,
+ aux_in_features: Optional[int] = None,
+ fc_hidden_features: Optional[tuple[int] | int] = None,
+ fc_groups: int = 1,
+ fc_activation: ModConvActivation = ModConvActivation.SIGMOID,
+ num_weights: Optional[int] = None,
+ norm_type: NormType = NormType.INSTANCE,
+ adain_hidden_features: Optional[tuple[int] | int] = None,
+ ):
+ """Inits :class:`ConvBlock`.
Parameters
----------
@@ -31,6 +132,22 @@ def __init__(self, in_channels: int, out_channels: int, dropout_probability: flo
Number of output channels.
dropout_probability: float
Dropout probability.
+ modulation : ModConvType
+ Modulation type. Default: ModConvType.NONE.
+ aux_in_features : int, optional
+ Number of auxiliary input features for modulation/AdaIN.
+ fc_hidden_features : int or tuple of int, optional
+ Hidden features for the modulation MLP.
+ fc_groups : int
+ Groups for the modulation MLP. Default: 1.
+ fc_activation : ModConvActivation
+ Activation for the modulation MLP. Default: ModConvActivation.SIGMOID.
+ num_weights : int, optional
+ Number of weight bases for ModConvType.SUM.
+ norm_type : NormType
+ Normalization type. Default: NormType.INSTANCE.
+ adain_hidden_features : int or tuple of int, optional
+ Hidden features for AdaIN. Required if norm_type is NormType.ADAIN.
"""
super().__init__()
@@ -38,45 +155,84 @@ def __init__(self, in_channels: int, out_channels: int, dropout_probability: flo
self.out_channels = out_channels
self.dropout_probability = dropout_probability
- self.layers = nn.Sequential(
- nn.Conv2d(in_channels, out_channels, kernel_size=3, padding=1, bias=False),
- nn.InstanceNorm2d(out_channels),
- nn.LeakyReLU(negative_slope=0.2, inplace=True),
- nn.Dropout2d(dropout_probability),
- nn.Conv2d(out_channels, out_channels, kernel_size=3, padding=1, bias=False),
- nn.InstanceNorm2d(out_channels),
- nn.LeakyReLU(negative_slope=0.2, inplace=True),
- nn.Dropout2d(dropout_probability),
+ if modulation_params is None:
+ modulation_params = ModulationParams(
+ modulation=modulation,
+ aux_in_features=aux_in_features,
+ fc_hidden_features=fc_hidden_features,
+ fc_groups=fc_groups,
+ fc_activation=fc_activation,
+ num_weights=num_weights,
+ )
+ self.modulation = modulation_params.modulation
+ self.norm_type = norm_type
+
+ self.layer_1 = ConvModule(
+ in_channels,
+ out_channels,
+ kernel_size=3,
+ padding=1,
+ bias=(ModConv2dBias.NONE if modulation_params.modulation == ModConvType.NONE else ModConv2dBias.LEARNED),
+ dropout_probability=dropout_probability,
+ modulation_params=modulation_params,
+ norm_type=norm_type,
+ adain_hidden_features=adain_hidden_features,
+ )
+ self.layer_2 = ConvModule(
+ out_channels,
+ out_channels,
+ kernel_size=3,
+ padding=1,
+ bias=(ModConv2dBias.NONE if modulation_params.modulation == ModConvType.NONE else ModConv2dBias.LEARNED),
+ dropout_probability=dropout_probability,
+ modulation_params=modulation_params,
+ norm_type=norm_type,
+ adain_hidden_features=adain_hidden_features,
)
- def forward(self, input_data: torch.Tensor) -> torch.Tensor:
+ def forward(self, input_data: torch.Tensor, aux_data: Optional[torch.Tensor] = None) -> torch.Tensor:
"""Performs the forward pass of :class:`ConvBlock`.
Parameters
----------
- input_data: torch.Tensor
+ input_data : torch.Tensor
+ aux_data : torch.Tensor, optional
Returns
-------
torch.Tensor
"""
- return self.layers(input_data)
+ if self.modulation != ModConvType.NONE or self.norm_type == NormType.ADAIN:
+ return self.layer_2(self.layer_1(input_data, aux_data), aux_data)
+ return self.layer_2(self.layer_1(input_data))
def __repr__(self):
- """Representation of :class:`ConvBlock`."""
return (
f"ConvBlock(in_channels={self.in_channels}, out_channels={self.out_channels}, "
- f"dropout_probability={self.dropout_probability})"
+ f"dropout_probability={self.dropout_probability}, modulation={self.modulation})"
)
class TransposeConvBlock(nn.Module):
- """U-Net Transpose Convolutional Block.
+ """U-Net Transpose Convolutional Block with optional modulation.
- It consists of one convolution transpose layers followed by instance normalization and LeakyReLU activation.
+ It consists of one convolution transpose layer followed by instance normalization and LeakyReLU activation.
"""
- def __init__(self, in_channels: int, out_channels: int):
+ def __init__(
+ self,
+ in_channels: int,
+ out_channels: int,
+ modulation: ModConvType = ModConvType.NONE,
+ modulation_params: Optional[ModulationParams] = None,
+ aux_in_features: Optional[int] = None,
+ fc_hidden_features: Optional[tuple[int] | int] = None,
+ fc_groups: int = 1,
+ fc_activation: ModConvActivation = ModConvActivation.SIGMOID,
+ num_weights: Optional[int] = None,
+ norm_type: NormType = NormType.INSTANCE,
+ adain_hidden_features: Optional[tuple[int] | int] = None,
+ ):
"""Inits :class:`TransposeConvBlock`.
Parameters
@@ -85,43 +241,100 @@ def __init__(self, in_channels: int, out_channels: int):
Number of input channels.
out_channels: int
Number of output channels.
+ modulation : ModConvType
+ Modulation type. Default: ModConvType.NONE.
+ aux_in_features : int, optional
+ Number of auxiliary input features.
+ fc_hidden_features : int or tuple of int, optional
+ Hidden features for the modulation MLP.
+ fc_groups : int
+ Groups for the modulation MLP. Default: 1.
+ fc_activation : ModConvActivation
+ Activation for the modulation MLP. Default: ModConvActivation.SIGMOID.
+ num_weights : int, optional
+ Number of weight bases for ModConvType.SUM.
+ norm_type : NormType
+ Normalization type. Default: NormType.INSTANCE.
+ adain_hidden_features : int or tuple of int, optional
+ Hidden features for AdaIN.
"""
super().__init__()
self.in_channels = in_channels
self.out_channels = out_channels
+ if modulation_params is None:
+ modulation_params = ModulationParams(
+ modulation=modulation,
+ aux_in_features=aux_in_features,
+ fc_hidden_features=fc_hidden_features,
+ fc_groups=fc_groups,
+ fc_activation=fc_activation,
+ num_weights=num_weights,
+ )
+ self.modulation = modulation_params.modulation
+ self.norm_type = norm_type
- self.layers = nn.Sequential(
- nn.ConvTranspose2d(in_channels, out_channels, kernel_size=2, stride=2, bias=False),
- nn.InstanceNorm2d(out_channels),
- nn.LeakyReLU(negative_slope=0.2, inplace=True),
+ self.conv = mod_conv_transpose2d(
+ in_channels=in_channels,
+ out_channels=out_channels,
+ kernel_size=2,
+ stride=2,
+ bias=(ModConv2dBias.NONE if modulation_params.modulation == ModConvType.NONE else ModConv2dBias.LEARNED),
+ modulation_params=modulation_params,
)
+ if norm_type == NormType.ADAIN:
+ if adain_hidden_features is None:
+ raise ValueError("AdaIN hidden features must be provided if norm_type is NormType.ADAIN.")
+ if modulation_params.aux_in_features is None:
+ raise ValueError("aux_in_features must be provided if norm_type is NormType.ADAIN.")
+ self.instance_norm = AdaIN2d(
+ num_channels=out_channels,
+ aux_in_features=modulation_params.aux_in_features,
+ hidden_features=adain_hidden_features,
+ )
+ else:
+ self.instance_norm = nn.InstanceNorm2d(out_channels)
+ self.leaky_relu = nn.LeakyReLU(negative_slope=0.2, inplace=True)
- def forward(self, input_data: torch.Tensor) -> torch.Tensor:
+ def forward(self, input_data: torch.Tensor, aux_data: Optional[torch.Tensor] = None) -> torch.Tensor:
"""Performs forward pass of :class:`TransposeConvBlock`.
Parameters
----------
input_data: torch.Tensor
+ aux_data: torch.Tensor, optional
Returns
-------
torch.Tensor
"""
- return self.layers(input_data)
+ if self.modulation != ModConvType.NONE:
+ x = self.conv(input_data, aux_data)
+ else:
+ x = self.conv(input_data)
+ if self.norm_type == NormType.ADAIN:
+ x = self.instance_norm(x, aux_data)
+ else:
+ x = self.instance_norm(x)
+ return self.leaky_relu(x)
def __repr__(self):
- """Representation of "class:`TransposeConvBlock`."""
- return f"ConvBlock(in_channels={self.in_channels}, out_channels={self.out_channels})"
+ return f"TransposeConvBlock(in_channels={self.in_channels}, out_channels={self.out_channels})"
class UnetModel2d(nn.Module):
"""PyTorch implementation of a U-Net model based on [1]_.
+ Supports optional modulated convolutions and AdaIN normalization conditioned
+ on an auxiliary input signal as proposed in [2]_.
+
References
----------
+ .. [1] Ronneberger, Olaf, et al. "U-Net: Convolutional Networks for Biomedical Image Segmentation." MICCAI 2015.
- .. [1] Ronneberger, Olaf, et al. “U-Net: Convolutional Networks for Biomedical Image Segmentation.” Medical Image Computing and Computer-Assisted Intervention – MICCAI 2015, edited by Nassir Navab et al., Springer International Publishing, 2015, pp. 234–41. Springer Link, https://doi.org/10.1007/978-3-319-24574-4_28.
+ .. [2] Moriakov, N., Yiasemis, G., Sonke, J.-J. & Teuwen, J. (2026). Conditional Learned Reconstruction for
+ Medical Imaging. Proceedings of The 9th International Conference on Medical Imaging with Deep Learning,
+ PMLR 315:754-780. https://proceedings.mlr.press/v315/moriakov26a.html
"""
def __init__(
@@ -131,6 +344,16 @@ def __init__(
num_filters: int,
num_pool_layers: int,
dropout_probability: float,
+ modulation: ModConvType = ModConvType.NONE,
+ modulation_params: Optional[ModulationParams] = None,
+ aux_in_features: Optional[int] = None,
+ fc_hidden_features: Optional[tuple[int] | int] = None,
+ fc_groups: int = 1,
+ fc_activation: ModConvActivation = ModConvActivation.SIGMOID,
+ num_weights: Optional[int] = None,
+ modulation_at_input: bool = False,
+ norm_type: NormType = NormType.INSTANCE,
+ adain_hidden_features: Optional[tuple[int] | int] = None,
):
"""Inits :class:`UnetModel2d`.
@@ -146,6 +369,24 @@ def __init__(
Number of down-sampling and up-sampling layers (depth).
dropout_probability: float
Dropout probability.
+ modulation : ModConvType
+ Modulation type. Default: ModConvType.NONE.
+ aux_in_features : int, optional
+ Number of auxiliary input features.
+ fc_hidden_features : int or tuple of int, optional
+ Hidden features for the modulation MLP.
+ fc_groups : int
+ Groups for the modulation MLP. Default: 1.
+ fc_activation : ModConvActivation
+ Activation for the modulation MLP. Default: ModConvActivation.SIGMOID.
+ num_weights : int, optional
+ Number of weight bases for ModConvType.SUM.
+ modulation_at_input : bool
+ If True, only the first conv block uses modulation. Default: False.
+ norm_type : NormType
+ Normalization type. Default: NormType.INSTANCE.
+ adain_hidden_features : int or tuple of int, optional
+ Hidden features for AdaIN.
"""
super().__init__()
@@ -154,35 +395,127 @@ def __init__(
self.num_filters = num_filters
self.num_pool_layers = num_pool_layers
self.dropout_probability = dropout_probability
-
- self.down_sample_layers = nn.ModuleList([ConvBlock(in_channels, num_filters, dropout_probability)])
+ if modulation_params is None:
+ modulation_params = ModulationParams(
+ modulation=modulation,
+ aux_in_features=aux_in_features,
+ fc_hidden_features=fc_hidden_features,
+ fc_groups=fc_groups,
+ fc_activation=fc_activation,
+ num_weights=num_weights,
+ )
+ self.modulation = modulation_params.modulation
+ self.norm_type = norm_type
+
+ self.down_sample_layers = nn.ModuleList(
+ [
+ ConvBlock(
+ in_channels,
+ num_filters,
+ dropout_probability,
+ modulation_params=modulation_params,
+ norm_type=norm_type,
+ adain_hidden_features=adain_hidden_features,
+ )
+ ]
+ )
ch = num_filters
+
+ block_modulation_params = modulation_params
+ if modulation_params.modulation != ModConvType.NONE and modulation_at_input:
+ block_modulation_params = ModulationParams(
+ modulation=ModConvType.NONE,
+ aux_in_features=modulation_params.aux_in_features,
+ fc_hidden_features=modulation_params.fc_hidden_features,
+ fc_groups=modulation_params.fc_groups,
+ fc_activation=modulation_params.fc_activation,
+ num_weights=modulation_params.num_weights,
+ fc_bias=modulation_params.fc_bias,
+ )
+
for _ in range(num_pool_layers - 1):
- self.down_sample_layers += [ConvBlock(ch, ch * 2, dropout_probability)]
+ self.down_sample_layers += [
+ ConvBlock(
+ ch,
+ ch * 2,
+ dropout_probability,
+ modulation_params=block_modulation_params,
+ norm_type=norm_type,
+ adain_hidden_features=adain_hidden_features,
+ )
+ ]
ch *= 2
- self.conv = ConvBlock(ch, ch * 2, dropout_probability)
+ self.conv = ConvBlock(
+ ch,
+ ch * 2,
+ dropout_probability,
+ modulation_params=block_modulation_params,
+ norm_type=norm_type,
+ adain_hidden_features=adain_hidden_features,
+ )
self.up_conv = nn.ModuleList()
self.up_transpose_conv = nn.ModuleList()
for _ in range(num_pool_layers - 1):
- self.up_transpose_conv += [TransposeConvBlock(ch * 2, ch)]
- self.up_conv += [ConvBlock(ch * 2, ch, dropout_probability)]
+ self.up_transpose_conv += [
+ TransposeConvBlock(
+ ch * 2,
+ ch,
+ modulation_params=block_modulation_params,
+ norm_type=norm_type,
+ adain_hidden_features=adain_hidden_features,
+ )
+ ]
+ self.up_conv += [
+ ConvBlock(
+ ch * 2,
+ ch,
+ dropout_probability,
+ modulation_params=block_modulation_params,
+ norm_type=norm_type,
+ adain_hidden_features=adain_hidden_features,
+ )
+ ]
ch //= 2
- self.up_transpose_conv += [TransposeConvBlock(ch * 2, ch)]
+ self.up_transpose_conv += [
+ TransposeConvBlock(
+ ch * 2,
+ ch,
+ modulation_params=block_modulation_params,
+ norm_type=norm_type,
+ adain_hidden_features=adain_hidden_features,
+ )
+ ]
self.up_conv += [
- nn.Sequential(
- ConvBlock(ch * 2, ch, dropout_probability),
- nn.Conv2d(ch, self.out_channels, kernel_size=1, stride=1),
+ ConvBlock(
+ ch * 2,
+ ch,
+ dropout_probability,
+ modulation_params=block_modulation_params,
+ norm_type=norm_type,
+ adain_hidden_features=adain_hidden_features,
)
]
+ self.conv_out = mod_conv2d(
+ ch,
+ self.out_channels,
+ kernel_size=1,
+ stride=1,
+ bias=(
+ ModConv2dBias.NONE if block_modulation_params.modulation == ModConvType.NONE else ModConv2dBias.LEARNED
+ ),
+ modulation_params=block_modulation_params,
+ )
- def forward(self, input_data: torch.Tensor) -> torch.Tensor:
+ def forward(self, input_data: torch.Tensor, aux_data: Optional[torch.Tensor] = None) -> torch.Tensor:
"""Performs forward pass of :class:`UnetModel2d`.
Parameters
----------
input_data: torch.Tensor
+ aux_data: torch.Tensor, optional
+ Auxiliary data for modulation/AdaIN.
Returns
-------
@@ -191,36 +524,50 @@ def forward(self, input_data: torch.Tensor) -> torch.Tensor:
stack = []
output = input_data
- # Apply down-sampling layers
- for _, layer in enumerate(self.down_sample_layers):
- output = layer(output)
+ for layer in self.down_sample_layers:
+ if self.modulation != ModConvType.NONE or self.norm_type == NormType.ADAIN:
+ output = layer(output, aux_data)
+ else:
+ output = layer(output)
stack.append(output)
output = F.avg_pool2d(output, kernel_size=2, stride=2, padding=0)
- output = self.conv(output)
+ if self.modulation != ModConvType.NONE or self.norm_type == NormType.ADAIN:
+ output = self.conv(output, aux_data)
+ else:
+ output = self.conv(output)
- # Apply up-sampling layers
for transpose_conv, conv in zip(self.up_transpose_conv, self.up_conv):
downsample_layer = stack.pop()
- output = transpose_conv(output)
+ if self.modulation != ModConvType.NONE or self.norm_type == NormType.ADAIN:
+ output = transpose_conv(output, aux_data)
+ else:
+ output = transpose_conv(output)
- # Reflect pad on the right/bottom if needed to handle odd input dimensions.
padding = [0, 0, 0, 0]
if output.shape[-1] != downsample_layer.shape[-1]:
- padding[1] = 1 # Padding right
+ padding[1] = 1
if output.shape[-2] != downsample_layer.shape[-2]:
- padding[3] = 1 # Padding bottom
+ padding[3] = 1
if sum(padding) != 0:
output = F.pad(output, padding, "reflect")
output = torch.cat([output, downsample_layer], dim=1)
- output = conv(output)
+ if self.modulation != ModConvType.NONE or self.norm_type == NormType.ADAIN:
+ output = conv(output, aux_data)
+ else:
+ output = conv(output)
+
+ if self.modulation != ModConvType.NONE or self.norm_type == NormType.ADAIN:
+ output = self.conv_out(output, aux_data)
+ else:
+ output = self.conv_out(output)
return output
class NormUnetModel2d(nn.Module):
- """Implementation of a Normalized U-Net model."""
+ """Implementation of a Normalized U-Net model with optional modulation support."""
def __init__(
self,
@@ -230,6 +577,16 @@ def __init__(
num_pool_layers: int,
dropout_probability: float,
norm_groups: int = 2,
+ modulation: ModConvType = ModConvType.NONE,
+ modulation_params: Optional[ModulationParams] = None,
+ aux_in_features: Optional[int] = None,
+ fc_hidden_features: Optional[tuple[int] | int] = None,
+ fc_groups: int = 1,
+ fc_activation: ModConvActivation = ModConvActivation.SIGMOID,
+ num_weights: Optional[int] = None,
+ modulation_at_input: bool = False,
+ norm_type: NormType = NormType.INSTANCE,
+ adain_hidden_features: Optional[tuple[int] | int] = None,
):
"""Inits :class:`NormUnetModel2d`.
@@ -245,8 +602,26 @@ def __init__(
Number of down-sampling and up-sampling layers (depth).
dropout_probability: float
Dropout probability.
- norm_groups: int,
+ norm_groups: int
Number of normalization groups.
+ modulation : ModConvType
+ Modulation type. Default: ModConvType.NONE.
+ aux_in_features : int, optional
+ Number of auxiliary input features.
+ fc_hidden_features : int or tuple of int, optional
+ Hidden features for the modulation MLP.
+ fc_groups : int
+ Groups for the modulation MLP. Default: 1.
+ fc_activation : ModConvActivation
+ Activation for the modulation MLP. Default: ModConvActivation.SIGMOID.
+ num_weights : int, optional
+ Number of weight bases for ModConvType.SUM.
+ modulation_at_input : bool
+ If True, only the first conv block uses modulation. Default: False.
+ norm_type : NormType
+ Normalization type. Default: NormType.INSTANCE.
+ adain_hidden_features : int or tuple of int, optional
+ Hidden features for AdaIN.
"""
super().__init__()
@@ -256,14 +631,23 @@ def __init__(
num_filters=num_filters,
num_pool_layers=num_pool_layers,
dropout_probability=dropout_probability,
+ modulation_params=modulation_params,
+ modulation=modulation,
+ aux_in_features=aux_in_features,
+ fc_hidden_features=fc_hidden_features,
+ fc_groups=fc_groups,
+ fc_activation=fc_activation,
+ modulation_at_input=modulation_at_input,
+ norm_type=norm_type,
+ adain_hidden_features=adain_hidden_features,
+ num_weights=num_weights,
)
-
+ self.modulation = self.unet2d.modulation
self.norm_groups = norm_groups
@staticmethod
def norm(input_data: torch.Tensor, groups: int) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""Performs group normalization."""
- # group norm
b, c, h, w = input_data.shape
input_data = input_data.reshape(b, groups, -1)
@@ -282,7 +666,9 @@ def unnorm(input_data: torch.Tensor, mean: torch.Tensor, std: torch.Tensor, grou
return (input_data * std + mean).reshape(b, c, h, w)
@staticmethod
- def pad(input_data: torch.Tensor) -> Tuple[torch.Tensor, Tuple[List[int], List[int], int, int]]:
+ def pad(
+ input_data: torch.Tensor,
+ ) -> Tuple[torch.Tensor, Tuple[List[int], List[int], int, int]]:
_, _, h, w = input_data.shape
w_mult = ((w - 1) | 15) + 1
h_mult = ((h - 1) | 15) + 1
@@ -302,21 +688,22 @@ def unpad(
) -> torch.Tensor:
return input_data[..., h_pad[0] : h_mult - h_pad[1], w_pad[0] : w_mult - w_pad[1]]
- def forward(self, input_data: torch.Tensor) -> torch.Tensor:
+ def forward(self, input_data: torch.Tensor, aux_data: Optional[torch.Tensor] = None) -> torch.Tensor:
"""Performs forward pass of :class:`NormUnetModel2d`.
Parameters
----------
- input_data: torch.Tensor
+ input_data : torch.Tensor
+ aux_data : torch.Tensor, optional
Returns
-------
torch.Tensor
"""
-
output, mean, std = self.norm(input_data, self.norm_groups)
output, pad_sizes = self.pad(output)
- output = self.unet2d(output)
+
+ output = self.unet2d(output, aux_data)
output = self.unpad(output, *pad_sizes)
output = self.unnorm(output, mean, std, self.norm_groups)
@@ -337,15 +724,22 @@ def __init__(
skip_connection: bool = False,
normalized: bool = False,
image_initialization: InitType = InitType.ZERO_FILLED,
+ conv_modulation: ModConvType = ModConvType.NONE,
+ modulation_params: Optional[ModulationParams] = None,
+ aux_in_features: Optional[int] = None,
+ fc_hidden_features: Optional[tuple[int] | int] = None,
+ fc_groups: int = 1,
+ fc_activation: ModConvActivation = ModConvActivation.SIGMOID,
+ num_weights: Optional[int] = None,
**kwargs,
):
"""Inits :class:`Unet2d`.
Parameters
----------
- forward_operator: Callable
+ forward_operator: FFTOperator
Forward Operator.
- backward_operator: Callable
+ backward_operator: FFTOperator
Backward Operator.
num_filters: int
Number of first layer filters.
@@ -362,11 +756,22 @@ def __init__(
kwargs: dict
"""
super().__init__()
+ modulation_params = ModulationParams(
+ modulation=conv_modulation,
+ aux_in_features=aux_in_features,
+ fc_hidden_features=fc_hidden_features,
+ fc_groups=fc_groups,
+ fc_activation=fc_activation,
+ num_weights=num_weights,
+ )
extra_keys = kwargs.keys()
for extra_key in extra_keys:
if extra_key not in [
"sensitivity_map_model",
"model_name",
+ "auxiliary_features",
+ "log_aux",
+ "conv_modulation",
]:
raise ValueError(f"{type(self).__name__} got key `{extra_key}` which is not supported.")
self.unet: nn.Module
@@ -377,6 +782,7 @@ def __init__(
num_filters=num_filters,
num_pool_layers=num_pool_layers,
dropout_probability=dropout_probability,
+ modulation_params=modulation_params,
)
else:
self.unet = UnetModel2d(
@@ -385,7 +791,10 @@ def __init__(
num_filters=num_filters,
num_pool_layers=num_pool_layers,
dropout_probability=dropout_probability,
+ modulation_params=modulation_params,
)
+ self.conv_modulation = conv_modulation
+ self.modulation = conv_modulation
self.forward_operator = forward_operator
self.backward_operator = backward_operator
self.skip_connection = skip_connection
@@ -424,6 +833,7 @@ def forward(
self,
masked_kspace: torch.Tensor,
sensitivity_map: Optional[torch.Tensor] = None,
+ auxiliary_data: Optional[torch.Tensor] = None,
) -> torch.Tensor:
"""Computes forward pass of Unet2d.
@@ -454,7 +864,10 @@ def forward(
f"Got {self.image_initialization}."
)
- output = self.unet(input_image.permute(0, 3, 1, 2)).permute(0, 2, 3, 1)
+ if self.modulation != ModConvType.NONE:
+ output = self.unet(input_image.permute(0, 3, 1, 2), auxiliary_data).permute(0, 2, 3, 1)
+ else:
+ output = self.unet(input_image.permute(0, 3, 1, 2)).permute(0, 2, 3, 1)
if self.skip_connection:
output += input_image
return output
diff --git a/direct/nn/unet/unet_3d.py b/direct/nn/unet/unet_3d.py
index 019161ba..2c4d497b 100644
--- a/direct/nn/unet/unet_3d.py
+++ b/direct/nn/unet/unet_3d.py
@@ -17,16 +17,103 @@
from __future__ import annotations
import math
+from typing import Optional
import torch
from torch import nn
from torch.nn import functional as F
+from direct.nn.adain.adain import AdaIN3d, NormType
+from direct.nn.conv.modulated import ModConv2dBias, ModConv3d, ModConvActivation, ModConvTranspose3d, ModConvType
+
+
+class ConvModule3D(nn.Module):
+ """Single 3D convolution + norm + activation + dropout module supporting modulated convolutions."""
+
+ def __init__(
+ self,
+ in_channels: int,
+ out_channels: int,
+ kernel_size: int,
+ padding: int,
+ dropout_probability: float,
+ modulation: ModConvType = ModConvType.NONE,
+ bias: ModConv2dBias = ModConv2dBias.PARAM,
+ aux_in_features: Optional[int] = None,
+ fc_hidden_features: Optional[tuple[int] | int] = None,
+ fc_groups: int = 1,
+ fc_activation: ModConvActivation = ModConvActivation.SIGMOID,
+ num_weights: Optional[int] = None,
+ norm_type: NormType = NormType.INSTANCE,
+ adain_hidden_features: Optional[tuple[int] | int] = None,
+ ):
+ super().__init__()
+
+ self.modulation = modulation
+ self.norm_type = norm_type
+
+ self.conv = ModConv3d(
+ in_channels=in_channels,
+ out_channels=out_channels,
+ kernel_size=kernel_size,
+ padding=padding,
+ bias=bias,
+ modulation=modulation,
+ aux_in_features=aux_in_features,
+ fc_hidden_features=fc_hidden_features,
+ fc_groups=fc_groups,
+ fc_activation=fc_activation,
+ num_weights=num_weights,
+ )
+ if norm_type == NormType.ADAIN:
+ if adain_hidden_features is None:
+ raise ValueError("AdaIN hidden features must be provided if norm_type is NormType.ADAIN.")
+ if aux_in_features is None:
+ raise ValueError("aux_in_features must be provided if norm_type is NormType.ADAIN.")
+ self.instance_norm = AdaIN3d(
+ num_channels=out_channels,
+ aux_in_features=aux_in_features,
+ hidden_features=adain_hidden_features,
+ )
+ else:
+ self.instance_norm = nn.InstanceNorm3d(out_channels)
+ self.leaky_relu = nn.LeakyReLU(negative_slope=0.2, inplace=True)
+ self.dropout = nn.Dropout3d(dropout_probability)
+
+ def forward(self, x: torch.Tensor, y: Optional[torch.Tensor] = None) -> torch.Tensor:
+ if self.modulation != ModConvType.NONE:
+ x = self.conv(x, y)
+ else:
+ x = self.conv(x)
+
+ if self.norm_type == NormType.ADAIN:
+ if y is None:
+ raise ValueError("AdaIN requires aux vector y, but got None.")
+ x = self.instance_norm(x, y)
+ else:
+ x = self.instance_norm(x)
+ x = self.leaky_relu(x)
+ x = self.dropout(x)
+ return x
+
class ConvBlock3D(nn.Module):
- """3D U-Net convolutional block."""
+ """3D U-Net convolutional block with optional modulation and AdaIN support."""
- def __init__(self, in_channels: int, out_channels: int, dropout_probability: float) -> None:
+ def __init__(
+ self,
+ in_channels: int,
+ out_channels: int,
+ dropout_probability: float,
+ modulation: ModConvType = ModConvType.NONE,
+ aux_in_features: Optional[int] = None,
+ fc_hidden_features: Optional[tuple[int] | int] = None,
+ fc_groups: int = 1,
+ fc_activation: ModConvActivation = ModConvActivation.SIGMOID,
+ num_weights: Optional[int] = None,
+ norm_type: NormType = NormType.INSTANCE,
+ adain_hidden_features: Optional[tuple[int] | int] = None,
+ ) -> None:
"""Inits :class:`ConvBlock3D`.
Parameters
@@ -37,43 +124,97 @@ def __init__(self, in_channels: int, out_channels: int, dropout_probability: flo
Number of channels produced by the convolutional layers.
dropout_probability : float
Dropout probability applied after convolutional layers.
+ modulation : ModConvType
+ Modulation type. Default: ModConvType.NONE.
+ aux_in_features : int, optional
+ Number of auxiliary input features.
+ fc_hidden_features : int or tuple of int, optional
+ Hidden features for the modulation MLP.
+ fc_groups : int
+ Groups for the modulation MLP. Default: 1.
+ fc_activation : ModConvActivation
+ Activation for the modulation MLP. Default: ModConvActivation.SIGMOID.
+ num_weights : int, optional
+ Number of weight bases for ModConvType.SUM.
+ norm_type : NormType
+ Normalization type. Default: NormType.INSTANCE.
+ adain_hidden_features : int or tuple of int, optional
+ Hidden features for AdaIN.
"""
super().__init__()
self.in_channels = in_channels
self.out_channels = out_channels
self.dropout_probability = dropout_probability
+ self.modulation = modulation
+ self.norm_type = norm_type
- self.layers = nn.Sequential(
- nn.Conv3d(in_channels, out_channels, kernel_size=3, padding=1, bias=False),
- nn.InstanceNorm3d(out_channels),
- nn.LeakyReLU(negative_slope=0.2, inplace=True),
- nn.Dropout3d(dropout_probability),
- nn.Conv3d(out_channels, out_channels, kernel_size=3, padding=1, bias=False),
- nn.InstanceNorm3d(out_channels),
- nn.LeakyReLU(negative_slope=0.2, inplace=True),
- nn.Dropout3d(dropout_probability),
+ self.layer_1 = ConvModule3D(
+ in_channels=in_channels,
+ out_channels=out_channels,
+ kernel_size=3,
+ padding=1,
+ dropout_probability=dropout_probability,
+ bias=(ModConv2dBias.NONE if modulation == ModConvType.NONE else ModConv2dBias.LEARNED),
+ modulation=modulation,
+ aux_in_features=aux_in_features,
+ fc_hidden_features=fc_hidden_features,
+ fc_groups=fc_groups,
+ fc_activation=fc_activation,
+ num_weights=num_weights,
+ norm_type=norm_type,
+ adain_hidden_features=adain_hidden_features,
+ )
+ self.layer_2 = ConvModule3D(
+ in_channels=out_channels,
+ out_channels=out_channels,
+ kernel_size=3,
+ padding=1,
+ dropout_probability=dropout_probability,
+ bias=(ModConv2dBias.NONE if modulation == ModConvType.NONE else ModConv2dBias.LEARNED),
+ modulation=modulation,
+ aux_in_features=aux_in_features,
+ fc_hidden_features=fc_hidden_features,
+ fc_groups=fc_groups,
+ fc_activation=fc_activation,
+ num_weights=num_weights,
+ norm_type=norm_type,
+ adain_hidden_features=adain_hidden_features,
)
- def forward(self, input_data: torch.Tensor) -> torch.Tensor:
- """Performs the forward pass of :class:`ConvBlock3D`..
+ def forward(self, input_data: torch.Tensor, aux_data: Optional[torch.Tensor] = None) -> torch.Tensor:
+ """Performs the forward pass of :class:`ConvBlock3D`.
Parameters
----------
input_data : torch.Tensor
- Input data.
+ aux_data : torch.Tensor, optional
Returns
-------
torch.Tensor
"""
- return self.layers(input_data)
+ if self.modulation != ModConvType.NONE or self.norm_type == NormType.ADAIN:
+ return self.layer_2(self.layer_1(input_data, aux_data), aux_data)
+ return self.layer_2(self.layer_1(input_data))
class TransposeConvBlock3D(nn.Module):
- """3D U-Net Transpose Convolutional Block."""
+ """3D U-Net Transpose Convolutional Block with optional modulation."""
- def __init__(self, in_channels: int, out_channels: int) -> None:
+ def __init__(
+ self,
+ in_channels: int,
+ out_channels: int,
+ modulation: ModConvType = ModConvType.NONE,
+ aux_in_features: Optional[int] = None,
+ fc_hidden_features: Optional[tuple[int] | int] = None,
+ fc_groups: int = 1,
+ fc_activation: ModConvActivation = ModConvActivation.SIGMOID,
+ num_weights: Optional[int] = None,
+ norm_type: NormType = NormType.INSTANCE,
+ adain_hidden_features: Optional[tuple[int] | int] = None,
+ ) -> None:
"""Inits :class:`TransposeConvBlock3D`.
Parameters
@@ -82,38 +223,93 @@ def __init__(self, in_channels: int, out_channels: int) -> None:
Number of channels in the input tensor.
out_channels : int
Number of channels produced by the convolutional layers.
+ modulation : ModConvType
+ Modulation type. Default: ModConvType.NONE.
+ aux_in_features : int, optional
+ Number of auxiliary input features.
+ fc_hidden_features : int or tuple of int, optional
+ Hidden features for the modulation MLP.
+ fc_groups : int
+ Groups for the modulation MLP. Default: 1.
+ fc_activation : ModConvActivation
+ Activation for the modulation MLP. Default: ModConvActivation.SIGMOID.
+ num_weights : int, optional
+ Number of weight bases for ModConvType.SUM.
+ norm_type : NormType
+ Normalization type. Default: NormType.INSTANCE.
+ adain_hidden_features : int or tuple of int, optional
+ Hidden features for AdaIN.
"""
super().__init__()
self.in_channels = in_channels
self.out_channels = out_channels
+ self.modulation = modulation
+ self.norm_type = norm_type
- self.layers = nn.Sequential(
- nn.ConvTranspose3d(in_channels, out_channels, kernel_size=2, stride=2, bias=False),
- nn.InstanceNorm3d(out_channels),
- nn.LeakyReLU(negative_slope=0.2, inplace=True),
+ self.conv = ModConvTranspose3d(
+ in_channels=in_channels,
+ out_channels=out_channels,
+ kernel_size=2,
+ stride=2,
+ modulation=modulation,
+ bias=(ModConv2dBias.NONE if modulation == ModConvType.NONE else ModConv2dBias.LEARNED),
+ aux_in_features=aux_in_features,
+ fc_hidden_features=fc_hidden_features,
+ fc_groups=fc_groups,
+ fc_activation=fc_activation,
+ num_weights=num_weights,
)
+ if norm_type == NormType.ADAIN:
+ if adain_hidden_features is None:
+ raise ValueError("AdaIN hidden features must be provided if norm_type is NormType.ADAIN.")
+ if aux_in_features is None:
+ raise ValueError("aux_in_features must be provided if norm_type is NormType.ADAIN.")
+ self.instance_norm = AdaIN3d(
+ num_channels=out_channels,
+ aux_in_features=aux_in_features,
+ hidden_features=adain_hidden_features,
+ )
+ else:
+ self.instance_norm = nn.InstanceNorm3d(out_channels)
+ self.leaky_relu = nn.LeakyReLU(negative_slope=0.2, inplace=True)
- def forward(self, input_data: torch.Tensor) -> torch.Tensor:
+ def forward(self, input_data: torch.Tensor, aux_data: Optional[torch.Tensor] = None) -> torch.Tensor:
"""Performs the forward pass of :class:`TransposeConvBlock3D`.
Parameters
----------
input_data : torch.Tensor
- Input data.
+ aux_data : torch.Tensor, optional
Returns
-------
torch.Tensor
"""
- return self.layers(input_data)
+ if self.modulation != ModConvType.NONE:
+ x = self.conv(input_data, aux_data)
+ else:
+ x = self.conv(input_data)
+ if self.norm_type == NormType.ADAIN:
+ x = self.instance_norm(x, aux_data)
+ else:
+ x = self.instance_norm(x)
+ return self.leaky_relu(x)
class UnetModel3d(nn.Module):
- """PyTorch implementation of a 3D U-Net model.
+ """PyTorch implementation of a 3D U-Net model with optional modulated convolutions.
This class defines a 3D U-Net architecture consisting of down-sampling and up-sampling layers with 3D convolutional
blocks. This is an extension to 3D volumes of :class:`direct.nn.unet.unet_2d.UnetModel2d`.
+ Modulated convolutions are based on [1]_.
+
+ References
+ ----------
+
+ .. [1] Moriakov, N., Yiasemis, G., Sonke, J.-J. & Teuwen, J. (2026). Conditional Learned Reconstruction for
+ Medical Imaging. Proceedings of The 9th International Conference on Medical Imaging with Deep Learning,
+ PMLR 315:754-780. https://proceedings.mlr.press/v315/moriakov26a.html
"""
def __init__(
@@ -123,6 +319,15 @@ def __init__(
num_filters: int,
num_pool_layers: int,
dropout_probability: float,
+ modulation: ModConvType = ModConvType.NONE,
+ aux_in_features: Optional[int] = None,
+ fc_hidden_features: Optional[tuple[int] | int] = None,
+ fc_groups: int = 1,
+ fc_activation: ModConvActivation = ModConvActivation.SIGMOID,
+ num_weights: Optional[int] = None,
+ modulation_at_input: bool = False,
+ norm_type: NormType = NormType.INSTANCE,
+ adain_hidden_features: Optional[tuple[int] | int] = None,
) -> None:
"""Inits :class:`UnetModel3d`.
@@ -138,6 +343,24 @@ def __init__(
Number of down-sampling and up-sampling layers (depth).
dropout_probability : float
Dropout probability.
+ modulation : ModConvType
+ Modulation type. Default: ModConvType.NONE.
+ aux_in_features : int, optional
+ Number of auxiliary input features.
+ fc_hidden_features : int or tuple of int, optional
+ Hidden features for the modulation MLP.
+ fc_groups : int
+ Groups for the modulation MLP. Default: 1.
+ fc_activation : ModConvActivation
+ Activation for the modulation MLP. Default: ModConvActivation.SIGMOID.
+ num_weights : int, optional
+ Number of weight bases for ModConvType.SUM.
+ modulation_at_input : bool
+ If True, only the first conv block uses modulation. Default: False.
+ norm_type : NormType
+ Normalization type. Default: NormType.INSTANCE.
+ adain_hidden_features : int or tuple of int, optional
+ Hidden features for AdaIN.
"""
super().__init__()
@@ -146,36 +369,140 @@ def __init__(
self.num_filters = num_filters
self.num_pool_layers = num_pool_layers
self.dropout_probability = dropout_probability
-
- self.down_sample_layers = nn.ModuleList([ConvBlock3D(in_channels, num_filters, dropout_probability)])
+ self.modulation = modulation
+ self.norm_type = norm_type
+
+ self.down_sample_layers = nn.ModuleList(
+ [
+ ConvBlock3D(
+ in_channels,
+ num_filters,
+ dropout_probability,
+ modulation,
+ aux_in_features,
+ fc_hidden_features,
+ fc_groups,
+ fc_activation,
+ num_weights,
+ norm_type,
+ adain_hidden_features,
+ )
+ ]
+ )
ch = num_filters
+
+ if modulation != ModConvType.NONE and modulation_at_input:
+ modulation = ModConvType.NONE
+
for _ in range(num_pool_layers - 1):
- self.down_sample_layers += [ConvBlock3D(ch, ch * 2, dropout_probability)]
+ self.down_sample_layers += [
+ ConvBlock3D(
+ ch,
+ ch * 2,
+ dropout_probability,
+ modulation,
+ aux_in_features,
+ fc_hidden_features,
+ fc_groups,
+ fc_activation,
+ num_weights,
+ norm_type,
+ adain_hidden_features,
+ )
+ ]
ch *= 2
- self.conv = ConvBlock3D(ch, ch * 2, dropout_probability)
+ self.conv = ConvBlock3D(
+ ch,
+ ch * 2,
+ dropout_probability,
+ modulation,
+ aux_in_features,
+ fc_hidden_features,
+ fc_groups,
+ fc_activation,
+ num_weights,
+ norm_type,
+ adain_hidden_features,
+ )
self.up_conv = nn.ModuleList()
self.up_transpose_conv = nn.ModuleList()
for _ in range(num_pool_layers - 1):
- self.up_transpose_conv += [TransposeConvBlock3D(ch * 2, ch)]
- self.up_conv += [ConvBlock3D(ch * 2, ch, dropout_probability)]
+ self.up_transpose_conv += [
+ TransposeConvBlock3D(
+ ch * 2,
+ ch,
+ modulation,
+ aux_in_features,
+ fc_hidden_features,
+ fc_groups,
+ fc_activation,
+ num_weights,
+ norm_type,
+ adain_hidden_features,
+ )
+ ]
+ self.up_conv += [
+ ConvBlock3D(
+ ch * 2,
+ ch,
+ dropout_probability,
+ modulation,
+ aux_in_features,
+ fc_hidden_features,
+ fc_groups,
+ fc_activation,
+ num_weights,
+ norm_type,
+ adain_hidden_features,
+ )
+ ]
ch //= 2
- self.up_transpose_conv += [TransposeConvBlock3D(ch * 2, ch)]
+ if modulation != ModConvType.NONE and modulation_at_input:
+ modulation = ModConvType.NONE
+ self.up_transpose_conv += [
+ TransposeConvBlock3D(
+ ch * 2,
+ ch,
+ modulation,
+ aux_in_features,
+ fc_hidden_features,
+ fc_groups,
+ fc_activation,
+ num_weights,
+ norm_type,
+ adain_hidden_features,
+ )
+ ]
self.up_conv += [
nn.Sequential(
- ConvBlock3D(ch * 2, ch, dropout_probability),
+ ConvBlock3D(
+ ch * 2,
+ ch,
+ dropout_probability,
+ modulation,
+ aux_in_features,
+ fc_hidden_features,
+ fc_groups,
+ fc_activation,
+ num_weights,
+ norm_type,
+ adain_hidden_features,
+ ),
nn.Conv3d(ch, out_channels, kernel_size=1, stride=1),
)
]
- def forward(self, input_data: torch.Tensor) -> torch.Tensor:
+ def forward(self, input_data: torch.Tensor, aux_data: Optional[torch.Tensor] = None) -> torch.Tensor:
"""Performs forward pass of :class:`UnetModel3d`.
Parameters
----------
input_data : torch.Tensor
Input tensor of shape (N, in_channels, slice/time, height, width).
+ aux_data : torch.Tensor, optional
+ Auxiliary data for modulation/AdaIN.
Returns
-------
@@ -185,18 +512,25 @@ def forward(self, input_data: torch.Tensor) -> torch.Tensor:
stack = []
output, inp_pad = pad_to_pow_of_2(input_data, self.num_pool_layers)
- # Apply down-sampling layers
- for _, layer in enumerate(self.down_sample_layers):
- output = layer(output)
+ for layer in self.down_sample_layers:
+ if self.modulation != ModConvType.NONE or self.norm_type == NormType.ADAIN:
+ output = layer(output, aux_data)
+ else:
+ output = layer(output)
stack.append(output)
output = F.avg_pool3d(output, kernel_size=2, stride=2, padding=0)
- output = self.conv(output)
+ if self.modulation != ModConvType.NONE or self.norm_type == NormType.ADAIN:
+ output = self.conv(output, aux_data)
+ else:
+ output = self.conv(output)
- # Apply up-sampling layers
for transpose_conv, conv in zip(self.up_transpose_conv, self.up_conv):
downsample_layer = stack.pop()
- output = transpose_conv(output)
+ if self.modulation != ModConvType.NONE or self.norm_type == NormType.ADAIN:
+ output = transpose_conv(output, aux_data)
+ else:
+ output = transpose_conv(output)
padding = [0, 0, 0, 0, 0, 0]
if output.shape[-1] != downsample_layer.shape[-1]:
@@ -209,7 +543,14 @@ def forward(self, input_data: torch.Tensor) -> torch.Tensor:
output = F.pad(output, padding, "reflect")
output = torch.cat([output, downsample_layer], dim=1)
- output = conv(output)
+ if self.modulation != ModConvType.NONE or self.norm_type == NormType.ADAIN:
+ if isinstance(conv, nn.Sequential):
+ output = conv[0](output, aux_data)
+ output = conv[1](output)
+ else:
+ output = conv(output, aux_data)
+ else:
+ output = conv(output)
if sum(inp_pad) != 0:
output = output[
@@ -224,7 +565,7 @@ def forward(self, input_data: torch.Tensor) -> torch.Tensor:
class NormUnetModel3d(nn.Module):
- """Implementation of a Normalized U-Net model for 3D data.
+ """Implementation of a Normalized U-Net model for 3D data with optional modulation.
This is an extension to 3D volumes of :class:`direct.nn.unet.unet_2d.NormUnetModel2d`.
"""
@@ -237,8 +578,17 @@ def __init__(
num_pool_layers: int,
dropout_probability: float,
norm_groups: int = 2,
+ modulation: ModConvType = ModConvType.NONE,
+ aux_in_features: Optional[int] = None,
+ fc_hidden_features: Optional[tuple[int] | int] = None,
+ fc_groups: int = 1,
+ fc_activation: ModConvActivation = ModConvActivation.SIGMOID,
+ num_weights: Optional[int] = None,
+ modulation_at_input: bool = False,
+ norm_type: NormType = NormType.INSTANCE,
+ adain_hidden_features: Optional[tuple[int] | int] = None,
) -> None:
- """Inits :class:`NormUnetModel3D`.
+ """Inits :class:`NormUnetModel3d`.
Parameters
----------
@@ -252,8 +602,26 @@ def __init__(
Number of down-sampling and up-sampling layers (depth).
dropout_probability : float
Dropout probability.
- norm_groups: int,
+ norm_groups: int
Number of normalization groups.
+ modulation : ModConvType
+ Modulation type. Default: ModConvType.NONE.
+ aux_in_features : int, optional
+ Number of auxiliary input features.
+ fc_hidden_features : int or tuple of int, optional
+ Hidden features for the modulation MLP.
+ fc_groups : int
+ Groups for the modulation MLP. Default: 1.
+ fc_activation : ModConvActivation
+ Activation for the modulation MLP. Default: ModConvActivation.SIGMOID.
+ num_weights : int, optional
+ Number of weight bases for ModConvType.SUM.
+ modulation_at_input : bool
+ If True, only the first conv block uses modulation. Default: False.
+ norm_type : NormType
+ Normalization type. Default: NormType.INSTANCE.
+ adain_hidden_features : int or tuple of int, optional
+ Hidden features for AdaIN.
"""
super().__init__()
@@ -263,27 +631,24 @@ def __init__(
num_filters=num_filters,
num_pool_layers=num_pool_layers,
dropout_probability=dropout_probability,
+ modulation=modulation,
+ aux_in_features=aux_in_features,
+ fc_hidden_features=fc_hidden_features,
+ fc_groups=fc_groups,
+ fc_activation=fc_activation,
+ num_weights=num_weights,
+ modulation_at_input=modulation_at_input,
+ norm_type=norm_type,
+ adain_hidden_features=adain_hidden_features,
)
self.norm_groups = norm_groups
+ self.modulation = modulation
+ self.norm_type = norm_type
@staticmethod
def norm(input_data: torch.Tensor, groups: int) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
- """Applies group normalization for 3D data.
-
- Parameters
- ----------
- input_data : torch.Tensor
- The input tensor to normalize.
- groups : int
- The number of groups to divide the tensor into for normalization.
-
- Returns
- -------
- tuple[torch.Tensor, torch.Tensor, torch.Tensor]
- A tuple containing the normalized tensor, the mean, and the standard deviation used for normalization.
- """
- # Group norm
+ """Performs group normalization."""
b, c, z, h, w = input_data.shape
input_data = input_data.reshape(b, groups, -1)
@@ -296,30 +661,7 @@ def norm(input_data: torch.Tensor, groups: int) -> tuple[torch.Tensor, torch.Ten
return output, mean, std
@staticmethod
- def unnorm(
- input_data: torch.Tensor,
- mean: torch.Tensor,
- std: torch.Tensor,
- groups: int,
- ) -> torch.Tensor:
- """Reverts the normalization applied to the 3D tensor.
-
- Parameters
- ----------
- input_data : torch.Tensor
- The normalized tensor to revert normalization on.
- mean : torch.Tensor
- The mean used during normalization.
- std : torch.Tensor
- The standard deviation used during normalization.
- groups : int
- The number of groups the tensor was divided into during normalization.
-
- Returns
- -------
- torch.Tensor
- The tensor after reverting the normalization.
- """
+ def unnorm(input_data: torch.Tensor, mean: torch.Tensor, std: torch.Tensor, groups: int) -> torch.Tensor:
b, c, z, h, w = input_data.shape
input_data = input_data.reshape(b, groups, -1)
return (input_data * std + mean).reshape(b, c, z, h, w)
@@ -328,19 +670,6 @@ def unnorm(
def pad(
input_data: torch.Tensor,
) -> tuple[torch.Tensor, tuple[list[int], list[int], list[int], int, int, int]]:
- """Applies padding to the input 3D tensor to ensure its dimensions are multiples of 16.
-
- Parameters
- ----------
- input_data : torch.Tensor
- The input tensor to pad.
-
- Returns
- -------
- tuple[torch.Tensor, tuple[list[int], list[int], int, int, list[int], list[int]]]
- A tuple containing the padded tensor and a tuple with the padding applied to each dimension
- (height, width, depth) and the target dimensions after padding.
- """
_, _, z, h, w = input_data.shape
w_mult = ((w - 1) | 15) + 1
h_mult = ((h - 1) | 15) + 1
@@ -362,50 +691,32 @@ def unpad(
w_mult: int,
z_mult: int,
) -> torch.Tensor:
- """Removes padding from the 3D input tensor, reverting it to its original dimensions before padding was applied.
-
- This method is typically used after the model has processed the padded input.
-
- Parameters
- ----------
- input_data : torch.Tensor
- The tensor from which padding will be removed.
- h_pad : list[int]
- Padding applied to the height, specified as [top, bottom].
- w_pad : list[int]
- Padding applied to the width, specified as [left, right].
- z_pad : list[int]
- Padding applied to the depth, specified as [front, back].
- h_mult : int
- The height as computed in the `pad` method.
- w_mult : int
- The width as computed in the `pad` method.
- z_mult : int
- The depth as computed in the `pad` method.
-
- Returns
- -------
- torch.Tensor
- The tensor with padding removed, restored to its original dimensions.
- """
- return input_data[..., z_pad[0] : z_mult - z_pad[1], h_pad[0] : h_mult - h_pad[1], w_pad[0] : w_mult - w_pad[1]]
+ return input_data[
+ ...,
+ z_pad[0] : z_mult - z_pad[1],
+ h_pad[0] : h_mult - h_pad[1],
+ w_pad[0] : w_mult - w_pad[1],
+ ]
- def forward(self, input_data: torch.Tensor) -> torch.Tensor:
- """Performs the forward pass of :class:`NormUnetModel3D`.
+ def forward(self, input_data: torch.Tensor, aux_data: Optional[torch.Tensor] = None) -> torch.Tensor:
+ """Performs the forward pass of :class:`NormUnetModel3d`.
Parameters
----------
input_data : torch.Tensor
- Input tensor of shape (N, in_channels, slice/time, height, width).
+ aux_data : torch.Tensor, optional
Returns
-------
torch.Tensor
- Output of shape (N, out_channels, slice/time, height, width).
"""
output, mean, std = self.norm(input_data, self.norm_groups)
output, pad_sizes = self.pad(output)
- output = self.unet3d(output)
+
+ if self.modulation != ModConvType.NONE or self.norm_type == NormType.ADAIN:
+ output = self.unet3d(output, aux_data)
+ else:
+ output = self.unet3d(output)
h_pad, w_pad, z_pad, h_mult, w_mult, z_mult = pad_sizes
output = self.unpad(output, h_pad, w_pad, z_pad, h_mult, w_mult, z_mult)
@@ -415,30 +726,19 @@ def forward(self, input_data: torch.Tensor) -> torch.Tensor:
def pad_to_pow_of_2(inp: torch.Tensor, k: int) -> tuple[torch.Tensor, list[int]]:
- """Pads the input tensor along the spatial dimensions (depth, height, width) to the nearest power of 2.
-
- This is necessary for certain operations in the 3D U-Net architecture to maintain dimensionality.
+ """Pads the input tensor along the spatial dimensions to the nearest power of 2.
Parameters
----------
inp : torch.Tensor
The input tensor to be padded.
k : int
- The exponent to which the base of 2 is raised to determine the padding. Used to calculate
- the target dimension size as a power of 2.
+ The exponent to which 2 is raised to determine target dimension size.
Returns
-------
tuple[torch.Tensor, list[int]]
- A tuple containing the padded tensor and a list of padding applied to each spatial dimension
- in the format [depth_front, depth_back, height_top, height_bottom, width_left, width_right].
-
- Examples
- --------
- >>> inp = torch.rand(1, 1, 15, 15, 15) # A random tensor with shape [1, 1, 15, 15, 15]
- >>> padded_inp, padding = pad_to_pow_of_2(inp, 4)
- >>> print(padded_inp.shape, padding)
- torch.Size([...]), [1, 1, 1, 1, 1, 1]
+ A tuple containing the padded tensor and the padding list.
"""
diffs = [_ - 2**k for _ in inp.shape[2:]]
padding = [0, 0, 0, 0, 0, 0]
diff --git a/direct/nn/unet/unet_engine.py b/direct/nn/unet/unet_engine.py
index beeedd09..a3ef4b97 100644
--- a/direct/nn/unet/unet_engine.py
+++ b/direct/nn/unet/unet_engine.py
@@ -110,7 +110,11 @@ def forward_function(self, data: dict[str, Any]) -> tuple[torch.Tensor, None]:
data["sensitivity_map"] if self.cfg.model.image_initialization == "sense" else None # type: ignore
)
- output_image = self.model(masked_kspace=data["masked_kspace"], sensitivity_map=sensitity_map)
+ output_image = self.model(
+ masked_kspace=data["masked_kspace"],
+ sensitivity_map=sensitity_map,
+ auxiliary_data=self.auxiliary_data_from(data),
+ )
output_image = T.modulus(output_image)
output_kspace = None
@@ -207,7 +211,11 @@ def forward_function(self, data: dict[str, Any]) -> tuple[torch.Tensor, None]:
data["sensitivity_map"] if self.cfg.model.image_initialization == "sense" else None # type: ignore
)
- output_image = self.model(masked_kspace=kspace, sensitivity_map=sensitity_map)
+ output_image = self.model(
+ masked_kspace=kspace,
+ sensitivity_map=sensitity_map,
+ auxiliary_data=self.auxiliary_data_from(data),
+ )
output_kspace = None
return output_image, output_kspace
@@ -308,7 +316,11 @@ def forward_function(self, data: dict[str, Any]) -> tuple[torch.Tensor, None]:
data["sensitivity_map"] if self.cfg.model.image_initialization == "sense" else None # type: ignore
)
- output_image = self.model(masked_kspace=kspace, sensitivity_map=sensitity_map)
+ output_image = self.model(
+ masked_kspace=kspace,
+ sensitivity_map=sensitity_map,
+ auxiliary_data=self.auxiliary_data_from(data),
+ )
output_kspace = None
return output_image, output_kspace
diff --git a/direct/nn/varnet/config.py b/direct/nn/varnet/config.py
index af49108c..9c8e5f57 100644
--- a/direct/nn/varnet/config.py
+++ b/direct/nn/varnet/config.py
@@ -12,8 +12,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from dataclasses import dataclass
+from typing import Optional
from direct.config.defaults import ModelConfig
+from direct.nn.conv.modulated import ModConvActivation, ModConvType
@dataclass
@@ -22,3 +24,11 @@ class EndToEndVarNetConfig(ModelConfig):
regularizer_num_filters: int = 18
regularizer_num_pull_layers: int = 4
regularizer_dropout: float = 0.0
+ conv_modulation: ModConvType = ModConvType.NONE
+ aux_in_features: Optional[int] = None
+ auxiliary_features: Optional[tuple[str, ...]] = None
+ log_aux: bool = False
+ fc_hidden_features: Optional[tuple[int, ...]] = None
+ fc_groups: int = 1
+ fc_activation: ModConvActivation = ModConvActivation.SIGMOID
+ num_weights: Optional[int] = None
diff --git a/direct/nn/varnet/varnet.py b/direct/nn/varnet/varnet.py
index 56b19d0a..59657ce0 100644
--- a/direct/nn/varnet/varnet.py
+++ b/direct/nn/varnet/varnet.py
@@ -12,10 +12,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import annotations
+
+from typing import Optional
+
import torch
import torch.nn as nn
from direct.data.transforms import expand_operator, reduce_operator
+from direct.nn.conv.modulated import ModConvActivation, ModConvType
from direct.nn.unet import UnetModel2d
from direct.types import FFTOperator
@@ -23,11 +28,17 @@
class EndToEndVarNet(nn.Module):
"""End-to-End Variational Network based on [1]_.
+ Supports conditional weight modulation as proposed in [2]_.
+
References
----------
- .. [1] Sriram, Anuroop, et al. “End-to-End Variational Networks for Accelerated MRI Reconstruction.”
+ .. [1] Sriram, Anuroop, et al. "End-to-End Variational Networks for Accelerated MRI Reconstruction."
ArXiv:2004.06688 [Cs, Eess], Apr. 2020. arXiv.org, http://arxiv.org/abs/2004.06688.
+
+ .. [2] Moriakov, N., Yiasemis, G., Sonke, J.-J. & Teuwen, J. (2026). Conditional Learned Reconstruction for
+ Medical Imaging. Proceedings of The 9th International Conference on Medical Imaging with Deep Learning,
+ PMLR 315:754-780. https://proceedings.mlr.press/v315/moriakov26a.html
"""
def __init__(
@@ -39,6 +50,12 @@ def __init__(
regularizer_num_pull_layers: int = 4,
regularizer_dropout: float = 0.0,
in_channels: int = 2,
+ conv_modulation: ModConvType = ModConvType.NONE,
+ aux_in_features: Optional[int] = None,
+ fc_hidden_features: Optional[tuple[int] | int] = None,
+ fc_groups: int = 1,
+ fc_activation: ModConvActivation = ModConvActivation.SIGMOID,
+ num_weights: Optional[int] = None,
**kwargs,
):
"""Inits :class:`EndToEndVarNet`.
@@ -57,15 +74,32 @@ def __init__(
Regularizer model number of pulling layers.
regularizer_dropout: float
Regularizer model dropout probability.
+ in_channels: int
+ Number of input channels. Default: 2.
+ conv_modulation : ModConvType
+ Modulation type for convolutional layers. Default: ModConvType.NONE.
+ aux_in_features : int, optional
+ Number of features in the auxiliary input for modulation.
+ fc_hidden_features : int or tuple of int, optional
+ Hidden features in the modulation MLP.
+ fc_groups : int
+ Groups for modulation MLP output. Default: 1.
+ fc_activation : ModConvActivation
+ Activation after modulation MLP. Default: ModConvActivation.SIGMOID.
+ num_weights : int, optional
+ Number of weight bases for ModConvType.SUM.
"""
super().__init__()
extra_keys = kwargs.keys()
for extra_key in extra_keys:
- if extra_key not in [
+ if extra_key not in (
"model_name",
- ]:
+ "log_aux",
+ "auxiliary_features",
+ ):
raise ValueError(f"{type(self).__name__} got key `{extra_key}` which is not supported.")
+ self.conv_modulation = conv_modulation
self.layers_list = nn.ModuleList()
for _ in range(num_layers):
@@ -79,12 +113,23 @@ def __init__(
num_filters=regularizer_num_filters,
num_pool_layers=regularizer_num_pull_layers,
dropout_probability=regularizer_dropout,
+ modulation=conv_modulation,
+ aux_in_features=aux_in_features,
+ fc_hidden_features=fc_hidden_features,
+ fc_groups=fc_groups,
+ fc_activation=fc_activation,
+ num_weights=num_weights,
),
+ conv_modulation=conv_modulation,
)
)
def forward(
- self, masked_kspace: torch.Tensor, sampling_mask: torch.Tensor, sensitivity_map: torch.Tensor
+ self,
+ masked_kspace: torch.Tensor,
+ sampling_mask: torch.Tensor,
+ sensitivity_map: torch.Tensor,
+ auxiliary_data: Optional[torch.Tensor] = None,
) -> torch.Tensor:
"""Performs the forward pass of :class:`EndToEndVarNet`.
@@ -96,6 +141,8 @@ def forward(
Sampling mask of shape (N, 1, height, width, 1).
sensitivity_map: torch.Tensor
Sensitivity map of shape (N, coil, height, width, complex=2).
+ auxiliary_data: torch.Tensor, optional
+ Auxiliary data for modulation of shape (N, aux_in_features).
Returns
-------
@@ -105,7 +152,13 @@ def forward(
kspace_prediction = masked_kspace.clone()
for layer in self.layers_list:
- kspace_prediction = layer(kspace_prediction, masked_kspace, sampling_mask, sensitivity_map)
+ kspace_prediction = layer(
+ kspace_prediction,
+ masked_kspace,
+ sampling_mask,
+ sensitivity_map,
+ auxiliary_data,
+ )
return kspace_prediction
@@ -117,6 +170,7 @@ def __init__(
forward_operator: FFTOperator,
backward_operator: FFTOperator,
regularizer_model: nn.Module,
+ conv_modulation: ModConvType = ModConvType.NONE,
):
"""Inits :class:`EndToEndVarNetBlock`.
@@ -128,12 +182,15 @@ def __init__(
Backward Operator.
regularizer_model: nn.Module
Regularizer model.
+ conv_modulation: ModConvType
+ Modulation type. Default: ModConvType.NONE.
"""
super().__init__()
self.regularizer_model = regularizer_model
self.forward_operator = forward_operator
self.backward_operator = backward_operator
self.learning_rate = nn.Parameter(torch.tensor([1.0]))
+ self.conv_modulation = conv_modulation
self._coil_dim = 1
self._complex_dim = -1
self._spatial_dims = (2, 3)
@@ -144,6 +201,7 @@ def forward(
masked_kspace: torch.Tensor,
sampling_mask: torch.Tensor,
sensitivity_map: torch.Tensor,
+ auxiliary_data: Optional[torch.Tensor] = None,
) -> torch.Tensor:
"""Performs the forward pass of :class:`EndToEndVarNetBlock`.
@@ -157,6 +215,8 @@ def forward(
Sampling mask of shape (N, 1, height, width, 1).
sensitivity_map: torch.Tensor
Sensitivity map of shape (N, coil, height, width, complex=2).
+ auxiliary_data: torch.Tensor, optional
+ Auxiliary data for modulation of shape (N, aux_in_features).
Returns
-------
@@ -171,17 +231,25 @@ def forward(
regularization_term = torch.cat(
[
reduce_operator(
- self.backward_operator(kspace, dim=self._spatial_dims), sensitivity_map, dim=self._coil_dim
+ self.backward_operator(kspace, dim=self._spatial_dims),
+ sensitivity_map,
+ dim=self._coil_dim,
)
for kspace in torch.split(current_kspace, 2, self._complex_dim)
],
dim=self._complex_dim,
).permute(0, 3, 1, 2)
- regularization_term = self.regularizer_model(regularization_term).permute(0, 2, 3, 1)
+
+ if self.conv_modulation != ModConvType.NONE:
+ regularization_term = self.regularizer_model(regularization_term, auxiliary_data).permute(0, 2, 3, 1)
+ else:
+ regularization_term = self.regularizer_model(regularization_term).permute(0, 2, 3, 1)
+
regularization_term = torch.cat(
[
self.forward_operator(
- expand_operator(image, sensitivity_map, dim=self._coil_dim), dim=self._spatial_dims
+ expand_operator(image, sensitivity_map, dim=self._coil_dim),
+ dim=self._spatial_dims,
)
for image in torch.split(regularization_term, 2, self._complex_dim)
],
diff --git a/direct/nn/varnet/varnet_engine.py b/direct/nn/varnet/varnet_engine.py
index 22fbd4b3..47df7b3d 100644
--- a/direct/nn/varnet/varnet_engine.py
+++ b/direct/nn/varnet/varnet_engine.py
@@ -92,10 +92,13 @@ def __init__(
)
def forward_function(self, data: dict[str, Any]) -> tuple[torch.Tensor, torch.Tensor]:
+ auxiliary_data = self.auxiliary_data_from(data)
+
output_kspace = self.model(
masked_kspace=data["masked_kspace"],
sampling_mask=data["sampling_mask"],
sensitivity_map=data["sensitivity_map"],
+ auxiliary_data=auxiliary_data,
)
output_image = T.root_sum_of_squares(
self.backward_operator(output_kspace, dim=self._spatial_dims),
@@ -197,6 +200,7 @@ def forward_function(self, data: dict[str, Any]) -> tuple[None, torch.Tensor]:
masked_kspace=kspace,
sampling_mask=mask,
sensitivity_map=data["sensitivity_map"],
+ auxiliary_data=self.auxiliary_data_from(data),
)
output_image = None
@@ -298,6 +302,7 @@ def forward_function(self, data: dict[str, Any]) -> tuple[None, torch.Tensor]:
masked_kspace=kspace,
sampling_mask=mask,
sensitivity_map=data["sensitivity_map"],
+ auxiliary_data=self.auxiliary_data_from(data),
)
output_image = None
diff --git a/direct/nn/vsharp/config.py b/direct/nn/vsharp/config.py
index 0a39e942..9d6eda32 100644
--- a/direct/nn/vsharp/config.py
+++ b/direct/nn/vsharp/config.py
@@ -15,8 +15,11 @@
from __future__ import annotations
from dataclasses import dataclass
+from typing import Optional
from direct.config.defaults import ModelConfig
+from direct.nn.adain.adain import NormType
+from direct.nn.conv.modulated import ModConvActivation, ModConvType
from direct.nn.types import ActivationType, InitType, ModelName
@@ -32,6 +35,15 @@ class VSharpNetConfig(ModelConfig):
initializer_dilations: tuple[int, ...] = (1, 1, 2, 4)
initializer_multiscale: int = 1
initializer_activation: ActivationType = ActivationType.PRELU
+ conv_modulation: ModConvType = ModConvType.NONE
+ aux_in_features: int = 2
+ auxiliary_features: Optional[tuple[str, ...]] = None
+ log_aux: bool = False
+ fc_hidden_features: Optional[tuple[int]] = None
+ fc_groups: int = 1
+ fc_activation: ModConvActivation = ModConvActivation.SIGMOID
+ num_weights: Optional[int] = None
+ modulation_at_input: bool = False
image_resnet_hidden_channels: int = 128
image_resnet_num_blocks: int = 15
image_resnet_batchnorm: bool = True
@@ -39,6 +51,8 @@ class VSharpNetConfig(ModelConfig):
image_unet_num_filters: int = 32
image_unet_num_pool_layers: int = 4
image_unet_dropout: float = 0.0
+ image_unet_norm_type: NormType = NormType.INSTANCE
+ image_unet_adain_hidden_features: Optional[tuple[int]] = None
image_didn_hidden_channels: int = 16
image_didn_num_dubs: int = 6
image_didn_num_convs_recon: int = 9
@@ -59,7 +73,18 @@ class VSharpNet3DConfig(ModelConfig):
initializer_dilations: tuple[int, ...] = (1, 1, 2, 4)
initializer_multiscale: int = 1
initializer_activation: ActivationType = ActivationType.PRELU
+ conv_modulation: ModConvType = ModConvType.NONE
+ aux_in_features: int = 2
+ auxiliary_features: Optional[tuple[str, ...]] = None
+ fc_hidden_features: Optional[tuple[int]] = None
+ fc_groups: int = 1
+ fc_activation: ModConvActivation = ModConvActivation.SIGMOID
+ num_weights: Optional[int] = None
+ modulation_at_input: bool = False
+ log_aux: bool = True
unet_num_filters: int = 32
unet_num_pool_layers: int = 4
unet_dropout: float = 0.0
unet_norm: bool = False
+ unet_norm_type: NormType = NormType.INSTANCE
+ unet_adain_hidden_features: Optional[tuple[int]] = None
diff --git a/direct/nn/vsharp/vsharp.py b/direct/nn/vsharp/vsharp.py
index e9c1b568..c25352c6 100644
--- a/direct/nn/vsharp/vsharp.py
+++ b/direct/nn/vsharp/vsharp.py
@@ -27,6 +27,8 @@
from __future__ import annotations
+from typing import Optional, cast
+
import numpy as np
import torch
import torch.nn.functional as F
@@ -34,6 +36,15 @@
from direct.constants import COMPLEX_SIZE
from direct.data.transforms import apply_mask, expand_operator, reduce_operator
+from direct.nn.adain.adain import NormType
+from direct.nn.conv.modulated import (
+ ModConv2dBias,
+ ModConv3d,
+ ModConvActivation,
+ ModConvType,
+ ModulationParams,
+ mod_conv2d,
+)
from direct.nn.get_nn_model_config import ModelName, _get_model_config, _get_relu_activation
from direct.nn.types import ActivationType, InitType
from direct.nn.unet.unet_3d import NormUnetModel3d, UnetModel3d
@@ -41,13 +52,13 @@
class LagrangeMultipliersInitializer(nn.Module):
- """A convolutional neural network model that initializers the Lagrange multiplier of the :class:`vSHARPNet` [1]_.
+ """A convolutional neural network model that initializes the Lagrange multiplier of the :class:`VSharpNet` [1]_.
More specifically, it produces an initial value for the Lagrange Multiplier based on the zero-filled image:
.. math::
- u^0 = \mathcal{G}_{\psi}(x^0).
+ u^0 = \\mathcal{G}_{\\psi}(x^0).
References
----------
@@ -63,6 +74,13 @@ def __init__(
dilations: tuple[int, ...],
multiscale_depth: int = 1,
activation: ActivationType = ActivationType.PRELU,
+ conv_modulation: ModConvType = ModConvType.NONE,
+ aux_in_features: Optional[int] = None,
+ fc_hidden_features: Optional[tuple[int] | int] = None,
+ fc_groups: int = 1,
+ fc_activation: ModConvActivation = ModConvActivation.SIGMOID,
+ num_weights: Optional[int] = None,
+ modulation_at_input: bool = False,
) -> None:
"""Inits :class:`LagrangeMultipliersInitializer`.
@@ -73,103 +91,149 @@ def __init__(
out_channels : int
Number of output channels.
channels : tuple of ints
- Tuple of integers specifying the number of output channels for each convolutional layer in the network.
+ Number of output channels for each convolutional layer.
dilations : tuple of ints
- Tuple of integers specifying the dilation factor for each convolutional layer in the network.
+ Dilation factor for each convolutional layer.
multiscale_depth : int
Number of multiscale features to include in the output. Default: 1.
activation : ActivationType
- Activation function to use on the output. Default: ActivationType.PRELU.
+ Activation function. Default: ActivationType.PRELU.
+ conv_modulation : ModConvType
+ Modulation type. Default: ModConvType.NONE.
+ aux_in_features : int, optional
+ Number of auxiliary input features.
+ fc_hidden_features : int or tuple of int, optional
+ Hidden features for the modulation MLP.
+ fc_groups : int
+ Groups for the modulation MLP. Default: 1.
+ fc_activation : ModConvActivation
+ Activation for the modulation MLP. Default: ModConvActivation.SIGMOID.
+ num_weights : int, optional
+ Number of weight bases for ModConvType.SUM.
+ modulation_at_input : bool
+ If True, modulation is only applied at the first layers. Default: False.
"""
super().__init__()
- # Define convolutional blocks
+ modulation_params = ModulationParams(
+ modulation=conv_modulation,
+ aux_in_features=aux_in_features,
+ fc_hidden_features=fc_hidden_features,
+ fc_groups=fc_groups,
+ fc_activation=fc_activation,
+ num_weights=num_weights,
+ )
+
self.conv_blocks = nn.ModuleList()
tch = in_channels
- for curr_channels, curr_dilations in zip(channels, dilations):
- block = nn.Sequential(
- nn.ReplicationPad2d(curr_dilations),
- nn.Conv2d(tch, curr_channels, 3, padding=0, dilation=curr_dilations),
+ for i, (curr_channels, curr_dilations) in enumerate(zip(channels, dilations)):
+ block_modulation_params = modulation_params
+ if conv_modulation != ModConvType.NONE:
+ if modulation_at_input and i > 1:
+ block_modulation_params = ModulationParams(
+ modulation=ModConvType.NONE,
+ aux_in_features=aux_in_features,
+ fc_hidden_features=fc_hidden_features,
+ fc_groups=fc_groups,
+ fc_activation=fc_activation,
+ num_weights=num_weights,
+ )
+
+ block = nn.ModuleList(
+ [
+ nn.ReplicationPad2d(curr_dilations),
+ mod_conv2d(
+ tch,
+ curr_channels,
+ kernel_size=3,
+ padding=0,
+ dilation=curr_dilations,
+ bias=(
+ ModConv2dBias.NONE
+ if block_modulation_params.modulation == ModConvType.NONE
+ else ModConv2dBias.LEARNED
+ ),
+ modulation_params=block_modulation_params,
+ ),
+ ]
)
tch = curr_channels
self.conv_blocks.append(block)
- # Define output block
- tch = np.sum(channels[-multiscale_depth:])
- block = nn.Conv2d(tch, out_channels, 1, padding=0)
- self.out_block = nn.Sequential(block)
+ out_modulation_params = modulation_params
+ if (conv_modulation != ModConvType.NONE) and modulation_at_input:
+ out_modulation_params = ModulationParams(
+ modulation=ModConvType.NONE,
+ aux_in_features=aux_in_features,
+ fc_hidden_features=fc_hidden_features,
+ fc_groups=fc_groups,
+ fc_activation=fc_activation,
+ num_weights=num_weights,
+ )
+ tch = np.sum(channels[-multiscale_depth:]).item()
+ self.out_block = mod_conv2d(
+ tch,
+ out_channels,
+ kernel_size=1,
+ padding=0,
+ bias=(
+ ModConv2dBias.NONE if out_modulation_params.modulation == ModConvType.NONE else ModConv2dBias.LEARNED
+ ),
+ modulation_params=out_modulation_params,
+ )
self.multiscale_depth = multiscale_depth
-
self.activation = _get_relu_activation(activation)
+ self.conv_modulation = conv_modulation
- def forward(self, x: torch.Tensor) -> torch.Tensor:
+ def forward(self, x: torch.Tensor, y: Optional[torch.Tensor] = None) -> torch.Tensor:
"""Forward pass of :class:`LagrangeMultipliersInitializer`.
Parameters
----------
x : torch.Tensor
Input tensor of shape (batch_size, in_channels, height, width).
+ y : torch.Tensor, optional
+ Auxiliary tensor of shape (batch_size, aux_in_features). Default: None.
Returns
-------
torch.Tensor
Output tensor of shape (batch_size, out_channels, height, width).
"""
-
features = []
for block in self.conv_blocks:
- x = F.relu(block(x), inplace=True)
+ block_modules = cast(nn.ModuleList, block)
+ x = block_modules[0](x)
+ if self.conv_modulation != ModConvType.NONE:
+ x = F.relu(block_modules[1](x, y), inplace=True)
+ else:
+ x = F.relu(block_modules[1](x), inplace=True)
if self.multiscale_depth > 1:
features.append(x)
if self.multiscale_depth > 1:
x = torch.cat(features[-self.multiscale_depth :], dim=1)
+ if self.conv_modulation != ModConvType.NONE:
+ return self.activation(self.out_block(x, y))
return self.activation(self.out_block(x))
class VSharpNet(nn.Module):
- """
- Variable Splitting Half-quadratic ADMM algorithm for Reconstruction of Parallel MRI [1]_.
-
- Variable Splitting Half Quadratic VSharpNet is a deep learning model that solves
- the augmented Lagrangian derivation of the variable half quadratic splitting problem
- using ADMM (Alternating Direction Method of Multipliers). It is specifically designed
- for solving inverse problems in magnetic resonance imaging (MRI).
-
- The VSharpNet model incorporates an iterative optimization algorithm that consists of
- three steps: z-step, x-step, and u-step. These steps are detailed mathematically as follows:
-
- .. math::
-
- z^{t+1} = \mathrm{argmin}_{z} \\lambda \mathcal{G}(z) + \\frac{\\rho}{2} || x^{t} - z +
- \\frac{u^t}{\\rho} ||_2^2 \\quad \mathrm{[z-step]}
-
- .. math::
-
- x^{t+1} = \mathrm{argmin}_{x} \\frac{1}{2} || \mathcal{A}_{\mathbf{U},\mathbf{S}}(x) - \\tilde{y} ||_2^2 +
- \\frac{\\rho}{2} || x - z^{t+1} + \\frac{u^t}{\\rho} ||_2^2 \\quad \mathrm{[x-step]}
-
- .. math::
-
- u^{t+1} = u^t + \\rho (x^{t+1} - z^{t+1}) \\quad \mathrm{[u-step]}
-
- During the z-step, the model minimizes the augmented Lagrangian function with respect to z, utilizing
- DL-based denoisers. In the x-step, it optimizes x by minimizing the data consistency term through
- unrolling a gradient descent scheme (DC-GD). The u-step involves updating the Lagrange multiplier u.
- These steps are iterated for a specified number of cycles.
-
- The model includes an initializer for Lagrange multipliers.
-
- It also allows for outputting auxiliary steps.
+ """Variable Splitting Half-quadratic ADMM algorithm for Reconstruction of Parallel MRI [1]_.
- :class:`VSharpNet` is tailored for 2D MRI data reconstruction.
+ This model incorporates an iterative optimization algorithm (z-step, x-step, u-step) and
+ supports optional modulated convolutions conditioned on auxiliary data as proposed in [2]_.
References
----------
.. [1] George Yiasemis et al., "VSHARP: Variable Splitting Half-quadratic ADMM Algorithm for Reconstruction
of Inverse Problems" (2023). https://arxiv.org/abs/2309.09954.
+
+ .. [2] Moriakov, N., Yiasemis, G., Sonke, J.-J. & Teuwen, J. (2026). Conditional Learned Reconstruction for
+ Medical Imaging. Proceedings of The 9th International Conference on Medical Imaging with Deep Learning,
+ PMLR 315:754-780. https://proceedings.mlr.press/v315/moriakov26a.html
"""
def __init__(
@@ -186,58 +250,80 @@ def __init__(
initializer_multiscale: int = 1,
initializer_activation: ActivationType = ActivationType.PRELU,
auxiliary_steps: int = 0,
+ conv_modulation: ModConvType = ModConvType.NONE,
+ aux_in_features: Optional[int] = None,
+ fc_hidden_features: Optional[tuple[int] | int] = None,
+ fc_groups: int = 1,
+ fc_activation: ModConvActivation = ModConvActivation.SIGMOID,
+ num_weights: Optional[int] = None,
+ modulation_at_input: bool = False,
**kwargs,
) -> None:
"""Inits :class:`VSharpNet`.
Parameters
----------
- forward_operator : Callable[[tuple[Any, ...]], torch.Tensor]
+ forward_operator : FFTOperator
Forward operator function.
- backward_operator : Callable[[tuple[Any, ...]], torch.Tensor]
+ backward_operator : FFTOperator
Backward operator function.
num_steps : int
Number of steps in the ADMM algorithm.
num_steps_dc_gd : int
- Number of steps in the Data Consistency using Gradient Descent step of ADMM.
- image_init : str
- Image initialization method. Default: 'sense'.
+ Number of gradient descent steps for data consistency.
+ image_init : InitType
+ Image initialization method. Default: InitType.SENSE.
no_parameter_sharing : bool
- Flag indicating whether parameter sharing is enabled in the denoiser blocks.
+ If True, each ADMM step has its own denoiser. Default: True.
image_model_architecture : ModelName
- Image model architecture. Default: ModelName.UNET.
+ Denoiser model architecture. Default: ModelName.UNET.
initializer_channels : tuple[int, ...]
- Tuple of integers specifying the number of output channels for each convolutional layer in the
- Lagrange multiplier initializer. Default: (32, 32, 64, 64).
+ Output channels for the Lagrange initializer layers. Default: (32, 32, 64, 64).
initializer_dilations : tuple[int, ...]
- Tuple of integers specifying the dilation factor for each convolutional layer in the Lagrange multiplier
- initializer. Default: (1, 1, 2, 4).
+ Dilations for the Lagrange initializer layers. Default: (1, 1, 2, 4).
initializer_multiscale : int
- Number of multiscale features to include in the Lagrange multiplier initializer output. Default: 1.
+ Multiscale depth for the initializer. Default: 1.
initializer_activation : ActivationType
- Activation type for the Lagrange multiplier initializer. Default: ActivationType.PRELU.
+ Activation for the initializer output. Default: ActivationType.PRELU.
auxiliary_steps : int
- Number of auxiliary steps to output. Can be -1 or a positive integer lower or equal to `num_steps`.
- If -1, it uses all steps. If I, the last I steps will be used.
- **kwargs: Additional keyword arguments.
- Can be `model_name` or `image_model_` where `` represent parameters of the selected
- image model architecture beyond the standard parameters.
- Depending on the `image_model_architecture` chosen, different kwargs will be applicable.
+ Number of auxiliary output steps. -1 uses all steps. Default: 0.
+ conv_modulation : ModConvType
+ Modulation type for convolutions. Default: ModConvType.NONE.
+ aux_in_features : int, optional
+ Number of auxiliary input features.
+ fc_hidden_features : int or tuple of int, optional
+ Hidden features for the modulation MLP.
+ fc_groups : int
+ Groups for the modulation MLP. Default: 1.
+ fc_activation : ModConvActivation
+ Activation for the modulation MLP. Default: ModConvActivation.SIGMOID.
+ num_weights : int, optional
+ Number of weight bases for ModConvType.SUM.
+ modulation_at_input : bool
+ If True, only the first layers use modulation. Default: False.
+ **kwargs
+ Additional keyword arguments for the image model.
"""
- # pylint: disable=too-many-locals
super().__init__()
for extra_key in kwargs:
- if extra_key != "model_name" and not extra_key.startswith("image_"):
+ if extra_key not in ("model_name", "log_aux", "auxiliary_features") and not extra_key.startswith("image_"):
raise ValueError(f"{type(self).__name__} got key `{extra_key}` which is not supported.")
self.num_steps = num_steps
self.num_steps_dc_gd = num_steps_dc_gd
-
self.no_parameter_sharing = no_parameter_sharing
+ self.conv_modulation = conv_modulation
image_model, image_model_kwargs = _get_model_config(
image_model_architecture,
in_channels=COMPLEX_SIZE * 3,
out_channels=COMPLEX_SIZE,
+ modulation=conv_modulation,
+ aux_in_features=aux_in_features,
+ fc_hidden_features=fc_hidden_features,
+ fc_groups=fc_groups,
+ fc_activation=fc_activation,
+ num_weights=num_weights,
+ modulation_at_input=modulation_at_input,
**{k.replace("image_", ""): v for (k, v) in kwargs.items() if "image_" in k},
)
@@ -252,6 +338,13 @@ def __init__(
dilations=initializer_dilations,
multiscale_depth=initializer_multiscale,
activation=initializer_activation,
+ conv_modulation=conv_modulation,
+ aux_in_features=aux_in_features,
+ fc_hidden_features=fc_hidden_features,
+ fc_groups=fc_groups,
+ fc_activation=fc_activation,
+ num_weights=num_weights,
+ modulation_at_input=modulation_at_input,
)
self.learning_rate_eta = nn.Parameter(torch.ones(num_steps_dc_gd, requires_grad=True))
@@ -265,9 +358,8 @@ def __init__(
if image_init not in [InitType.SENSE, InitType.ZERO_FILLED]:
raise ValueError(
- f"Unknown image_initialization. Expected `InitType.SENSE` or `InitType.ZERO_FILLED`. Got {image_init}."
+ f"Unknown image_initialization. Expected InitType.SENSE or InitType.ZERO_FILLED. Got {image_init}."
)
-
self.image_init = image_init
if not ((auxiliary_steps == -1) or (0 < auxiliary_steps <= num_steps)):
@@ -289,6 +381,7 @@ def forward(
masked_kspace: torch.Tensor,
sensitivity_map: torch.Tensor,
sampling_mask: torch.Tensor,
+ auxiliary_data: Optional[torch.Tensor] = None,
) -> list[torch.Tensor]:
"""Computes forward pass of :class:`VSharpNet`.
@@ -300,6 +393,8 @@ def forward(
Sensitivity map of shape (N, coil, height, width, complex=2).
sampling_mask: torch.Tensor
Sampling mask of shape (N, 1, height, width, 1).
+ auxiliary_data: torch.Tensor, optional
+ Auxiliary tensor of shape (N, aux_in_features). Default: None.
Returns
-------
@@ -318,19 +413,35 @@ def forward(
z = x.clone()
- u = self.initializer(x.permute(0, 3, 1, 2)).permute(0, 2, 3, 1)
+ if self.conv_modulation != ModConvType.NONE:
+ u = self.initializer(x.permute(0, 3, 1, 2), auxiliary_data).permute(0, 2, 3, 1)
+ else:
+ u = self.initializer(x.permute(0, 3, 1, 2)).permute(0, 2, 3, 1)
for admm_step in range(self.num_steps):
- z = self.denoiser_blocks[admm_step if self.no_parameter_sharing else 0](
+ denoiser_input = [
torch.cat(
[z, x, u / self.rho[admm_step]],
dim=self._complex_dim,
).permute(0, 3, 1, 2)
- ).permute(0, 2, 3, 1)
+ ]
+ if auxiliary_data is not None and (
+ self.conv_modulation != ModConvType.NONE
+ or (
+ hasattr(self.denoiser_blocks[0], "norm_type")
+ and self.denoiser_blocks[0].norm_type == NormType.ADAIN
+ )
+ ):
+ denoiser_input.append(auxiliary_data)
+
+ z = self.denoiser_blocks[admm_step if self.no_parameter_sharing else 0](*denoiser_input).permute(0, 2, 3, 1)
for dc_gd_step in range(self.num_steps_dc_gd):
dc = apply_mask(
- self.forward_operator(expand_operator(x, sensitivity_map, self._coil_dim), dim=self._spatial_dims)
+ self.forward_operator(
+ expand_operator(x, sensitivity_map, self._coil_dim),
+ dim=self._spatial_dims,
+ )
- masked_kspace,
sampling_mask,
return_mask=False,
@@ -362,8 +473,15 @@ def __init__(
dilations: tuple[int, ...],
multiscale_depth: int = 1,
activation: ActivationType = ActivationType.PRELU,
+ conv_modulation: ModConvType = ModConvType.NONE,
+ aux_in_features: Optional[int] = None,
+ fc_hidden_features: Optional[tuple[int] | int] = None,
+ fc_groups: int = 1,
+ fc_activation: ModConvActivation = ModConvActivation.SIGMOID,
+ num_weights: Optional[int] = None,
+ modulation_at_input: bool = False,
):
- """Initializes LagrangeMultipliersInitializer3D.
+ """Initializes :class:`LagrangeMultipliersInitializer3D`.
Parameters
----------
@@ -372,70 +490,130 @@ def __init__(
out_channels : int
Number of output channels.
channels : tuple of ints
- Tuple of integers specifying the number of output channels for each convolutional layer in the network.
+ Number of output channels for each convolutional layer.
dilations : tuple of ints
- Tuple of integers specifying the dilation factor for each convolutional layer in the network.
+ Dilation factor for each convolutional layer.
multiscale_depth : int
Number of multiscale features to include in the output. Default: 1.
activation : ActivationType
- Activation function to use on the output. Default: ActivationType.PRELU.
+ Activation function. Default: ActivationType.PRELU.
+ conv_modulation : ModConvType
+ Modulation type. Default: ModConvType.NONE.
+ aux_in_features : int, optional
+ Number of auxiliary input features.
+ fc_hidden_features : int or tuple of int, optional
+ Hidden features for the modulation MLP.
+ fc_groups : int
+ Groups for the modulation MLP. Default: 1.
+ fc_activation : ModConvActivation
+ Activation for the modulation MLP. Default: ModConvActivation.SIGMOID.
+ num_weights : int, optional
+ Number of weight bases for ModConvType.SUM.
+ modulation_at_input : bool
+ If True, modulation is only applied at the first layers. Default: False.
"""
super().__init__()
- # Define convolutional blocks
self.conv_blocks = nn.ModuleList()
tch = in_channels
- for curr_channels, curr_dilations in zip(channels, dilations):
- block = nn.Sequential(
- nn.ReplicationPad3d(curr_dilations),
- nn.Conv3d(tch, curr_channels, 3, padding=0, dilation=curr_dilations),
+ for i, (curr_channels, curr_dilations) in enumerate(zip(channels, dilations)):
+ modulation = conv_modulation
+ if conv_modulation != ModConvType.NONE:
+ if modulation_at_input and i > 1:
+ modulation = ModConvType.NONE
+
+ block = nn.ModuleList(
+ [
+ nn.ReplicationPad3d(curr_dilations),
+ ModConv3d(
+ tch,
+ curr_channels,
+ kernel_size=3,
+ padding=0,
+ dilation=curr_dilations,
+ modulation=modulation,
+ bias=(ModConv2dBias.NONE if modulation == ModConvType.NONE else ModConv2dBias.LEARNED),
+ aux_in_features=aux_in_features,
+ fc_hidden_features=fc_hidden_features,
+ fc_groups=fc_groups,
+ fc_activation=fc_activation,
+ num_weights=num_weights,
+ ),
+ ]
)
tch = curr_channels
self.conv_blocks.append(block)
- # Define output block
- tch = np.sum(channels[-multiscale_depth:])
- block = nn.Conv3d(tch, out_channels, 1, padding=0)
- self.out_block = nn.Sequential(block)
+ modulation = (
+ ModConvType.NONE if ((conv_modulation != ModConvType.NONE) and modulation_at_input) else conv_modulation
+ )
+ tch = np.sum(channels[-multiscale_depth:]).item()
+ self.out_block = ModConv3d(
+ tch,
+ out_channels,
+ kernel_size=1,
+ padding=0,
+ modulation=modulation,
+ bias=(ModConv2dBias.NONE if modulation == ModConvType.NONE else ModConv2dBias.LEARNED),
+ aux_in_features=aux_in_features,
+ fc_hidden_features=fc_hidden_features,
+ fc_groups=fc_groups,
+ fc_activation=fc_activation,
+ num_weights=num_weights,
+ )
self.multiscale_depth = multiscale_depth
self.activation = _get_relu_activation(activation)
+ self.conv_modulation = conv_modulation
- def forward(self, x: torch.Tensor) -> torch.Tensor:
+ def forward(self, x: torch.Tensor, y: Optional[torch.Tensor] = None) -> torch.Tensor:
"""Forward pass of :class:`LagrangeMultipliersInitializer3D`.
Parameters
----------
x : torch.Tensor
Input tensor of shape (batch_size, in_channels, z, x, y).
+ y : torch.Tensor, optional
+ Auxiliary tensor of shape (batch_size, aux_in_features). Default: None.
Returns
-------
torch.Tensor
Output tensor of shape (batch_size, out_channels, z, x, y).
"""
-
features = []
for block in self.conv_blocks:
- x = F.relu(block(x), inplace=True)
+ block_modules = cast(nn.ModuleList, block)
+ x = block_modules[0](x)
+ if self.conv_modulation != ModConvType.NONE:
+ x = F.relu(block_modules[1](x, y), inplace=True)
+ else:
+ x = F.relu(block_modules[1](x), inplace=True)
if self.multiscale_depth > 1:
features.append(x)
if self.multiscale_depth > 1:
x = torch.cat(features[-self.multiscale_depth :], dim=1)
+ if self.conv_modulation != ModConvType.NONE:
+ return self.activation(self.out_block(x, y))
return self.activation(self.out_block(x))
class VSharpNet3D(nn.Module):
- """VharpNet 3D version using 3D U-Nets as denoisers.
+ """VSharpNet 3D version using 3D U-Nets as denoisers.
This is an extension to 3D of :class:`VSharpNet`. For the original paper refer to [1]_.
+ Supports conditional weight modulation as proposed in [2]_.
References
----------
.. [1] George Yiasemis et al., "VSHARP: Variable Splitting Half-quadratic ADMM Algorithm for Reconstruction
of Inverse Problems" (2023). https://arxiv.org/abs/2309.09954.
+
+ .. [2] Moriakov, N., Yiasemis, G., Sonke, J.-J. & Teuwen, J. (2026). Conditional Learned Reconstruction for
+ Medical Imaging. Proceedings of The 9th International Conference on Medical Imaging with Deep Learning,
+ PMLR 315:754-780. https://proceedings.mlr.press/v315/moriakov26a.html
"""
def __init__(
@@ -451,71 +629,106 @@ def __init__(
initializer_multiscale: int = 1,
initializer_activation: ActivationType = ActivationType.PRELU,
auxiliary_steps: int = -1,
+ conv_modulation: ModConvType = ModConvType.NONE,
+ aux_in_features: Optional[int] = None,
+ fc_hidden_features: Optional[tuple[int] | int] = None,
+ fc_groups: int = 1,
+ fc_activation: ModConvActivation = ModConvActivation.SIGMOID,
+ num_weights: Optional[int] = None,
+ modulation_at_input: bool = False,
unet_num_filters: int = 32,
unet_num_pool_layers: int = 4,
unet_dropout: float = 0.0,
unet_norm: bool = False,
+ unet_norm_type: NormType = NormType.INSTANCE,
+ unet_adain_hidden_features: Optional[tuple[int]] = None,
**kwargs,
):
"""Inits :class:`VSharpNet3D`.
Parameters
----------
- forward_operator : Callable[[tuple[Any, ...]], torch.Tensor]
+ forward_operator : FFTOperator
Forward operator function.
- backward_operator : Callable[[tuple[Any, ...]], torch.Tensor]
+ backward_operator : FFTOperator
Backward operator function.
num_steps : int
Number of steps in the ADMM algorithm.
num_steps_dc_gd : int
- Number of steps in the Data Consistency using Gradient Descent step of ADMM.
- image_init : str
- Image initialization method. Default: 'sense'.
+ Number of gradient descent steps for data consistency.
+ image_init : InitType
+ Image initialization method. Default: InitType.SENSE.
no_parameter_sharing : bool
- Flag indicating whether parameter sharing is enabled in the denoiser blocks.
+ If True, each ADMM step has its own denoiser. Default: True.
initializer_channels : tuple[int, ...]
- Tuple of integers specifying the number of output channels for each convolutional layer in the
- Lagrange multiplier initializer. Default: (32, 32, 64, 64).
+ Output channels for the Lagrange initializer layers. Default: (32, 32, 64, 64).
initializer_dilations : tuple[int, ...]
- Tuple of integers specifying the dilation factor for each convolutional layer in the Lagrange multiplier
- initializer. Default: (1, 1, 2, 4).
+ Dilations for the Lagrange initializer layers. Default: (1, 1, 2, 4).
initializer_multiscale : int
- Number of multiscale features to include in the Lagrange multiplier initializer output. Default: 1.
+ Multiscale depth for the initializer. Default: 1.
initializer_activation : ActivationType
- Activation type for the Lagrange multiplier initializer. Default: ActivationType.PReLU.
+ Activation for the initializer output. Default: ActivationType.PRELU.
auxiliary_steps : int
- Number of auxiliary steps to output. Can be -1 or a positive integer lower or equal to `num_steps`.
- If -1, it uses all steps. If I, the last I steps will be used.
+ Number of auxiliary output steps. -1 uses all steps. Default: -1.
+ conv_modulation : ModConvType
+ Modulation type for convolutions. Default: ModConvType.NONE.
+ aux_in_features : int, optional
+ Number of auxiliary input features.
+ fc_hidden_features : int or tuple of int, optional
+ Hidden features for the modulation MLP.
+ fc_groups : int
+ Groups for the modulation MLP. Default: 1.
+ fc_activation : ModConvActivation
+ Activation for the modulation MLP. Default: ModConvActivation.SIGMOID.
+ num_weights : int, optional
+ Number of weight bases for ModConvType.SUM.
+ modulation_at_input : bool
+ If True, only the first layers use modulation. Default: False.
unet_num_filters : int
- U-Net denoisers number of output channels of the first convolutional layer. Default: 32.
+ U-Net first layer filter count. Default: 32.
unet_num_pool_layers : int
- U-Net denoisers number of down-sampling and up-sampling layers (depth). Default: 4.
+ U-Net depth. Default: 4.
unet_dropout : float
- U-Net denoisers dropout probability. Default: 0.0
+ U-Net dropout. Default: 0.0.
unet_norm : bool
- Whether to use normalized U-Net as denoiser or not. Default: False.
- **kwargs: Additional keyword arguments.
- Can be `model_name`.
+ Whether to use normalized U-Net. Default: False.
+ unet_norm_type : NormType
+ Normalization type for U-Net. Default: NormType.INSTANCE.
+ unet_adain_hidden_features : tuple[int], optional
+ Hidden features for AdaIN in U-Net.
+ **kwargs
+ Additional keyword arguments (e.g., model_name, log_aux).
"""
- # pylint: disable=too-many-locals
super().__init__()
for extra_key in kwargs:
- if extra_key != "model_name":
+ if extra_key not in ("model_name", "log_aux", "auxiliary_features"):
raise ValueError(f"{type(self).__name__} got key `{extra_key}` which is not supported.")
self.num_steps = num_steps
self.num_steps_dc_gd = num_steps_dc_gd
-
self.no_parameter_sharing = no_parameter_sharing
+ self.conv_modulation = conv_modulation
+ self.unet_norm_type = unet_norm_type
+
+ unet = UnetModel3d if not unet_norm else NormUnetModel3d
self.denoiser_blocks = nn.ModuleList()
for _ in range(num_steps if self.no_parameter_sharing else 1):
self.denoiser_blocks.append(
- (UnetModel3d if not unet_norm else NormUnetModel3d)(
+ unet(
in_channels=COMPLEX_SIZE * 3,
out_channels=COMPLEX_SIZE,
num_filters=unet_num_filters,
num_pool_layers=unet_num_pool_layers,
dropout_probability=unet_dropout,
+ modulation=conv_modulation,
+ aux_in_features=aux_in_features,
+ fc_hidden_features=fc_hidden_features,
+ fc_groups=fc_groups,
+ fc_activation=fc_activation,
+ num_weights=num_weights,
+ modulation_at_input=modulation_at_input,
+ norm_type=unet_norm_type,
+ adain_hidden_features=unet_adain_hidden_features,
)
)
@@ -526,6 +739,13 @@ def __init__(
dilations=initializer_dilations,
multiscale_depth=initializer_multiscale,
activation=initializer_activation,
+ conv_modulation=conv_modulation,
+ aux_in_features=aux_in_features,
+ fc_hidden_features=fc_hidden_features,
+ fc_groups=fc_groups,
+ fc_activation=fc_activation,
+ num_weights=num_weights,
+ modulation_at_input=modulation_at_input,
)
self.learning_rate_eta = nn.Parameter(torch.ones(num_steps_dc_gd, requires_grad=True))
@@ -537,9 +757,10 @@ def __init__(
self.forward_operator = forward_operator
self.backward_operator = backward_operator
- if image_init not in ["sense", "zero_filled"]:
- raise ValueError(f"Unknown image_initialization. Expected 'sense' or 'zero_filled'. Got {image_init}.")
-
+ if image_init not in [InitType.SENSE, InitType.ZERO_FILLED]:
+ raise ValueError(
+ f"Unknown image_initialization. Expected InitType.SENSE or InitType.ZERO_FILLED. Got {image_init}."
+ )
self.image_init = image_init
if not (auxiliary_steps == -1 or 0 < auxiliary_steps <= num_steps):
@@ -561,22 +782,25 @@ def forward(
masked_kspace: torch.Tensor,
sensitivity_map: torch.Tensor,
sampling_mask: torch.Tensor,
+ auxiliary_data: Optional[torch.Tensor] = None,
) -> list[torch.Tensor]:
"""Computes forward pass of :class:`VSharpNet3D`.
Parameters
----------
- masked_kspace : torch.Tensor
+ masked_kspace: torch.Tensor
Masked k-space of shape (N, coil, slice, height, width, complex=2).
- sensitivity_map : torch.Tensor
+ sensitivity_map: torch.Tensor
Sensitivity map of shape (N, coil, slice, height, width, complex=2).
- sampling_mask : torch.Tensor
+ sampling_mask: torch.Tensor
Sampling mask of shape (N, 1, 1 or slice, height, width, 1).
+ auxiliary_data: torch.Tensor, optional
+ Auxiliary tensor of shape (N, aux_in_features). Default: None.
Returns
-------
out : list of torch.Tensors
- List of output images each of shape (N, slice, height, width, complex=2).
+ List of output images of shape (N, slice, height, width, complex=2).
"""
out = []
if self.image_init == InitType.SENSE:
@@ -590,19 +814,33 @@ def forward(
z = x.clone()
- u = self.initializer(x.permute(0, 4, 1, 2, 3)).permute(0, 2, 3, 4, 1)
+ if self.conv_modulation != ModConvType.NONE:
+ u = self.initializer(x.permute(0, 4, 1, 2, 3), auxiliary_data).permute(0, 2, 3, 4, 1)
+ else:
+ u = self.initializer(x.permute(0, 4, 1, 2, 3)).permute(0, 2, 3, 4, 1)
for admm_step in range(self.num_steps):
- z = self.denoiser_blocks[admm_step if self.no_parameter_sharing else 0](
+ denoiser_input = [
torch.cat(
[z, x, u / self.rho[admm_step]],
dim=self._complex_dim,
).permute(0, 4, 1, 2, 3)
- ).permute(0, 2, 3, 4, 1)
+ ]
+ if auxiliary_data is not None and (
+ self.conv_modulation != ModConvType.NONE or self.unet_norm_type == NormType.ADAIN
+ ):
+ denoiser_input.append(auxiliary_data)
+
+ z = self.denoiser_blocks[admm_step if self.no_parameter_sharing else 0](*denoiser_input).permute(
+ 0, 2, 3, 4, 1
+ )
for dc_gd_step in range(self.num_steps_dc_gd):
dc = apply_mask(
- self.forward_operator(expand_operator(x, sensitivity_map, self._coil_dim), dim=self._spatial_dims)
+ self.forward_operator(
+ expand_operator(x, sensitivity_map, self._coil_dim),
+ dim=self._spatial_dims,
+ )
- masked_kspace,
sampling_mask,
return_mask=False,
diff --git a/direct/nn/vsharp/vsharp_engine.py b/direct/nn/vsharp/vsharp_engine.py
index 76a9ff87..27421041 100644
--- a/direct/nn/vsharp/vsharp_engine.py
+++ b/direct/nn/vsharp/vsharp_engine.py
@@ -119,6 +119,8 @@ def _do_iteration(
output_kspace: TensorOrNone
with autocast("cuda", enabled=self.mixed_precision):
+ self._attach_auxiliary_data(data)
+
output_images, output_kspace = self.forward_function(data)
output_images = [T.modulus_if_complex(_, complex_axis=self._complex_dim) for _ in output_images]
loss_dict = {k: torch.tensor([0.0], dtype=data["target"].dtype).to(self.device) for k in loss_fns.keys()}
@@ -126,7 +128,12 @@ def _do_iteration(
auxiliary_loss_weights = torch.logspace(-1, 0, steps=len(output_images)).to(output_images[0])
for i, output_image in enumerate(output_images):
loss_dict = self.compute_loss_on_data(
- loss_dict, loss_fns, data, output_image, None, auxiliary_loss_weights[i]
+ loss_dict,
+ loss_fns,
+ data,
+ output_image,
+ None,
+ auxiliary_loss_weights[i],
)
# Compute loss on k-space
loss_dict = self.compute_loss_on_data(loss_dict, loss_fns, data, None, output_kspace)
@@ -152,6 +159,7 @@ def forward_function(self, data: dict[str, Any]) -> tuple[torch.Tensor, None]:
masked_kspace=data["masked_kspace"],
sampling_mask=data["sampling_mask"],
sensitivity_map=data["sensitivity_map"],
+ auxiliary_data=self.auxiliary_data_from(data),
) # shape (batch, height, width, complex[=2])
output_image = output_images[-1]
@@ -245,6 +253,8 @@ def _do_iteration(
output_kspace: TensorOrNone
with autocast("cuda", enabled=self.mixed_precision):
+ self._attach_auxiliary_data(data)
+
output_images, output_kspace = self.forward_function(data)
output_images = [T.modulus_if_complex(_, complex_axis=self._complex_dim) for _ in output_images]
loss_dict = {k: torch.tensor([0.0], dtype=data["target"].dtype).to(self.device) for k in loss_fns.keys()}
@@ -252,7 +262,12 @@ def _do_iteration(
auxiliary_loss_weights = torch.logspace(-1, 0, steps=len(output_images)).to(output_images[0])
for i, output_image in enumerate(output_images):
loss_dict = self.compute_loss_on_data(
- loss_dict, loss_fns, data, output_image, None, auxiliary_loss_weights[i]
+ loss_dict,
+ loss_fns,
+ data,
+ output_image,
+ None,
+ auxiliary_loss_weights[i],
)
# Compute loss on k-space
loss_dict = self.compute_loss_on_data(loss_dict, loss_fns, data, None, output_kspace)
@@ -278,6 +293,7 @@ def forward_function(self, data: dict[str, Any]) -> tuple[torch.Tensor, torch.Te
masked_kspace=data["masked_kspace"],
sampling_mask=data["sampling_mask"],
sensitivity_map=data["sensitivity_map"],
+ auxiliary_data=self.auxiliary_data_from(data),
) # shape (batch, height, width, complex[=2])
output_image = output_images[-1]
@@ -417,6 +433,7 @@ def _do_iteration(
# Move data to device
data = dict_to_device(data, self.device)
+ self._attach_auxiliary_data(data)
# Get the k-space and mask which differ during training and inference for SSL
if self.model.training:
@@ -437,6 +454,7 @@ def _do_iteration(
masked_kspace=kspace,
sampling_mask=mask,
sensitivity_map=data["sensitivity_map"],
+ auxiliary_data=self.auxiliary_data_from(data),
)
if self.model.training:
@@ -458,10 +476,20 @@ def _do_iteration(
# Compute k-space loss per auxiliary step
loss_dict = self.compute_loss_on_data(
- loss_dict, loss_fns, data, None, output_kspace, auxiliary_loss_weights[i]
+ loss_dict,
+ loss_fns,
+ data,
+ None,
+ output_kspace,
+ auxiliary_loss_weights[i],
)
regularizer_dict = self.compute_loss_on_data(
- regularizer_dict, regularizer_fns, data, None, output_kspace, auxiliary_loss_weights[i]
+ regularizer_dict,
+ regularizer_fns,
+ data,
+ None,
+ output_kspace,
+ auxiliary_loss_weights[i],
)
# SENSE reconstruction
@@ -475,10 +503,20 @@ def _do_iteration(
# Compute image loss per auxiliary step
loss_dict = self.compute_loss_on_data(
- loss_dict, loss_fns, data, output_images[i], None, auxiliary_loss_weights[i]
+ loss_dict,
+ loss_fns,
+ data,
+ output_images[i],
+ None,
+ auxiliary_loss_weights[i],
)
regularizer_dict = self.compute_loss_on_data(
- regularizer_dict, regularizer_fns, data, output_images[i], None, auxiliary_loss_weights[i]
+ regularizer_dict,
+ regularizer_fns,
+ data,
+ output_images[i],
+ None,
+ auxiliary_loss_weights[i],
)
loss = sum(loss_dict.values()) + sum(regularizer_dict.values()) # type: ignore
@@ -631,6 +669,7 @@ def _do_iteration(
# Move data to device
data = dict_to_device(data, self.device)
+ self._attach_auxiliary_data(data)
# Get a boolean indicating if the sample is for SSL training
# This will expect the input data to contain the keys "input_kspace" and "input_sampling_mask" if SSL training
@@ -656,6 +695,7 @@ def _do_iteration(
masked_kspace=kspace,
sampling_mask=mask,
sensitivity_map=data["sensitivity_map"],
+ auxiliary_data=self.auxiliary_data_from(data),
)
if self.model.training:
@@ -674,14 +714,28 @@ def _do_iteration(
)
if is_ssl:
# Project predicted k-space onto target k-space if SSL
- output_kspace = T.apply_mask(output_kspace, data["target_sampling_mask"], return_mask=False)
+ output_kspace = T.apply_mask(
+ output_kspace,
+ data["target_sampling_mask"],
+ return_mask=False,
+ )
# Compute k-space loss per auxiliary step
loss_dict = self.compute_loss_on_data(
- loss_dict, loss_fns, data, None, output_kspace, auxiliary_loss_weights[i]
+ loss_dict,
+ loss_fns,
+ data,
+ None,
+ output_kspace,
+ auxiliary_loss_weights[i],
)
regularizer_dict = self.compute_loss_on_data(
- regularizer_dict, regularizer_fns, data, None, output_kspace, auxiliary_loss_weights[i]
+ regularizer_dict,
+ regularizer_fns,
+ data,
+ None,
+ output_kspace,
+ auxiliary_loss_weights[i],
)
# SENSE reconstruction if SSL else modulus if supervised
@@ -697,10 +751,20 @@ def _do_iteration(
# Compute image loss per auxiliary step
loss_dict = self.compute_loss_on_data(
- loss_dict, loss_fns, data, output_images[i], None, auxiliary_loss_weights[i]
+ loss_dict,
+ loss_fns,
+ data,
+ output_images[i],
+ None,
+ auxiliary_loss_weights[i],
)
regularizer_dict = self.compute_loss_on_data(
- regularizer_dict, regularizer_fns, data, output_images[i], None, auxiliary_loss_weights[i]
+ regularizer_dict,
+ regularizer_fns,
+ data,
+ output_images[i],
+ None,
+ auxiliary_loss_weights[i],
)
loss = sum(loss_dict.values()) + sum(regularizer_dict.values()) # type: ignore
diff --git a/direct/nn/xpdnet/config.py b/direct/nn/xpdnet/config.py
index b6cc7d6a..5971a6ce 100644
--- a/direct/nn/xpdnet/config.py
+++ b/direct/nn/xpdnet/config.py
@@ -12,8 +12,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from dataclasses import dataclass
+from typing import Optional
from direct.config.defaults import ModelConfig
+from direct.nn.conv.modulated import ModConvActivation, ModConvType
@dataclass
@@ -34,3 +36,11 @@ class XPDNetConfig(ModelConfig):
mwcnn_bias: bool = True
mwcnn_batchnorm: bool = False
normalize: bool = False
+ conv_modulation: ModConvType = ModConvType.NONE
+ aux_in_features: Optional[int] = None
+ auxiliary_features: Optional[tuple[str, ...]] = None
+ log_aux: bool = False
+ fc_hidden_features: Optional[tuple[int, ...]] = None
+ fc_groups: int = 1
+ fc_activation: ModConvActivation = ModConvActivation.SIGMOID
+ num_weights: Optional[int] = None
diff --git a/direct/nn/xpdnet/xpdnet.py b/direct/nn/xpdnet/xpdnet.py
index 61fd06f4..93d16b4e 100644
--- a/direct/nn/xpdnet/xpdnet.py
+++ b/direct/nn/xpdnet/xpdnet.py
@@ -11,11 +11,15 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import annotations
+
from typing import Optional
+import torch
import torch.nn as nn
from direct.nn.conv.conv import Conv2d
+from direct.nn.conv.modulated import ModConvActivation, ModConvType, ModulationParams
from direct.nn.crossdomain.crossdomain import CrossDomainNetwork
from direct.nn.crossdomain.multicoil import MultiCoil
from direct.nn.didn.didn import DIDN
@@ -23,6 +27,28 @@
from direct.types import FFTOperator
+class XPDNetPrimalBlock(nn.Module):
+ """Primal image block: MWCNN feature extractor followed by a channel projection."""
+
+ def __init__(
+ self,
+ mwcnn: MWCNN,
+ out_conv: nn.Conv2d,
+ conv_modulation: ModConvType = ModConvType.NONE,
+ ) -> None:
+ super().__init__()
+ self.mwcnn = mwcnn
+ self.out_conv = out_conv
+ self.conv_modulation = conv_modulation
+
+ def forward(self, x: torch.Tensor, y: Optional[torch.Tensor] = None) -> torch.Tensor:
+ if self.conv_modulation != ModConvType.NONE:
+ x = self.mwcnn(x, y)
+ else:
+ x = self.mwcnn(x)
+ return self.out_conv(x)
+
+
class XPDNet(CrossDomainNetwork):
"""XPDNet as implemented in [1]_.
@@ -43,6 +69,12 @@ def __init__(
image_model_architecture: str = "MWCNN",
kspace_model_architecture: Optional[str] = None,
normalize: bool = False,
+ conv_modulation: ModConvType = ModConvType.NONE,
+ aux_in_features: Optional[int] = None,
+ fc_hidden_features: Optional[tuple[int] | int] = None,
+ fc_groups: int = 1,
+ fc_activation: ModConvActivation = ModConvActivation.SIGMOID,
+ num_weights: Optional[int] = None,
**kwargs,
):
"""Inits :class:`XPDNet`.
@@ -67,9 +99,30 @@ def __init__(
Dual-kspace model architecture. Currently only implemented for CONV and DIDN.
normalize: bool
Normalize input. Default: False.
+ conv_modulation : ModConvType
+ Modulation type for convolutional sub-networks.
+ aux_in_features : int, optional
+ Number of auxiliary conditioning features.
+ fc_hidden_features : int or tuple of int, optional
+ Hidden features in the modulation MLP.
+ fc_groups : int
+ Modulation MLP groups. Default: 1.
+ fc_activation : ModConvActivation
+ Modulation MLP activation. Default: SIGMOID.
+ num_weights : int, optional
+ Number of weight bases for SUM modulation.
kwargs: dict
Keyword arguments for model architectures.
"""
+ modulation_params = ModulationParams(
+ modulation=conv_modulation,
+ aux_in_features=aux_in_features,
+ fc_hidden_features=fc_hidden_features,
+ fc_groups=fc_groups,
+ fc_activation=fc_activation,
+ num_weights=num_weights,
+ )
+
if use_primal_only:
kspace_model_list = None
num_dual = 1
@@ -83,6 +136,7 @@ def __init__(
kwargs.get("dual_conv_hidden_channels", 16),
kwargs.get("dual_conv_n_convs", 4),
batchnorm=kwargs.get("dual_conv_batchnorm", False),
+ modulation_params=modulation_params,
)
)
for _ in range(num_iter)
@@ -98,6 +152,7 @@ def __init__(
hidden_channels=kwargs.get("dual_didn_hidden_channels", 16),
num_dubs=kwargs.get("dual_didn_num_dubs", 6),
num_convs_recon=kwargs.get("dual_didn_num_convs_recon", 9),
+ modulation_params=modulation_params,
)
)
for _ in range(num_iter)
@@ -112,15 +167,22 @@ def __init__(
if image_model_architecture == "MWCNN":
image_model_list = nn.ModuleList(
[
- nn.Sequential(
+ XPDNetPrimalBlock(
MWCNN(
input_channels=2 * (num_primal + num_dual),
first_conv_hidden_channels=kwargs.get("mwcnn_hidden_channels", 32),
num_scales=kwargs.get("mwcnn_num_scales", 4),
bias=kwargs.get("mwcnn_bias", False),
batchnorm=kwargs.get("mwcnn_batchnorm", False),
+ modulation_params=modulation_params,
+ ),
+ nn.Conv2d(
+ 2 * (num_primal + num_dual),
+ 2 * num_primal,
+ kernel_size=3,
+ padding=1,
),
- nn.Conv2d(2 * (num_primal + num_dual), 2 * num_primal, kernel_size=3, padding=1),
+ conv_modulation=conv_modulation,
)
for _ in range(num_iter)
]
@@ -139,4 +201,5 @@ def __init__(
image_buffer_size=num_primal,
kspace_buffer_size=num_dual,
normalize_image=normalize,
+ conv_modulation=conv_modulation,
)
diff --git a/direct/nn/xpdnet/xpdnet_engine.py b/direct/nn/xpdnet/xpdnet_engine.py
index 97a9917a..289efc8a 100644
--- a/direct/nn/xpdnet/xpdnet_engine.py
+++ b/direct/nn/xpdnet/xpdnet_engine.py
@@ -52,6 +52,7 @@ def forward_function(self, data: Dict[str, Any]) -> Tuple[torch.Tensor, None]:
sampling_mask=data["sampling_mask"],
sensitivity_map=data["sensitivity_map"],
scaling_factor=data["scaling_factor"],
+ auxiliary_data=self.auxiliary_data_from(data),
) # shape (batch, height, width, complex[=2])
output_kspace = None
diff --git a/direct/types.py b/direct/types.py
index 304e78df..788e0d7e 100644
--- a/direct/types.py
+++ b/direct/types.py
@@ -28,6 +28,7 @@
DictOrDictConfig = Union[dict, DictConfig]
Number = Union[float, int]
+IntOrTuple = Union[int, tuple]
PathOrString = Union[pathlib.Path, str]
FileOrUrl = PathOrString
HasStateDict = Union[nn.Module, torch.optim.Optimizer, torch.optim.lr_scheduler._LRScheduler, GradScaler]
@@ -101,6 +102,12 @@ class MaskFuncMode(DirectEnum):
MULTISLICE = "multislice"
+class RangeMode(DirectEnum):
+ DISCRETE = "discrete"
+ UNIFORM = "uniform"
+ LINEAR = "linear"
+
+
class IntegerListOrTupleStringMeta(type):
"""Metaclass for the :class:`IntegerListOrTupleString` class.
diff --git a/direct/utils/dataset.py b/direct/utils/dataset.py
index 1c18fc00..f9353559 100644
--- a/direct/utils/dataset.py
+++ b/direct/utils/dataset.py
@@ -11,13 +11,76 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import annotations
+
import pathlib
+import re
import urllib.parse
-from typing import List
+from typing import Any, List, Optional, Union
+
+import numpy as np
from direct.types import PathOrString
from direct.utils.io import check_is_valid_url, read_list
+# Matches tokens such as ``15T``, ``30T``, ``055T``, ``70T`` in filenames
+# (e.g. CMRxRecon2025: ``..._UIH_15T_umr670_...``).
+_FIELD_STRENGTH_TOKEN = re.compile(r"(?:^|[^0-9A-Za-z])(\d+)T(?:[^0-9A-Za-z]|$)", re.IGNORECASE)
+
+
+def parse_field_strength_tesla(filename: Union[str, pathlib.Path]) -> Optional[float]:
+ """Parse magnetic field strength in Tesla from an ``XT`` token in ``filename``.
+
+ Digits before ``T`` are interpreted with a decimal point before the last digit when
+ there is more than one digit:
+
+ * ``7T`` / ``70T`` → 7.0 T
+ * ``15T`` → 1.5 T
+ * ``30T`` → 3.0 T
+ * ``055T`` → 0.55 T
+
+ Parameters
+ ----------
+ filename : str or pathlib.Path
+ Path or basename that may contain a field-strength token.
+
+ Returns
+ -------
+ float or None
+ Field strength in Tesla, or ``None`` if no ``XT`` token is present.
+ """
+ name = pathlib.Path(filename).name
+ matches = _FIELD_STRENGTH_TOKEN.findall(name)
+ if not matches:
+ return None
+
+ digits = matches[-1]
+ if not digits or int(digits) == 0:
+ return None
+
+ return float(int(digits)) / float(10 ** max(len(digits) - 1, 0))
+
+
+def maybe_attach_field_strength(sample: dict[str, Any]) -> dict[str, Any]:
+ """Attach ``field_strength`` to ``sample`` when the filename encodes an ``XT`` token.
+
+ If no token is found, ``sample`` is left unchanged (key is not added).
+
+ Parameters
+ ----------
+ sample : dict[str, Any]
+ Dataset sample; expects optional ``filename`` key.
+
+ Returns
+ -------
+ dict[str, Any]
+ The same sample dict, possibly with ``field_strength`` as a length-1 ``np.ndarray``.
+ """
+ value = parse_field_strength_tesla(sample.get("filename", ""))
+ if value is not None:
+ sample["field_strength"] = np.array([value], dtype=np.float64)
+ return sample
+
def get_filenames_for_datasets_from_config(cfg, files_root: PathOrString, data_root: PathOrString):
"""Given a configuration object it returns a list of filenames.
diff --git a/direct/utils/distributions.py b/direct/utils/distributions.py
new file mode 100644
index 00000000..34912df1
--- /dev/null
+++ b/direct/utils/distributions.py
@@ -0,0 +1,56 @@
+# Copyright 2026 AI for Oncology Research Group. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Probability distributions for sampling."""
+
+from __future__ import annotations
+
+import numpy as np
+
+
+def triangular_distribution(
+ a: float,
+ b: float,
+ n: int,
+ rng: np.random.RandomState | None = None,
+) -> np.ndarray:
+ """Sample from a triangular distribution on ``[a, b]`` with mode at ``b``.
+
+ The probability density is proportional to ``x`` on ``[a, b]``, so values near
+ the upper endpoint are sampled more often than values near the lower endpoint.
+
+ Parameters
+ ----------
+ a : float
+ Left endpoint of the distribution.
+ b : float
+ Right endpoint of the distribution.
+ n : int
+ Number of samples to draw.
+ rng : np.random.RandomState, optional
+ Random number generator. Default: None.
+
+ Returns
+ -------
+ np.ndarray
+ Array of ``n`` samples.
+ """
+
+ def inverse_cdf(u: np.ndarray) -> np.ndarray:
+ return np.sqrt(u * (b**2 - a**2) + a**2)
+
+ if rng is None:
+ rng = np.random.RandomState()
+
+ uniform_samples = rng.uniform(0, 1, n)
+ return inverse_cdf(uniform_samples)
diff --git a/projects/modulated_convolution/README.rst b/projects/modulated_convolution/README.rst
new file mode 100644
index 00000000..da9de0f4
--- /dev/null
+++ b/projects/modulated_convolution/README.rst
@@ -0,0 +1,370 @@
+=================================================================================
+Modulated Convolution for Conditional MRI Reconstruction
+=================================================================================
+
+This folder contains configuration files and a short tutorial for **modulated
+convolutions** in DIRECT, as introduced in:
+
+`Conditional Learned Reconstruction for Medical Imaging
+`__
+(Moriakov et al., MIDL 2026, PMLR 315:754–780).
+
+`OpenReview submission (PDF) `__
+
+Modulated convolutions let a reconstruction network adapt its convolutional
+filters to **acquisition metadata** (for example acceleration factor and ACS
+fraction) via a small MLP. The same backbone can therefore be trained once and
+conditioned at inference time on the actual undersampling pattern.
+
+Paper overview
+==============
+
+The paper proposes **conditional learned iterative schemes**: convolutional
+weights in unrolled reconstruction networks are modulated by learned functions of
+acquisition parameters (Section 3, `OpenReview PDF
+`__). This addresses variability in
+protocol-dependent settings—MRI acceleration and ACS fraction, CT tube current
+and projection count—that standard learned iterative models typically do not
+model explicitly (Introduction, Section 1).
+
+.. figure:: figures/modconv_architecture.png
+ :alt: Modulated convolution schematic (paper Figure 1)
+ :width: 90%
+
+ **Figure 1** (`paper `__): standard
+ convolution vs. modulated convolution. An auxiliary vector
+ :math:`\mathbf{z}` (acquisition characteristics) drives MLPs
+ :math:`f_\theta, g_\psi` that produce modulating weights and bias; the
+ modulated kernel is applied to the input feature maps (Section 3.1, Eq. 6).
+
+.. figure:: figures/modulation_mlp.png
+ :alt: Modulator MLP architecture (paper Figure 2)
+ :width: 85%
+
+ **Figure 2** (`paper `__):
+ modulator architecture. The auxiliary variable
+ :math:`\mathbf{z} \in \mathbb{R}^N` passes through linear layers with PReLU
+ activations; a final Softplus yields modulating weights
+ :math:`\mathbf{W} \in \mathbb{R}^{M}` (Section 3.1).
+
+For accelerated MRI, the paper uses (Section 4.3.3, Eq. 7):
+
+.. math::
+
+ \mathbf{z} = \log([R,\; 100 \cdot r_{\mathrm{acs}}]) \in \mathbb{R}^2
+
+where :math:`R` is the acceleration factor and :math:`r_{\mathrm{acs}}` the ACS
+fraction. Training samples :math:`R \in [4, 16]` with triangular sampling
+toward higher acceleration (Section 4.3.2); equispaced masks are used in the
+MRI experiments (Section 4.3.2). Modulated convolutions consistently outperform
+non-modulated baselines on fastMRI prostate and knee (Section 4.3.5, Table 1).
+
+.. figure:: figures/knee_reconstruction.png
+ :alt: Knee reconstruction example at 16x acceleration (paper Figure 3)
+ :width: 95%
+
+ **Figure 3** (`paper `__):
+ qualitative knee example at :math:`R = 16`, :math:`r_{\mathrm{acs}} = 0.02`.
+ Modulated models recover sharper detail than the non-modulated baseline
+ (Section 4.3.5).
+
+DIRECT maps the paper notation to config fields as follows:
+
++---------------------------+-----------------------------------------------+
+| Paper | DIRECT config / batch |
++===========================+===============================================+
+| :math:`\mathbf{z}` | ``auxiliary_data`` from ``prepare_auxiliary_ |
+| | data()`` |
++---------------------------+-----------------------------------------------+
+| :math:`R` | ``acceleration`` (batch key) |
++---------------------------+-----------------------------------------------+
+| :math:`r_{\mathrm{acs}}` | ``center_fraction`` (batch key) |
++---------------------------+-----------------------------------------------+
+| ``log_aux: true`` | applies :math:`\log` as in Eq. 7 |
++---------------------------+-----------------------------------------------+
+| MOD S / M / L MLP sizes | ``fc_hidden_features: [32, 8]`` etc. |
+| (Section 4.1) | |
++---------------------------+-----------------------------------------------+
+| ``FEATURES`` modulation | element-wise weight modulation (Section 3.1, |
+| | Appendix B.1) |
++---------------------------+-----------------------------------------------+
+
+Overview
+========
+
+Standard convolutions use fixed weights. A modulated convolution keeps a base
+weight tensor and multiplies it element-wise by an MLP output derived from an
+auxiliary vector ``y``:
+
+.. code-block:: text
+
+ x ──► ModConv2d(·, y) ──► output
+ ▲
+ │
+ y = [log(acceleration), log(100 * center_fraction), ...]
+
+In DIRECT this is implemented as a drop-in replacement for ``torch.nn.Conv2d`` /
+``Conv3d`` inside U-Nets, VarNets, vSHARP denoisers, and other unrolled models.
+
+Code layout
+===========
+
+All modulated-convolution code lives under ``direct/nn/conv/modulated/``:
+
+``modulated_conv.py``
+ Core layers: ``ModConv2d``, ``ModConv3d``, transposed variants, and enums
+ ``ModConvType``, ``ModConvActivation``, ``ModConv2dBias``.
+
+``auxiliary_data.py``
+ Registry-based auxiliary feature pipeline:
+
+ * ``prepare_auxiliary_data(data, cfg)`` — builds ``(batch, aux_in_features)``
+ from the batch dict.
+ * ``register_auxiliary_feature()`` — add custom conditioning channels.
+ * Default features (in order): ``acceleration``, ``center_fraction``,
+ ``field_strength`` (when present in the batch; typically parsed from an
+ ``XT`` token in the filename, e.g. CMRxRecon2025 ``15T`` / ``30T``).
+
+``__init__.py``
+ Public re-exports used throughout the codebase.
+
+Related integration points:
+
+* **UNet backbone** — ``direct/nn/unet/unet_2d.py`` swaps ``Conv2d`` blocks for
+ ``ModConv2d`` when ``conv_modulation != NONE``.
+* **vSHARP** — ``direct/nn/vsharp/vsharp.py`` passes ``auxiliary_data`` to the
+ initializer and image denoiser U-Net (Section 3.3).
+* **VarNet** — ``direct/nn/varnet/varnet.py`` modulates the regularizer U-Net.
+* **Other unrolled models** — KIKINet, JointICNet, IterDualNet, LPD, Conv2d,
+ DIDN, MWCNN (see their ``config.py`` files for ``conv_modulation``).
+* **Engines** — ``MRIModelEngine._attach_auxiliary_data()`` runs in every supervised,
+ SSL, and JSSL iteration. Conv-based engines pass ``auxiliary_data`` into their model
+ ``forward``.
+* **Data pipeline** — ``direct/data/mri_transforms.py`` enables
+ ``return_acceleration`` on mask functions; sampled values land in the batch as
+ ``acceleration`` and ``center_fraction``.
+* **Triangular acceleration sampling** — set ``range_mode: LINEAR`` in masking
+ config to match Section 4.3.2 (see ``direct/common/subsample.py``).
+
+Modulation types
+================
+
+Set ``conv_modulation`` on model configs to one of:
+
+``NONE``
+ Standard convolution (default). Auxiliary inputs are ignored.
+
+``FEATURES``
+ MLP output has the same shape as the convolution weight tensor; element-wise
+ product with the base weights. Most experiments in this folder use this mode
+ (Section 3.1; see also Appendix B.1 for other variants).
+
+``FULL``
+ MLP output modulates the full weight tensor (one scalar factor per weight).
+
+``PARTIAL_IN`` / ``PARTIAL_OUT``
+ Modulate along input or output channel dimension respectively.
+
+``SUM``
+ Learn ``num_weights`` weight bases; MLP produces mixture coefficients.
+
+Key config fields
+=================
+
+Model section (example from vSHARP):
+
+.. code-block:: yaml
+
+ conv_modulation: FEATURES # ModConvType
+ aux_in_features: 2 # length of y (paper Eq. 7: R + r_acs)
+ auxiliary_features: # optional; default: first N registry keys
+ - acceleration
+ - center_fraction
+ log_aux: true # apply log() as in Eq. 7
+ fc_hidden_features: [32, 8] # MLP hidden layers (MOD S in Section 4.1)
+ fc_activation: SOFTPLUS # SIGMOID or SOFTPLUS (paper: Softplus output)
+ fc_groups: 1 # optional grouped low-rank modulation
+ num_weights: null # only for SUM modulation
+
+Masking section (training with variable acceleration):
+
+.. code-block:: yaml
+
+ masking:
+ name: FastMRIEquispaced
+ accelerations: [4, 16]
+ center_fractions: [0.08, 0.02]
+ range_mode: LINEAR # triangular sampling toward higher R (Sec. 4.3.2)
+
+When ``log_aux: true``, ``center_fraction`` is multiplied by 100 before logging
+(see ``AuxiliaryFeature.log_scale`` in the registry), matching Eq. 7.
+``acceleration`` uses scale 1.
+
+End-to-end data flow
+====================
+
+1. **Dataset transform** — ``CreateSamplingMask`` calls the mask function with
+ ``return_acceleration=True`` and stores ``acceleration`` / ``center_fraction``
+ in the sample dict.
+2. **Collate / device** — tensors move to the training device with the batch.
+3. **Engine** — ``MRIModelEngine._attach_auxiliary_data()`` calls
+ ``prepare_auxiliary_data()`` when modulation is enabled.
+4. **Model forward** — each ``ModConv2d`` layer receives ``(x, auxiliary_data)``.
+
+To add a custom auxiliary channel:
+
+.. code-block:: python
+
+ from direct.nn.conv.modulated import AuxiliaryFeature, register_auxiliary_feature
+
+ register_auxiliary_feature(AuxiliaryFeature("my_feature"))
+ # then set auxiliary_features: [acceleration, my_feature] in the yaml
+
+Configuration files
+===================
+
+Paper experiment configs are in ``projects/modulated_convolution/configs/vsharp/``:
+
+**fastMRI knee** (``configs/vsharp/knee/``, 80k iterations unless noted):
+
++-------------------------------+-----------------------------------------------+
+| Config file | Description |
++===============================+===============================================+
+| ``vsharp_triang.yaml`` | Baseline vSHARP, no modulation |
++-------------------------------+-----------------------------------------------+
+| ``vsharp_modconv_features_ | FEATURES modulation, MLP ``[32, 32]`` (MOD L) |
+| triang.yaml`` | |
++-------------------------------+-----------------------------------------------+
+| ``vsharp_modconv_features_ | FEATURES modulation, MLP ``[32, 8]`` (MOD S) |
+| triang_32_8.yaml`` | |
++-------------------------------+-----------------------------------------------+
+| ``vsharp_modconv_features_ | FEATURES modulation, MLP ``[32, 16]`` (MOD M) |
+| triang_32_16.yaml`` | |
++-------------------------------+-----------------------------------------------+
+| ``vsharp_modconv_features_ | MOD M with ``modulation_at_input: true``, |
+| triang_32_16_mod_inp.yaml`` | 150k iterations |
++-------------------------------+-----------------------------------------------+
+| ``vsharp_modconv_partial_in_ | PARTIAL_IN modulation, MLP ``[32, 32]`` |
+| triang.yaml`` | |
++-------------------------------+-----------------------------------------------+
+| ``vsharp_adain_triang_32_16. | AdaIN baseline, MLP ``[32, 16]``, 150k iters |
+| yaml`` | |
++-------------------------------+-----------------------------------------------+
+
+**fastMRI prostate** (``configs/vsharp/prostate/``, 150k iterations):
+
+Same variants as knee except AdaIN and modulation-at-input configs (knee-only ablations).
+
+Other example configs:
+
+``varnet_prostate_modconv_accel_16_16.yaml``
+ End-to-end VarNet on prostate. Acceleration-only conditioning
+ (``aux_in_features: 1``), MLP ``[16, 16]``.
+
+Training
+========
+
+Create an experiment directory, copy or symlink a config as ``config.yaml`` inside it
+(or pass ``--cfg`` explicitly). Then:
+
+.. code-block:: bash
+
+ direct train /path/to/experiments/my_run \
+ --cfg projects/modulated_convolution/configs/vsharp/knee/vsharp_modconv_features_triang_32_8.yaml \
+ --training-root /path/to/fastmri/knee/ \
+ --validation-root /path/to/fastmri/knee/val/ \
+ --device mps
+
+Prostate (training and validation roots are the same directory):
+
+.. code-block:: bash
+
+ direct train /path/to/experiments/my_run \
+ --cfg projects/modulated_convolution/configs/vsharp/prostate/vsharp_modconv_features_triang_32_16.yaml \
+ --training-root /path/to/fastmri/prostate/ \
+ --validation-root /path/to/fastmri/prostate/ \
+ --device mps
+
+Working knee example (matches the Snellius vSHARP mod-conv checkpoint layout):
+
+.. code-block:: bash
+
+ direct train ./experiments/base_vsharp_modconv_softplus_features_double_MLP_triang_32_8 \
+ --cfg projects/modulated_convolution/configs/vsharp/knee/vsharp_modconv_features_triang_32_8.yaml \
+ --training-root /path/to/fastmri/knee/ \
+ --validation-root /path/to/fastmri/knee/val/ \
+ --device mps
+
+Resume from a checkpoint:
+
+.. code-block:: bash
+
+ direct train ./experiments/base_vsharp_modconv_softplus_features_double_MLP_triang_32_8 \
+ --cfg ./experiments/base_vsharp_modconv_softplus_features_double_MLP_triang_32_8/config.yaml \
+ --training-root /path/to/fastmri/knee/ \
+ --validation-root /path/to/fastmri/knee/val/ \
+ --device mps \
+ --resume
+
+Inference
+=========
+
+.. code-block:: bash
+
+ direct predict ./output \
+ --cfg projects/modulated_convolution/configs/vsharp/knee/vsharp_modconv_features_triang_32_8.yaml \
+ --checkpoint /path/to/model_80000.pt \
+ --data-root /path/to/fastmri/knee/val \
+ --device cuda:0
+
+The ``inference`` block in each yaml fixes the validation acceleration (default
+4×). Edit ``inference.dataset.transforms.masking`` to evaluate other rates.
+
+Tests
+=====
+
+Modulated convolution unit tests:
+
+.. code-block:: bash
+
+ pytest tests/tests_nn/modulated_conv_test.py tests/tests_nn/auxiliary_data_test.py
+
+Subsampling / triangular range tests:
+
+.. code-block:: bash
+
+ pytest tests/tests_common/subsample_test.py -k "linear_range or equispaced"
+
+Citing this work
+================
+
+.. code-block:: BibTeX
+
+ @inproceedings{moriakov2026conditional,
+ title = {Conditional Learned Reconstruction for Medical Imaging},
+ author = {Moriakov, Nikita and Yiasemis, George and Sonke, Jan-Jakob and Teuwen, Jonas},
+ booktitle = {Proceedings of The 9th International Conference on Medical Imaging with Deep Learning},
+ pages = {754--780},
+ year = {2026},
+ volume = {315},
+ series = {Proceedings of Machine Learning Research},
+ publisher = {PMLR},
+ url = {https://proceedings.mlr.press/v315/moriakov26a.html}
+ }
+
+ @article{DIRECTTOOLKIT,
+ doi = {10.21105/joss.04278},
+ url = {https://doi.org/10.21105/joss.04278},
+ year = {2022},
+ publisher = {The Open Journal},
+ volume = {7},
+ number = {73},
+ pages = {4278},
+ author = {George Yiasemis and Nikita Moriakov and Dimitrios Karkalousos and Matthan Caan and Jonas Teuwen},
+ title = {DIRECT: Deep Image REConstruction Toolkit},
+ journal = {Journal of Open Source Software}
+ }
+
+Figures in ``figures/`` are adapted from the paper (`OpenReview PDF
+`__); source diagrams:
+``modconv.pdf`` and ``modulation.pdf``.
diff --git a/projects/modulated_convolution/configs/varnet_prostate_modconv_accel_16_16.yaml b/projects/modulated_convolution/configs/varnet_prostate_modconv_accel_16_16.yaml
new file mode 100644
index 00000000..84f012dd
--- /dev/null
+++ b/projects/modulated_convolution/configs/varnet_prostate_modconv_accel_16_16.yaml
@@ -0,0 +1,164 @@
+model:
+ model_name: varnet.varnet.EndToEndVarNet
+ engine_name: null
+ num_layers: 12
+ regularizer_num_filters: 64
+ regularizer_num_pull_layers: 4
+ regularizer_dropout: 0.0
+ conv_modulation: FEATURES
+ aux_in_features: 1
+ auxiliary_features:
+ - acceleration
+ log_aux: true
+ fc_hidden_features:
+ - 16
+ - 16
+ fc_groups: 1
+ fc_activation: SOFTPLUS
+ num_weights: null
+additional_models:
+ sensitivity_model:
+ model_name: unet.unet_2d.UnetModel2d
+ engine_name: null
+ in_channels: 2
+ out_channels: 2
+ num_filters: 16
+ num_pool_layers: 4
+ dropout_probability: 0.0
+ modulation: NONE
+ aux_in_features: null
+ fc_hidden_features: null
+ fc_groups: 1
+ fc_activation: SIGMOID
+ num_weights: null
+physics:
+ forward_operator: fft2
+ backward_operator: ifft2
+ use_noise_matrix: false
+ noise_matrix_scaling: 1.0
+training:
+ datasets:
+ - name: FastMRI
+ transforms:
+ crop: reconstruction_size
+ estimate_sensitivity_maps: true
+ scaling_key: masked_kspace
+ image_center_crop: false
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 4
+ - 16
+ center_fractions:
+ - 0.08
+ - 0.02
+ scale_percentile: 0.995
+ use_seed: false
+ delete_kspace: false
+ model_checkpoint: null
+ optimizer: Adam
+ lr: 0.002
+ weight_decay: 0.0
+ batch_size: 1
+ lr_step_size: 30000
+ lr_gamma: 0.8
+ lr_warmup_iter: 1000
+ swa_start_iter: null
+ num_iterations: 150000
+ validation_steps: 4000
+ gradient_steps: 1
+ gradient_clipping: 0.0
+ gradient_debug: false
+ loss:
+ crop: header
+ losses:
+ - function: l1_loss
+ multiplier: 1.0
+ - function: ssim_loss
+ multiplier: 1.0
+ - function: hfen_l2_norm_loss
+ multiplier: 1.0
+ - function: hfen_l1_norm_loss
+ multiplier: 1.0
+ - function: kspace_nmae_loss
+ multiplier: 1.0
+ - function: kspace_nmse_loss
+ multiplier: 1.0
+ checkpointer:
+ checkpoint_steps: 4000
+ metrics: []
+ regularizers: []
+validation:
+ datasets:
+ - name: FastMRI
+ transforms:
+ estimate_sensitivity_maps: true
+ scaling_key: masked_kspace
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 4
+ center_fractions:
+ - 0.08
+ scale_percentile: 0.995
+ use_seed: true
+ text_description: 4x
+ - name: FastMRI
+ transforms:
+ estimate_sensitivity_maps: true
+ scaling_key: masked_kspace
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 8
+ center_fractions:
+ - 0.04
+ scale_percentile: 0.995
+ use_seed: true
+ text_description: 8x
+ - name: FastMRI
+ transforms:
+ estimate_sensitivity_maps: true
+ scaling_key: masked_kspace
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 16
+ center_fractions:
+ - 0.02
+ scale_percentile: 0.995
+ use_seed: true
+ text_description: 16x
+ batch_size: 20
+ metrics:
+ - fastmri_psnr
+ - fastmri_ssim
+ - fastmri_nmse
+ regularizers: []
+ crop: header
+inference:
+ dataset:
+ name: FastMRI
+ transforms:
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 4.0
+ center_fractions:
+ - 0.08
+ mode: STATIC
+ cropping:
+ crop: null
+ sensitivity_map_estimation:
+ estimate_sensitivity_maps: true
+ normalization:
+ scaling_key: masked_kspace
+ scale_percentile: 0.995
+ use_seed: true
+ text_description: inference-4x
+ batch_size: 5
+ crop: header
+logging:
+ log_as_image: null
+ tensorboard:
+ num_images: 4
diff --git a/projects/modulated_convolution/configs/vsharp/knee/vsharp_adain_triang_32_16.yaml b/projects/modulated_convolution/configs/vsharp/knee/vsharp_adain_triang_32_16.yaml
new file mode 100644
index 00000000..ccf182d0
--- /dev/null
+++ b/projects/modulated_convolution/configs/vsharp/knee/vsharp_adain_triang_32_16.yaml
@@ -0,0 +1,196 @@
+model:
+ model_name: vsharp.vsharp.VSharpNet
+ num_steps: 12
+ num_steps_dc_gd: 10
+ image_init: SENSE
+ no_parameter_sharing: true
+ auxiliary_steps: -1
+ image_model_architecture: UNET
+ initializer_channels:
+ - 32
+ - 32
+ - 64
+ - 64
+ initializer_dilations:
+ - 1
+ - 1
+ - 2
+ - 4
+ initializer_multiscale: 1
+ initializer_activation: PRELU
+ conv_modulation: NONE
+ aux_in_features: 2
+ log_aux: true
+ fc_hidden_features:
+ - 32
+ - 32
+ fc_groups: 1
+ fc_activation: SOFTPLUS
+ num_weights: null
+ modulation_at_input: false
+ image_resnet_hidden_channels: 128
+ image_resnet_num_blocks: 15
+ image_resnet_batchnorm: true
+ image_resnet_scale: 0.1
+ image_unet_num_filters: 32
+ image_unet_num_pool_layers: 4
+ image_unet_dropout: 0.0
+ image_unet_norm_type: ADAIN
+ image_unet_adain_hidden_features:
+ - 32
+ - 16
+ image_didn_hidden_channels: 16
+ image_didn_num_dubs: 6
+ image_didn_num_convs_recon: 9
+ image_conv_hidden_channels: 64
+ image_conv_n_convs: 15
+ image_conv_activation: ActivationType.RELU
+ image_conv_batchnorm: false
+additional_models:
+ sensitivity_model:
+ model_name: unet.unet_2d.UnetModel2d
+ in_channels: 2
+ out_channels: 2
+ num_filters: 16
+ num_pool_layers: 4
+ dropout_probability: 0.0
+ modulation: NONE
+ aux_in_features: null
+ fc_hidden_features: null
+ fc_groups: 1
+ fc_activation: SIGMOID
+ num_weights: null
+physics:
+ forward_operator: fft2
+ backward_operator: ifft2
+ use_noise_matrix: false
+ noise_matrix_scaling: 1.0
+training:
+ datasets:
+ - name: FastMRI
+ transforms:
+ crop: reconstruction_size
+ estimate_sensitivity_maps: true
+ scaling_key: masked_kspace
+ image_center_crop: false
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 4
+ - 16
+ center_fractions:
+ - 0.08
+ - 0.02
+ range_mode: LINEAR
+ scale_percentile: 0.995
+ use_seed: false
+ delete_kspace: false
+ random_flip_probability: 0.5
+ random_rotation_probability: 0.5
+ model_checkpoint: null
+ optimizer: Adam
+ lr: 0.002
+ weight_decay: 0.0
+ batch_size: 1
+ lr_step_size: 15000
+ lr_gamma: 0.8
+ lr_warmup_iter: 1000
+ swa_start_iter: null
+ num_iterations: 150001
+ validation_steps: 4000
+ gradient_steps: 1
+ gradient_clipping: 0.0
+ gradient_debug: false
+ loss:
+ crop: header
+ losses:
+ - function: l1_loss
+ multiplier: 1.0
+ - function: ssim_loss
+ multiplier: 1.0
+ - function: hfen_l2_norm_loss
+ multiplier: 1.0
+ - function: hfen_l1_norm_loss
+ multiplier: 1.0
+ - function: kspace_nmae_loss
+ multiplier: 1.0
+ - function: kspace_nmse_loss
+ multiplier: 1.0
+ checkpointer:
+ checkpoint_steps: 4000
+ metrics: []
+ regularizers: []
+validation:
+ datasets:
+ - name: FastMRI
+ transforms:
+ estimate_sensitivity_maps: true
+ scaling_key: masked_kspace
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 4
+ center_fractions:
+ - 0.08
+ scale_percentile: 0.995
+ use_seed: true
+ text_description: 4x
+ - name: FastMRI
+ transforms:
+ estimate_sensitivity_maps: true
+ scaling_key: masked_kspace
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 8
+ center_fractions:
+ - 0.04
+ scale_percentile: 0.995
+ use_seed: true
+ text_description: 8x
+ - name: FastMRI
+ transforms:
+ estimate_sensitivity_maps: true
+ scaling_key: masked_kspace
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 16
+ center_fractions:
+ - 0.02
+ scale_percentile: 0.995
+ use_seed: true
+ text_description: 16x
+ batch_size: 20
+ metrics:
+ - fastmri_psnr
+ - fastmri_ssim
+ - fastmri_nmse
+ regularizers: []
+ crop: header
+inference:
+ dataset:
+ name: FastMRI
+ transforms:
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 4.0
+ center_fractions:
+ - 0.08
+ mode: STATIC
+ cropping:
+ crop: null
+ sensitivity_map_estimation:
+ estimate_sensitivity_maps: true
+ normalization:
+ scaling_key: masked_kspace
+ scale_percentile: 0.995
+ use_seed: true
+ text_description: inference-4x
+ batch_size: 1
+ crop: header
+logging:
+ log_as_image: null
+ tensorboard:
+ num_images: 4
diff --git a/projects/modulated_convolution/configs/vsharp/knee/vsharp_modconv_features_triang.yaml b/projects/modulated_convolution/configs/vsharp/knee/vsharp_modconv_features_triang.yaml
new file mode 100644
index 00000000..c00e01e5
--- /dev/null
+++ b/projects/modulated_convolution/configs/vsharp/knee/vsharp_modconv_features_triang.yaml
@@ -0,0 +1,194 @@
+model:
+ model_name: vsharp.vsharp.VSharpNet
+ num_steps: 12
+ num_steps_dc_gd: 10
+ image_init: SENSE
+ no_parameter_sharing: true
+ auxiliary_steps: -1
+ image_model_architecture: UNET
+ initializer_channels:
+ - 32
+ - 32
+ - 64
+ - 64
+ initializer_dilations:
+ - 1
+ - 1
+ - 2
+ - 4
+ initializer_multiscale: 1
+ initializer_activation: PRELU
+ conv_modulation: FEATURES
+ aux_in_features: 2
+ log_aux: true
+ fc_hidden_features:
+ - 32
+ - 32
+ fc_groups: 1
+ fc_activation: SOFTPLUS
+ num_weights: null
+ modulation_at_input: false
+ image_resnet_hidden_channels: 128
+ image_resnet_num_blocks: 15
+ image_resnet_batchnorm: true
+ image_resnet_scale: 0.1
+ image_unet_num_filters: 32
+ image_unet_num_pool_layers: 4
+ image_unet_dropout: 0.0
+ image_unet_norm_type: INSTANCE
+ image_unet_adain_hidden_features: null
+ image_didn_hidden_channels: 16
+ image_didn_num_dubs: 6
+ image_didn_num_convs_recon: 9
+ image_conv_hidden_channels: 64
+ image_conv_n_convs: 15
+ image_conv_activation: ActivationType.RELU
+ image_conv_batchnorm: false
+additional_models:
+ sensitivity_model:
+ model_name: unet.unet_2d.UnetModel2d
+ in_channels: 2
+ out_channels: 2
+ num_filters: 16
+ num_pool_layers: 4
+ dropout_probability: 0.0
+ modulation: NONE
+ aux_in_features: null
+ fc_hidden_features: null
+ fc_groups: 1
+ fc_activation: SIGMOID
+ num_weights: null
+physics:
+ forward_operator: fft2
+ backward_operator: ifft2
+ use_noise_matrix: false
+ noise_matrix_scaling: 1.0
+training:
+ datasets:
+ - name: FastMRI
+ transforms:
+ crop: reconstruction_size
+ estimate_sensitivity_maps: true
+ scaling_key: masked_kspace
+ image_center_crop: false
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 4
+ - 16
+ center_fractions:
+ - 0.08
+ - 0.02
+ range_mode: LINEAR
+ scale_percentile: 0.995
+ use_seed: false
+ delete_kspace: false
+ random_flip_probability: 0.5
+ random_rotation_probability: 0.5
+ model_checkpoint: null
+ optimizer: Adam
+ lr: 0.002
+ weight_decay: 0.0
+ batch_size: 1
+ lr_step_size: 15000
+ lr_gamma: 0.8
+ lr_warmup_iter: 1000
+ swa_start_iter: null
+ num_iterations: 80001
+ validation_steps: 4000
+ gradient_steps: 1
+ gradient_clipping: 0.0
+ gradient_debug: false
+ loss:
+ crop: header
+ losses:
+ - function: l1_loss
+ multiplier: 1.0
+ - function: ssim_loss
+ multiplier: 1.0
+ - function: hfen_l2_norm_loss
+ multiplier: 1.0
+ - function: hfen_l1_norm_loss
+ multiplier: 1.0
+ - function: kspace_nmae_loss
+ multiplier: 1.0
+ - function: kspace_nmse_loss
+ multiplier: 1.0
+ checkpointer:
+ checkpoint_steps: 4000
+ metrics: []
+ regularizers: []
+validation:
+ datasets:
+ - name: FastMRI
+ transforms:
+ estimate_sensitivity_maps: true
+ scaling_key: masked_kspace
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 4
+ center_fractions:
+ - 0.08
+ scale_percentile: 0.995
+ use_seed: true
+ text_description: 4x
+ - name: FastMRI
+ transforms:
+ estimate_sensitivity_maps: true
+ scaling_key: masked_kspace
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 8
+ center_fractions:
+ - 0.04
+ scale_percentile: 0.995
+ use_seed: true
+ text_description: 8x
+ - name: FastMRI
+ transforms:
+ estimate_sensitivity_maps: true
+ scaling_key: masked_kspace
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 16
+ center_fractions:
+ - 0.02
+ scale_percentile: 0.995
+ use_seed: true
+ text_description: 16x
+ batch_size: 20
+ metrics:
+ - fastmri_psnr
+ - fastmri_ssim
+ - fastmri_nmse
+ regularizers: []
+ crop: header
+inference:
+ dataset:
+ name: FastMRI
+ transforms:
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 4.0
+ center_fractions:
+ - 0.08
+ mode: STATIC
+ cropping:
+ crop: null
+ sensitivity_map_estimation:
+ estimate_sensitivity_maps: true
+ normalization:
+ scaling_key: masked_kspace
+ scale_percentile: 0.995
+ use_seed: true
+ text_description: inference-4x
+ batch_size: 1
+ crop: header
+logging:
+ log_as_image: null
+ tensorboard:
+ num_images: 4
diff --git a/projects/modulated_convolution/configs/vsharp/knee/vsharp_modconv_features_triang_32_16.yaml b/projects/modulated_convolution/configs/vsharp/knee/vsharp_modconv_features_triang_32_16.yaml
new file mode 100644
index 00000000..0bba5f89
--- /dev/null
+++ b/projects/modulated_convolution/configs/vsharp/knee/vsharp_modconv_features_triang_32_16.yaml
@@ -0,0 +1,194 @@
+model:
+ model_name: vsharp.vsharp.VSharpNet
+ num_steps: 12
+ num_steps_dc_gd: 10
+ image_init: SENSE
+ no_parameter_sharing: true
+ auxiliary_steps: -1
+ image_model_architecture: UNET
+ initializer_channels:
+ - 32
+ - 32
+ - 64
+ - 64
+ initializer_dilations:
+ - 1
+ - 1
+ - 2
+ - 4
+ initializer_multiscale: 1
+ initializer_activation: PRELU
+ conv_modulation: FEATURES
+ aux_in_features: 2
+ log_aux: true
+ fc_hidden_features:
+ - 32
+ - 16
+ fc_groups: 1
+ fc_activation: SOFTPLUS
+ num_weights: null
+ modulation_at_input: false
+ image_resnet_hidden_channels: 128
+ image_resnet_num_blocks: 15
+ image_resnet_batchnorm: true
+ image_resnet_scale: 0.1
+ image_unet_num_filters: 32
+ image_unet_num_pool_layers: 4
+ image_unet_dropout: 0.0
+ image_unet_norm_type: INSTANCE
+ image_unet_adain_hidden_features: null
+ image_didn_hidden_channels: 16
+ image_didn_num_dubs: 6
+ image_didn_num_convs_recon: 9
+ image_conv_hidden_channels: 64
+ image_conv_n_convs: 15
+ image_conv_activation: ActivationType.RELU
+ image_conv_batchnorm: false
+additional_models:
+ sensitivity_model:
+ model_name: unet.unet_2d.UnetModel2d
+ in_channels: 2
+ out_channels: 2
+ num_filters: 16
+ num_pool_layers: 4
+ dropout_probability: 0.0
+ modulation: NONE
+ aux_in_features: null
+ fc_hidden_features: null
+ fc_groups: 1
+ fc_activation: SIGMOID
+ num_weights: null
+physics:
+ forward_operator: fft2
+ backward_operator: ifft2
+ use_noise_matrix: false
+ noise_matrix_scaling: 1.0
+training:
+ datasets:
+ - name: FastMRI
+ transforms:
+ crop: reconstruction_size
+ estimate_sensitivity_maps: true
+ scaling_key: masked_kspace
+ image_center_crop: false
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 4
+ - 16
+ center_fractions:
+ - 0.08
+ - 0.02
+ range_mode: LINEAR
+ scale_percentile: 0.995
+ use_seed: false
+ delete_kspace: false
+ random_flip_probability: 0.5
+ random_rotation_probability: 0.5
+ model_checkpoint: null
+ optimizer: Adam
+ lr: 0.002
+ weight_decay: 0.0
+ batch_size: 1
+ lr_step_size: 15000
+ lr_gamma: 0.8
+ lr_warmup_iter: 1000
+ swa_start_iter: null
+ num_iterations: 80001
+ validation_steps: 4000
+ gradient_steps: 1
+ gradient_clipping: 0.0
+ gradient_debug: false
+ loss:
+ crop: header
+ losses:
+ - function: l1_loss
+ multiplier: 1.0
+ - function: ssim_loss
+ multiplier: 1.0
+ - function: hfen_l2_norm_loss
+ multiplier: 1.0
+ - function: hfen_l1_norm_loss
+ multiplier: 1.0
+ - function: kspace_nmae_loss
+ multiplier: 1.0
+ - function: kspace_nmse_loss
+ multiplier: 1.0
+ checkpointer:
+ checkpoint_steps: 4000
+ metrics: []
+ regularizers: []
+validation:
+ datasets:
+ - name: FastMRI
+ transforms:
+ estimate_sensitivity_maps: true
+ scaling_key: masked_kspace
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 4
+ center_fractions:
+ - 0.08
+ scale_percentile: 0.995
+ use_seed: true
+ text_description: 4x
+ - name: FastMRI
+ transforms:
+ estimate_sensitivity_maps: true
+ scaling_key: masked_kspace
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 8
+ center_fractions:
+ - 0.04
+ scale_percentile: 0.995
+ use_seed: true
+ text_description: 8x
+ - name: FastMRI
+ transforms:
+ estimate_sensitivity_maps: true
+ scaling_key: masked_kspace
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 16
+ center_fractions:
+ - 0.02
+ scale_percentile: 0.995
+ use_seed: true
+ text_description: 16x
+ batch_size: 20
+ metrics:
+ - fastmri_psnr
+ - fastmri_ssim
+ - fastmri_nmse
+ regularizers: []
+ crop: header
+inference:
+ dataset:
+ name: FastMRI
+ transforms:
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 4.0
+ center_fractions:
+ - 0.08
+ mode: STATIC
+ cropping:
+ crop: null
+ sensitivity_map_estimation:
+ estimate_sensitivity_maps: true
+ normalization:
+ scaling_key: masked_kspace
+ scale_percentile: 0.995
+ use_seed: true
+ text_description: inference-4x
+ batch_size: 1
+ crop: header
+logging:
+ log_as_image: null
+ tensorboard:
+ num_images: 4
diff --git a/projects/modulated_convolution/configs/vsharp/knee/vsharp_modconv_features_triang_32_16_mod_inp.yaml b/projects/modulated_convolution/configs/vsharp/knee/vsharp_modconv_features_triang_32_16_mod_inp.yaml
new file mode 100644
index 00000000..626d6229
--- /dev/null
+++ b/projects/modulated_convolution/configs/vsharp/knee/vsharp_modconv_features_triang_32_16_mod_inp.yaml
@@ -0,0 +1,194 @@
+model:
+ model_name: vsharp.vsharp.VSharpNet
+ num_steps: 12
+ num_steps_dc_gd: 10
+ image_init: SENSE
+ no_parameter_sharing: true
+ auxiliary_steps: -1
+ image_model_architecture: UNET
+ initializer_channels:
+ - 32
+ - 32
+ - 64
+ - 64
+ initializer_dilations:
+ - 1
+ - 1
+ - 2
+ - 4
+ initializer_multiscale: 1
+ initializer_activation: PRELU
+ conv_modulation: FEATURES
+ aux_in_features: 2
+ log_aux: true
+ fc_hidden_features:
+ - 32
+ - 16
+ fc_groups: 1
+ fc_activation: SOFTPLUS
+ num_weights: null
+ modulation_at_input: true
+ image_resnet_hidden_channels: 128
+ image_resnet_num_blocks: 15
+ image_resnet_batchnorm: true
+ image_resnet_scale: 0.1
+ image_unet_num_filters: 32
+ image_unet_num_pool_layers: 4
+ image_unet_dropout: 0.0
+ image_unet_norm_type: INSTANCE
+ image_unet_adain_hidden_features: null
+ image_didn_hidden_channels: 16
+ image_didn_num_dubs: 6
+ image_didn_num_convs_recon: 9
+ image_conv_hidden_channels: 64
+ image_conv_n_convs: 15
+ image_conv_activation: ActivationType.RELU
+ image_conv_batchnorm: false
+additional_models:
+ sensitivity_model:
+ model_name: unet.unet_2d.UnetModel2d
+ in_channels: 2
+ out_channels: 2
+ num_filters: 16
+ num_pool_layers: 4
+ dropout_probability: 0.0
+ modulation: NONE
+ aux_in_features: null
+ fc_hidden_features: null
+ fc_groups: 1
+ fc_activation: SIGMOID
+ num_weights: null
+physics:
+ forward_operator: fft2
+ backward_operator: ifft2
+ use_noise_matrix: false
+ noise_matrix_scaling: 1.0
+training:
+ datasets:
+ - name: FastMRI
+ transforms:
+ crop: reconstruction_size
+ estimate_sensitivity_maps: true
+ scaling_key: masked_kspace
+ image_center_crop: false
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 4
+ - 16
+ center_fractions:
+ - 0.08
+ - 0.02
+ range_mode: LINEAR
+ scale_percentile: 0.995
+ use_seed: false
+ delete_kspace: false
+ random_flip_probability: 0.5
+ random_rotation_probability: 0.5
+ model_checkpoint: null
+ optimizer: Adam
+ lr: 0.002
+ weight_decay: 0.0
+ batch_size: 1
+ lr_step_size: 15000
+ lr_gamma: 0.8
+ lr_warmup_iter: 1000
+ swa_start_iter: null
+ num_iterations: 150001
+ validation_steps: 4000
+ gradient_steps: 1
+ gradient_clipping: 0.0
+ gradient_debug: false
+ loss:
+ crop: header
+ losses:
+ - function: l1_loss
+ multiplier: 1.0
+ - function: ssim_loss
+ multiplier: 1.0
+ - function: hfen_l2_norm_loss
+ multiplier: 1.0
+ - function: hfen_l1_norm_loss
+ multiplier: 1.0
+ - function: kspace_nmae_loss
+ multiplier: 1.0
+ - function: kspace_nmse_loss
+ multiplier: 1.0
+ checkpointer:
+ checkpoint_steps: 4000
+ metrics: []
+ regularizers: []
+validation:
+ datasets:
+ - name: FastMRI
+ transforms:
+ estimate_sensitivity_maps: true
+ scaling_key: masked_kspace
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 4
+ center_fractions:
+ - 0.08
+ scale_percentile: 0.995
+ use_seed: true
+ text_description: 4x
+ - name: FastMRI
+ transforms:
+ estimate_sensitivity_maps: true
+ scaling_key: masked_kspace
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 8
+ center_fractions:
+ - 0.04
+ scale_percentile: 0.995
+ use_seed: true
+ text_description: 8x
+ - name: FastMRI
+ transforms:
+ estimate_sensitivity_maps: true
+ scaling_key: masked_kspace
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 16
+ center_fractions:
+ - 0.02
+ scale_percentile: 0.995
+ use_seed: true
+ text_description: 16x
+ batch_size: 20
+ metrics:
+ - fastmri_psnr
+ - fastmri_ssim
+ - fastmri_nmse
+ regularizers: []
+ crop: header
+inference:
+ dataset:
+ name: FastMRI
+ transforms:
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 4.0
+ center_fractions:
+ - 0.08
+ mode: STATIC
+ cropping:
+ crop: null
+ sensitivity_map_estimation:
+ estimate_sensitivity_maps: true
+ normalization:
+ scaling_key: masked_kspace
+ scale_percentile: 0.995
+ use_seed: true
+ text_description: inference-4x
+ batch_size: 1
+ crop: header
+logging:
+ log_as_image: null
+ tensorboard:
+ num_images: 4
diff --git a/projects/modulated_convolution/configs/vsharp/knee/vsharp_modconv_features_triang_32_8.yaml b/projects/modulated_convolution/configs/vsharp/knee/vsharp_modconv_features_triang_32_8.yaml
new file mode 100644
index 00000000..b599ca9a
--- /dev/null
+++ b/projects/modulated_convolution/configs/vsharp/knee/vsharp_modconv_features_triang_32_8.yaml
@@ -0,0 +1,194 @@
+model:
+ model_name: vsharp.vsharp.VSharpNet
+ num_steps: 12
+ num_steps_dc_gd: 10
+ image_init: SENSE
+ no_parameter_sharing: true
+ auxiliary_steps: -1
+ image_model_architecture: UNET
+ initializer_channels:
+ - 32
+ - 32
+ - 64
+ - 64
+ initializer_dilations:
+ - 1
+ - 1
+ - 2
+ - 4
+ initializer_multiscale: 1
+ initializer_activation: PRELU
+ conv_modulation: FEATURES
+ aux_in_features: 2
+ log_aux: true
+ fc_hidden_features:
+ - 32
+ - 8
+ fc_groups: 1
+ fc_activation: SOFTPLUS
+ num_weights: null
+ modulation_at_input: false
+ image_resnet_hidden_channels: 128
+ image_resnet_num_blocks: 15
+ image_resnet_batchnorm: true
+ image_resnet_scale: 0.1
+ image_unet_num_filters: 32
+ image_unet_num_pool_layers: 4
+ image_unet_dropout: 0.0
+ image_unet_norm_type: INSTANCE
+ image_unet_adain_hidden_features: null
+ image_didn_hidden_channels: 16
+ image_didn_num_dubs: 6
+ image_didn_num_convs_recon: 9
+ image_conv_hidden_channels: 64
+ image_conv_n_convs: 15
+ image_conv_activation: ActivationType.RELU
+ image_conv_batchnorm: false
+additional_models:
+ sensitivity_model:
+ model_name: unet.unet_2d.UnetModel2d
+ in_channels: 2
+ out_channels: 2
+ num_filters: 16
+ num_pool_layers: 4
+ dropout_probability: 0.0
+ modulation: NONE
+ aux_in_features: null
+ fc_hidden_features: null
+ fc_groups: 1
+ fc_activation: SIGMOID
+ num_weights: null
+physics:
+ forward_operator: fft2
+ backward_operator: ifft2
+ use_noise_matrix: false
+ noise_matrix_scaling: 1.0
+training:
+ datasets:
+ - name: FastMRI
+ transforms:
+ crop: reconstruction_size
+ estimate_sensitivity_maps: true
+ scaling_key: masked_kspace
+ image_center_crop: false
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 4
+ - 16
+ center_fractions:
+ - 0.08
+ - 0.02
+ range_mode: LINEAR
+ scale_percentile: 0.995
+ use_seed: false
+ delete_kspace: false
+ random_flip_probability: 0.5
+ random_rotation_probability: 0.5
+ model_checkpoint: null
+ optimizer: Adam
+ lr: 0.002
+ weight_decay: 0.0
+ batch_size: 1
+ lr_step_size: 15000
+ lr_gamma: 0.8
+ lr_warmup_iter: 1000
+ swa_start_iter: null
+ num_iterations: 80001
+ validation_steps: 4000
+ gradient_steps: 1
+ gradient_clipping: 0.0
+ gradient_debug: false
+ loss:
+ crop: header
+ losses:
+ - function: l1_loss
+ multiplier: 1.0
+ - function: ssim_loss
+ multiplier: 1.0
+ - function: hfen_l2_norm_loss
+ multiplier: 1.0
+ - function: hfen_l1_norm_loss
+ multiplier: 1.0
+ - function: kspace_nmae_loss
+ multiplier: 1.0
+ - function: kspace_nmse_loss
+ multiplier: 1.0
+ checkpointer:
+ checkpoint_steps: 4000
+ metrics: []
+ regularizers: []
+validation:
+ datasets:
+ - name: FastMRI
+ transforms:
+ estimate_sensitivity_maps: true
+ scaling_key: masked_kspace
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 4
+ center_fractions:
+ - 0.08
+ scale_percentile: 0.995
+ use_seed: true
+ text_description: 4x
+ - name: FastMRI
+ transforms:
+ estimate_sensitivity_maps: true
+ scaling_key: masked_kspace
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 8
+ center_fractions:
+ - 0.04
+ scale_percentile: 0.995
+ use_seed: true
+ text_description: 8x
+ - name: FastMRI
+ transforms:
+ estimate_sensitivity_maps: true
+ scaling_key: masked_kspace
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 16
+ center_fractions:
+ - 0.02
+ scale_percentile: 0.995
+ use_seed: true
+ text_description: 16x
+ batch_size: 20
+ metrics:
+ - fastmri_psnr
+ - fastmri_ssim
+ - fastmri_nmse
+ regularizers: []
+ crop: header
+inference:
+ dataset:
+ name: FastMRI
+ transforms:
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 4.0
+ center_fractions:
+ - 0.08
+ mode: STATIC
+ cropping:
+ crop: null
+ sensitivity_map_estimation:
+ estimate_sensitivity_maps: true
+ normalization:
+ scaling_key: masked_kspace
+ scale_percentile: 0.995
+ use_seed: true
+ text_description: inference-4x
+ batch_size: 1
+ crop: header
+logging:
+ log_as_image: null
+ tensorboard:
+ num_images: 4
diff --git a/projects/modulated_convolution/configs/vsharp/knee/vsharp_modconv_partial_in_triang.yaml b/projects/modulated_convolution/configs/vsharp/knee/vsharp_modconv_partial_in_triang.yaml
new file mode 100644
index 00000000..f95d6315
--- /dev/null
+++ b/projects/modulated_convolution/configs/vsharp/knee/vsharp_modconv_partial_in_triang.yaml
@@ -0,0 +1,194 @@
+model:
+ model_name: vsharp.vsharp.VSharpNet
+ num_steps: 12
+ num_steps_dc_gd: 10
+ image_init: SENSE
+ no_parameter_sharing: true
+ auxiliary_steps: -1
+ image_model_architecture: UNET
+ initializer_channels:
+ - 32
+ - 32
+ - 64
+ - 64
+ initializer_dilations:
+ - 1
+ - 1
+ - 2
+ - 4
+ initializer_multiscale: 1
+ initializer_activation: PRELU
+ conv_modulation: PARTIAL_IN
+ aux_in_features: 2
+ log_aux: true
+ fc_hidden_features:
+ - 32
+ - 32
+ fc_groups: 1
+ fc_activation: SOFTPLUS
+ num_weights: null
+ modulation_at_input: false
+ image_resnet_hidden_channels: 128
+ image_resnet_num_blocks: 15
+ image_resnet_batchnorm: true
+ image_resnet_scale: 0.1
+ image_unet_num_filters: 32
+ image_unet_num_pool_layers: 4
+ image_unet_dropout: 0.0
+ image_unet_norm_type: INSTANCE
+ image_unet_adain_hidden_features: null
+ image_didn_hidden_channels: 16
+ image_didn_num_dubs: 6
+ image_didn_num_convs_recon: 9
+ image_conv_hidden_channels: 64
+ image_conv_n_convs: 15
+ image_conv_activation: ActivationType.RELU
+ image_conv_batchnorm: false
+additional_models:
+ sensitivity_model:
+ model_name: unet.unet_2d.UnetModel2d
+ in_channels: 2
+ out_channels: 2
+ num_filters: 16
+ num_pool_layers: 4
+ dropout_probability: 0.0
+ modulation: NONE
+ aux_in_features: null
+ fc_hidden_features: null
+ fc_groups: 1
+ fc_activation: SIGMOID
+ num_weights: null
+physics:
+ forward_operator: fft2
+ backward_operator: ifft2
+ use_noise_matrix: false
+ noise_matrix_scaling: 1.0
+training:
+ datasets:
+ - name: FastMRI
+ transforms:
+ crop: reconstruction_size
+ estimate_sensitivity_maps: true
+ scaling_key: masked_kspace
+ image_center_crop: false
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 4
+ - 16
+ center_fractions:
+ - 0.08
+ - 0.02
+ range_mode: LINEAR
+ scale_percentile: 0.995
+ use_seed: false
+ delete_kspace: false
+ random_flip_probability: 0.5
+ random_rotation_probability: 0.5
+ model_checkpoint: null
+ optimizer: Adam
+ lr: 0.002
+ weight_decay: 0.0
+ batch_size: 1
+ lr_step_size: 15000
+ lr_gamma: 0.8
+ lr_warmup_iter: 1000
+ swa_start_iter: null
+ num_iterations: 80001
+ validation_steps: 4000
+ gradient_steps: 1
+ gradient_clipping: 0.0
+ gradient_debug: false
+ loss:
+ crop: header
+ losses:
+ - function: l1_loss
+ multiplier: 1.0
+ - function: ssim_loss
+ multiplier: 1.0
+ - function: hfen_l2_norm_loss
+ multiplier: 1.0
+ - function: hfen_l1_norm_loss
+ multiplier: 1.0
+ - function: kspace_nmae_loss
+ multiplier: 1.0
+ - function: kspace_nmse_loss
+ multiplier: 1.0
+ checkpointer:
+ checkpoint_steps: 4000
+ metrics: []
+ regularizers: []
+validation:
+ datasets:
+ - name: FastMRI
+ transforms:
+ estimate_sensitivity_maps: true
+ scaling_key: masked_kspace
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 4
+ center_fractions:
+ - 0.08
+ scale_percentile: 0.995
+ use_seed: true
+ text_description: 4x
+ - name: FastMRI
+ transforms:
+ estimate_sensitivity_maps: true
+ scaling_key: masked_kspace
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 8
+ center_fractions:
+ - 0.04
+ scale_percentile: 0.995
+ use_seed: true
+ text_description: 8x
+ - name: FastMRI
+ transforms:
+ estimate_sensitivity_maps: true
+ scaling_key: masked_kspace
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 16
+ center_fractions:
+ - 0.02
+ scale_percentile: 0.995
+ use_seed: true
+ text_description: 16x
+ batch_size: 20
+ metrics:
+ - fastmri_psnr
+ - fastmri_ssim
+ - fastmri_nmse
+ regularizers: []
+ crop: header
+inference:
+ dataset:
+ name: FastMRI
+ transforms:
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 4.0
+ center_fractions:
+ - 0.08
+ mode: STATIC
+ cropping:
+ crop: null
+ sensitivity_map_estimation:
+ estimate_sensitivity_maps: true
+ normalization:
+ scaling_key: masked_kspace
+ scale_percentile: 0.995
+ use_seed: true
+ text_description: inference-4x
+ batch_size: 1
+ crop: header
+logging:
+ log_as_image: null
+ tensorboard:
+ num_images: 4
diff --git a/projects/modulated_convolution/configs/vsharp/knee/vsharp_triang.yaml b/projects/modulated_convolution/configs/vsharp/knee/vsharp_triang.yaml
new file mode 100644
index 00000000..3a8d4de9
--- /dev/null
+++ b/projects/modulated_convolution/configs/vsharp/knee/vsharp_triang.yaml
@@ -0,0 +1,194 @@
+model:
+ model_name: vsharp.vsharp.VSharpNet
+ num_steps: 12
+ num_steps_dc_gd: 10
+ image_init: SENSE
+ no_parameter_sharing: true
+ auxiliary_steps: -1
+ image_model_architecture: UNET
+ initializer_channels:
+ - 32
+ - 32
+ - 64
+ - 64
+ initializer_dilations:
+ - 1
+ - 1
+ - 2
+ - 4
+ initializer_multiscale: 1
+ initializer_activation: PRELU
+ conv_modulation: NONE
+ aux_in_features: 2
+ log_aux: true
+ fc_hidden_features:
+ - 32
+ - 32
+ fc_groups: 1
+ fc_activation: SOFTPLUS
+ num_weights: null
+ modulation_at_input: false
+ image_resnet_hidden_channels: 128
+ image_resnet_num_blocks: 15
+ image_resnet_batchnorm: true
+ image_resnet_scale: 0.1
+ image_unet_num_filters: 32
+ image_unet_num_pool_layers: 4
+ image_unet_dropout: 0.0
+ image_unet_norm_type: INSTANCE
+ image_unet_adain_hidden_features: null
+ image_didn_hidden_channels: 16
+ image_didn_num_dubs: 6
+ image_didn_num_convs_recon: 9
+ image_conv_hidden_channels: 64
+ image_conv_n_convs: 15
+ image_conv_activation: ActivationType.RELU
+ image_conv_batchnorm: false
+additional_models:
+ sensitivity_model:
+ model_name: unet.unet_2d.UnetModel2d
+ in_channels: 2
+ out_channels: 2
+ num_filters: 16
+ num_pool_layers: 4
+ dropout_probability: 0.0
+ modulation: NONE
+ aux_in_features: null
+ fc_hidden_features: null
+ fc_groups: 1
+ fc_activation: SIGMOID
+ num_weights: null
+physics:
+ forward_operator: fft2
+ backward_operator: ifft2
+ use_noise_matrix: false
+ noise_matrix_scaling: 1.0
+training:
+ datasets:
+ - name: FastMRI
+ transforms:
+ crop: reconstruction_size
+ estimate_sensitivity_maps: true
+ scaling_key: masked_kspace
+ image_center_crop: false
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 4
+ - 16
+ center_fractions:
+ - 0.08
+ - 0.02
+ range_mode: LINEAR
+ scale_percentile: 0.995
+ use_seed: false
+ delete_kspace: false
+ random_flip_probability: 0.5
+ random_rotation_probability: 0.5
+ model_checkpoint: null
+ optimizer: Adam
+ lr: 0.002
+ weight_decay: 0.0
+ batch_size: 1
+ lr_step_size: 15000
+ lr_gamma: 0.8
+ lr_warmup_iter: 1000
+ swa_start_iter: null
+ num_iterations: 80001
+ validation_steps: 4000
+ gradient_steps: 1
+ gradient_clipping: 0.0
+ gradient_debug: false
+ loss:
+ crop: header
+ losses:
+ - function: l1_loss
+ multiplier: 1.0
+ - function: ssim_loss
+ multiplier: 1.0
+ - function: hfen_l2_norm_loss
+ multiplier: 1.0
+ - function: hfen_l1_norm_loss
+ multiplier: 1.0
+ - function: kspace_nmae_loss
+ multiplier: 1.0
+ - function: kspace_nmse_loss
+ multiplier: 1.0
+ checkpointer:
+ checkpoint_steps: 4000
+ metrics: []
+ regularizers: []
+validation:
+ datasets:
+ - name: FastMRI
+ transforms:
+ estimate_sensitivity_maps: true
+ scaling_key: masked_kspace
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 4
+ center_fractions:
+ - 0.08
+ scale_percentile: 0.995
+ use_seed: true
+ text_description: 4x
+ - name: FastMRI
+ transforms:
+ estimate_sensitivity_maps: true
+ scaling_key: masked_kspace
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 8
+ center_fractions:
+ - 0.04
+ scale_percentile: 0.995
+ use_seed: true
+ text_description: 8x
+ - name: FastMRI
+ transforms:
+ estimate_sensitivity_maps: true
+ scaling_key: masked_kspace
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 16
+ center_fractions:
+ - 0.02
+ scale_percentile: 0.995
+ use_seed: true
+ text_description: 16x
+ batch_size: 20
+ metrics:
+ - fastmri_psnr
+ - fastmri_ssim
+ - fastmri_nmse
+ regularizers: []
+ crop: header
+inference:
+ dataset:
+ name: FastMRI
+ transforms:
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 4.0
+ center_fractions:
+ - 0.08
+ mode: STATIC
+ cropping:
+ crop: null
+ sensitivity_map_estimation:
+ estimate_sensitivity_maps: true
+ normalization:
+ scaling_key: masked_kspace
+ scale_percentile: 0.995
+ use_seed: true
+ text_description: inference-4x
+ batch_size: 1
+ crop: header
+logging:
+ log_as_image: null
+ tensorboard:
+ num_images: 4
diff --git a/projects/modulated_convolution/configs/vsharp/prostate/vsharp_modconv_features_triang.yaml b/projects/modulated_convolution/configs/vsharp/prostate/vsharp_modconv_features_triang.yaml
new file mode 100644
index 00000000..de0c1356
--- /dev/null
+++ b/projects/modulated_convolution/configs/vsharp/prostate/vsharp_modconv_features_triang.yaml
@@ -0,0 +1,192 @@
+model:
+ model_name: vsharp.vsharp.VSharpNet
+ num_steps: 12
+ num_steps_dc_gd: 10
+ image_init: SENSE
+ no_parameter_sharing: true
+ auxiliary_steps: -1
+ image_model_architecture: UNET
+ initializer_channels:
+ - 32
+ - 32
+ - 64
+ - 64
+ initializer_dilations:
+ - 1
+ - 1
+ - 2
+ - 4
+ initializer_multiscale: 1
+ initializer_activation: PRELU
+ conv_modulation: FEATURES
+ aux_in_features: 2
+ log_aux: true
+ fc_hidden_features:
+ - 32
+ - 32
+ fc_groups: 1
+ fc_activation: SOFTPLUS
+ num_weights: null
+ modulation_at_input: false
+ image_resnet_hidden_channels: 128
+ image_resnet_num_blocks: 15
+ image_resnet_batchnorm: true
+ image_resnet_scale: 0.1
+ image_unet_num_filters: 32
+ image_unet_num_pool_layers: 4
+ image_unet_dropout: 0.0
+ image_unet_norm_type: INSTANCE
+ image_unet_adain_hidden_features: null
+ image_didn_hidden_channels: 16
+ image_didn_num_dubs: 6
+ image_didn_num_convs_recon: 9
+ image_conv_hidden_channels: 64
+ image_conv_n_convs: 15
+ image_conv_activation: ActivationType.RELU
+ image_conv_batchnorm: false
+additional_models:
+ sensitivity_model:
+ model_name: unet.unet_2d.UnetModel2d
+ in_channels: 2
+ out_channels: 2
+ num_filters: 16
+ num_pool_layers: 4
+ dropout_probability: 0.0
+ modulation: NONE
+ aux_in_features: null
+ fc_hidden_features: null
+ fc_groups: 1
+ fc_activation: SIGMOID
+ num_weights: null
+physics:
+ forward_operator: fft2
+ backward_operator: ifft2
+ use_noise_matrix: false
+ noise_matrix_scaling: 1.0
+training:
+ datasets:
+ - name: FastMRI
+ transforms:
+ crop: reconstruction_size
+ estimate_sensitivity_maps: true
+ scaling_key: masked_kspace
+ image_center_crop: false
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 4
+ - 16
+ center_fractions:
+ - 0.08
+ - 0.02
+ range_mode: LINEAR
+ scale_percentile: 0.995
+ use_seed: false
+ delete_kspace: false
+ model_checkpoint: null
+ optimizer: Adam
+ lr: 0.002
+ weight_decay: 0.0
+ batch_size: 1
+ lr_step_size: 30000
+ lr_gamma: 0.8
+ lr_warmup_iter: 1000
+ swa_start_iter: null
+ num_iterations: 150000
+ validation_steps: 4000
+ gradient_steps: 1
+ gradient_clipping: 0.0
+ gradient_debug: false
+ loss:
+ crop: header
+ losses:
+ - function: l1_loss
+ multiplier: 1.0
+ - function: ssim_loss
+ multiplier: 1.0
+ - function: hfen_l2_norm_loss
+ multiplier: 1.0
+ - function: hfen_l1_norm_loss
+ multiplier: 1.0
+ - function: kspace_nmae_loss
+ multiplier: 1.0
+ - function: kspace_nmse_loss
+ multiplier: 1.0
+ checkpointer:
+ checkpoint_steps: 4000
+ metrics: []
+ regularizers: []
+validation:
+ datasets:
+ - name: FastMRI
+ transforms:
+ estimate_sensitivity_maps: true
+ scaling_key: masked_kspace
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 4
+ center_fractions:
+ - 0.08
+ scale_percentile: 0.995
+ use_seed: true
+ text_description: 4x
+ - name: FastMRI
+ transforms:
+ estimate_sensitivity_maps: true
+ scaling_key: masked_kspace
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 8
+ center_fractions:
+ - 0.04
+ scale_percentile: 0.995
+ use_seed: true
+ text_description: 8x
+ - name: FastMRI
+ transforms:
+ estimate_sensitivity_maps: true
+ scaling_key: masked_kspace
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 16
+ center_fractions:
+ - 0.02
+ scale_percentile: 0.995
+ use_seed: true
+ text_description: 16x
+ batch_size: 20
+ metrics:
+ - fastmri_psnr
+ - fastmri_ssim
+ - fastmri_nmse
+ regularizers: []
+ crop: header
+inference:
+ dataset:
+ name: FastMRI
+ transforms:
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 4.0
+ center_fractions:
+ - 0.08
+ mode: STATIC
+ cropping:
+ crop: null
+ sensitivity_map_estimation:
+ estimate_sensitivity_maps: true
+ normalization:
+ scaling_key: masked_kspace
+ scale_percentile: 0.995
+ use_seed: true
+ text_description: inference-4x
+ batch_size: 1
+ crop: header
+logging:
+ log_as_image: null
+ tensorboard:
+ num_images: 4
diff --git a/projects/modulated_convolution/configs/vsharp/prostate/vsharp_modconv_features_triang_32_16.yaml b/projects/modulated_convolution/configs/vsharp/prostate/vsharp_modconv_features_triang_32_16.yaml
new file mode 100644
index 00000000..b02a3a84
--- /dev/null
+++ b/projects/modulated_convolution/configs/vsharp/prostate/vsharp_modconv_features_triang_32_16.yaml
@@ -0,0 +1,192 @@
+model:
+ model_name: vsharp.vsharp.VSharpNet
+ num_steps: 12
+ num_steps_dc_gd: 10
+ image_init: SENSE
+ no_parameter_sharing: true
+ auxiliary_steps: -1
+ image_model_architecture: UNET
+ initializer_channels:
+ - 32
+ - 32
+ - 64
+ - 64
+ initializer_dilations:
+ - 1
+ - 1
+ - 2
+ - 4
+ initializer_multiscale: 1
+ initializer_activation: PRELU
+ conv_modulation: FEATURES
+ aux_in_features: 2
+ log_aux: true
+ fc_hidden_features:
+ - 32
+ - 16
+ fc_groups: 1
+ fc_activation: SOFTPLUS
+ num_weights: null
+ modulation_at_input: false
+ image_resnet_hidden_channels: 128
+ image_resnet_num_blocks: 15
+ image_resnet_batchnorm: true
+ image_resnet_scale: 0.1
+ image_unet_num_filters: 32
+ image_unet_num_pool_layers: 4
+ image_unet_dropout: 0.0
+ image_unet_norm_type: INSTANCE
+ image_unet_adain_hidden_features: null
+ image_didn_hidden_channels: 16
+ image_didn_num_dubs: 6
+ image_didn_num_convs_recon: 9
+ image_conv_hidden_channels: 64
+ image_conv_n_convs: 15
+ image_conv_activation: ActivationType.RELU
+ image_conv_batchnorm: false
+additional_models:
+ sensitivity_model:
+ model_name: unet.unet_2d.UnetModel2d
+ in_channels: 2
+ out_channels: 2
+ num_filters: 16
+ num_pool_layers: 4
+ dropout_probability: 0.0
+ modulation: NONE
+ aux_in_features: null
+ fc_hidden_features: null
+ fc_groups: 1
+ fc_activation: SIGMOID
+ num_weights: null
+physics:
+ forward_operator: fft2
+ backward_operator: ifft2
+ use_noise_matrix: false
+ noise_matrix_scaling: 1.0
+training:
+ datasets:
+ - name: FastMRI
+ transforms:
+ crop: reconstruction_size
+ estimate_sensitivity_maps: true
+ scaling_key: masked_kspace
+ image_center_crop: false
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 4
+ - 16
+ center_fractions:
+ - 0.08
+ - 0.02
+ range_mode: LINEAR
+ scale_percentile: 0.995
+ use_seed: false
+ delete_kspace: false
+ model_checkpoint: null
+ optimizer: Adam
+ lr: 0.002
+ weight_decay: 0.0
+ batch_size: 1
+ lr_step_size: 30000
+ lr_gamma: 0.8
+ lr_warmup_iter: 1000
+ swa_start_iter: null
+ num_iterations: 150000
+ validation_steps: 4000
+ gradient_steps: 1
+ gradient_clipping: 0.0
+ gradient_debug: false
+ loss:
+ crop: header
+ losses:
+ - function: l1_loss
+ multiplier: 1.0
+ - function: ssim_loss
+ multiplier: 1.0
+ - function: hfen_l2_norm_loss
+ multiplier: 1.0
+ - function: hfen_l1_norm_loss
+ multiplier: 1.0
+ - function: kspace_nmae_loss
+ multiplier: 1.0
+ - function: kspace_nmse_loss
+ multiplier: 1.0
+ checkpointer:
+ checkpoint_steps: 4000
+ metrics: []
+ regularizers: []
+validation:
+ datasets:
+ - name: FastMRI
+ transforms:
+ estimate_sensitivity_maps: true
+ scaling_key: masked_kspace
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 4
+ center_fractions:
+ - 0.08
+ scale_percentile: 0.995
+ use_seed: true
+ text_description: 4x
+ - name: FastMRI
+ transforms:
+ estimate_sensitivity_maps: true
+ scaling_key: masked_kspace
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 8
+ center_fractions:
+ - 0.04
+ scale_percentile: 0.995
+ use_seed: true
+ text_description: 8x
+ - name: FastMRI
+ transforms:
+ estimate_sensitivity_maps: true
+ scaling_key: masked_kspace
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 16
+ center_fractions:
+ - 0.02
+ scale_percentile: 0.995
+ use_seed: true
+ text_description: 16x
+ batch_size: 20
+ metrics:
+ - fastmri_psnr
+ - fastmri_ssim
+ - fastmri_nmse
+ regularizers: []
+ crop: header
+inference:
+ dataset:
+ name: FastMRI
+ transforms:
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 4.0
+ center_fractions:
+ - 0.08
+ mode: STATIC
+ cropping:
+ crop: null
+ sensitivity_map_estimation:
+ estimate_sensitivity_maps: true
+ normalization:
+ scaling_key: masked_kspace
+ scale_percentile: 0.995
+ use_seed: true
+ text_description: inference-4x
+ batch_size: 1
+ crop: header
+logging:
+ log_as_image: null
+ tensorboard:
+ num_images: 4
diff --git a/projects/modulated_convolution/configs/vsharp/prostate/vsharp_modconv_features_triang_32_8.yaml b/projects/modulated_convolution/configs/vsharp/prostate/vsharp_modconv_features_triang_32_8.yaml
new file mode 100644
index 00000000..fe961f49
--- /dev/null
+++ b/projects/modulated_convolution/configs/vsharp/prostate/vsharp_modconv_features_triang_32_8.yaml
@@ -0,0 +1,192 @@
+model:
+ model_name: vsharp.vsharp.VSharpNet
+ num_steps: 12
+ num_steps_dc_gd: 10
+ image_init: SENSE
+ no_parameter_sharing: true
+ auxiliary_steps: -1
+ image_model_architecture: UNET
+ initializer_channels:
+ - 32
+ - 32
+ - 64
+ - 64
+ initializer_dilations:
+ - 1
+ - 1
+ - 2
+ - 4
+ initializer_multiscale: 1
+ initializer_activation: PRELU
+ conv_modulation: FEATURES
+ aux_in_features: 2
+ log_aux: true
+ fc_hidden_features:
+ - 32
+ - 8
+ fc_groups: 1
+ fc_activation: SOFTPLUS
+ num_weights: null
+ modulation_at_input: false
+ image_resnet_hidden_channels: 128
+ image_resnet_num_blocks: 15
+ image_resnet_batchnorm: true
+ image_resnet_scale: 0.1
+ image_unet_num_filters: 32
+ image_unet_num_pool_layers: 4
+ image_unet_dropout: 0.0
+ image_unet_norm_type: INSTANCE
+ image_unet_adain_hidden_features: null
+ image_didn_hidden_channels: 16
+ image_didn_num_dubs: 6
+ image_didn_num_convs_recon: 9
+ image_conv_hidden_channels: 64
+ image_conv_n_convs: 15
+ image_conv_activation: ActivationType.RELU
+ image_conv_batchnorm: false
+additional_models:
+ sensitivity_model:
+ model_name: unet.unet_2d.UnetModel2d
+ in_channels: 2
+ out_channels: 2
+ num_filters: 16
+ num_pool_layers: 4
+ dropout_probability: 0.0
+ modulation: NONE
+ aux_in_features: null
+ fc_hidden_features: null
+ fc_groups: 1
+ fc_activation: SIGMOID
+ num_weights: null
+physics:
+ forward_operator: fft2
+ backward_operator: ifft2
+ use_noise_matrix: false
+ noise_matrix_scaling: 1.0
+training:
+ datasets:
+ - name: FastMRI
+ transforms:
+ crop: reconstruction_size
+ estimate_sensitivity_maps: true
+ scaling_key: masked_kspace
+ image_center_crop: false
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 4
+ - 16
+ center_fractions:
+ - 0.08
+ - 0.02
+ range_mode: LINEAR
+ scale_percentile: 0.995
+ use_seed: false
+ delete_kspace: false
+ model_checkpoint: null
+ optimizer: Adam
+ lr: 0.002
+ weight_decay: 0.0
+ batch_size: 1
+ lr_step_size: 30000
+ lr_gamma: 0.8
+ lr_warmup_iter: 1000
+ swa_start_iter: null
+ num_iterations: 150000
+ validation_steps: 4000
+ gradient_steps: 1
+ gradient_clipping: 0.0
+ gradient_debug: false
+ loss:
+ crop: header
+ losses:
+ - function: l1_loss
+ multiplier: 1.0
+ - function: ssim_loss
+ multiplier: 1.0
+ - function: hfen_l2_norm_loss
+ multiplier: 1.0
+ - function: hfen_l1_norm_loss
+ multiplier: 1.0
+ - function: kspace_nmae_loss
+ multiplier: 1.0
+ - function: kspace_nmse_loss
+ multiplier: 1.0
+ checkpointer:
+ checkpoint_steps: 4000
+ metrics: []
+ regularizers: []
+validation:
+ datasets:
+ - name: FastMRI
+ transforms:
+ estimate_sensitivity_maps: true
+ scaling_key: masked_kspace
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 4
+ center_fractions:
+ - 0.08
+ scale_percentile: 0.995
+ use_seed: true
+ text_description: 4x
+ - name: FastMRI
+ transforms:
+ estimate_sensitivity_maps: true
+ scaling_key: masked_kspace
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 8
+ center_fractions:
+ - 0.04
+ scale_percentile: 0.995
+ use_seed: true
+ text_description: 8x
+ - name: FastMRI
+ transforms:
+ estimate_sensitivity_maps: true
+ scaling_key: masked_kspace
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 16
+ center_fractions:
+ - 0.02
+ scale_percentile: 0.995
+ use_seed: true
+ text_description: 16x
+ batch_size: 20
+ metrics:
+ - fastmri_psnr
+ - fastmri_ssim
+ - fastmri_nmse
+ regularizers: []
+ crop: header
+inference:
+ dataset:
+ name: FastMRI
+ transforms:
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 4.0
+ center_fractions:
+ - 0.08
+ mode: STATIC
+ cropping:
+ crop: null
+ sensitivity_map_estimation:
+ estimate_sensitivity_maps: true
+ normalization:
+ scaling_key: masked_kspace
+ scale_percentile: 0.995
+ use_seed: true
+ text_description: inference-4x
+ batch_size: 1
+ crop: header
+logging:
+ log_as_image: null
+ tensorboard:
+ num_images: 4
diff --git a/projects/modulated_convolution/configs/vsharp/prostate/vsharp_modconv_partial_in_triang.yaml b/projects/modulated_convolution/configs/vsharp/prostate/vsharp_modconv_partial_in_triang.yaml
new file mode 100644
index 00000000..96f8d1c3
--- /dev/null
+++ b/projects/modulated_convolution/configs/vsharp/prostate/vsharp_modconv_partial_in_triang.yaml
@@ -0,0 +1,192 @@
+model:
+ model_name: vsharp.vsharp.VSharpNet
+ num_steps: 12
+ num_steps_dc_gd: 10
+ image_init: SENSE
+ no_parameter_sharing: true
+ auxiliary_steps: -1
+ image_model_architecture: UNET
+ initializer_channels:
+ - 32
+ - 32
+ - 64
+ - 64
+ initializer_dilations:
+ - 1
+ - 1
+ - 2
+ - 4
+ initializer_multiscale: 1
+ initializer_activation: PRELU
+ conv_modulation: PARTIAL_IN
+ aux_in_features: 2
+ log_aux: true
+ fc_hidden_features:
+ - 32
+ - 32
+ fc_groups: 1
+ fc_activation: SOFTPLUS
+ num_weights: null
+ modulation_at_input: false
+ image_resnet_hidden_channels: 128
+ image_resnet_num_blocks: 15
+ image_resnet_batchnorm: true
+ image_resnet_scale: 0.1
+ image_unet_num_filters: 32
+ image_unet_num_pool_layers: 4
+ image_unet_dropout: 0.0
+ image_unet_norm_type: INSTANCE
+ image_unet_adain_hidden_features: null
+ image_didn_hidden_channels: 16
+ image_didn_num_dubs: 6
+ image_didn_num_convs_recon: 9
+ image_conv_hidden_channels: 64
+ image_conv_n_convs: 15
+ image_conv_activation: ActivationType.RELU
+ image_conv_batchnorm: false
+additional_models:
+ sensitivity_model:
+ model_name: unet.unet_2d.UnetModel2d
+ in_channels: 2
+ out_channels: 2
+ num_filters: 16
+ num_pool_layers: 4
+ dropout_probability: 0.0
+ modulation: NONE
+ aux_in_features: null
+ fc_hidden_features: null
+ fc_groups: 1
+ fc_activation: SIGMOID
+ num_weights: null
+physics:
+ forward_operator: fft2
+ backward_operator: ifft2
+ use_noise_matrix: false
+ noise_matrix_scaling: 1.0
+training:
+ datasets:
+ - name: FastMRI
+ transforms:
+ crop: reconstruction_size
+ estimate_sensitivity_maps: true
+ scaling_key: masked_kspace
+ image_center_crop: false
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 4
+ - 16
+ center_fractions:
+ - 0.08
+ - 0.02
+ range_mode: LINEAR
+ scale_percentile: 0.995
+ use_seed: false
+ delete_kspace: false
+ model_checkpoint: null
+ optimizer: Adam
+ lr: 0.002
+ weight_decay: 0.0
+ batch_size: 1
+ lr_step_size: 30000
+ lr_gamma: 0.8
+ lr_warmup_iter: 1000
+ swa_start_iter: null
+ num_iterations: 150000
+ validation_steps: 4000
+ gradient_steps: 1
+ gradient_clipping: 0.0
+ gradient_debug: false
+ loss:
+ crop: header
+ losses:
+ - function: l1_loss
+ multiplier: 1.0
+ - function: ssim_loss
+ multiplier: 1.0
+ - function: hfen_l2_norm_loss
+ multiplier: 1.0
+ - function: hfen_l1_norm_loss
+ multiplier: 1.0
+ - function: kspace_nmae_loss
+ multiplier: 1.0
+ - function: kspace_nmse_loss
+ multiplier: 1.0
+ checkpointer:
+ checkpoint_steps: 4000
+ metrics: []
+ regularizers: []
+validation:
+ datasets:
+ - name: FastMRI
+ transforms:
+ estimate_sensitivity_maps: true
+ scaling_key: masked_kspace
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 4
+ center_fractions:
+ - 0.08
+ scale_percentile: 0.995
+ use_seed: true
+ text_description: 4x
+ - name: FastMRI
+ transforms:
+ estimate_sensitivity_maps: true
+ scaling_key: masked_kspace
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 8
+ center_fractions:
+ - 0.04
+ scale_percentile: 0.995
+ use_seed: true
+ text_description: 8x
+ - name: FastMRI
+ transforms:
+ estimate_sensitivity_maps: true
+ scaling_key: masked_kspace
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 16
+ center_fractions:
+ - 0.02
+ scale_percentile: 0.995
+ use_seed: true
+ text_description: 16x
+ batch_size: 20
+ metrics:
+ - fastmri_psnr
+ - fastmri_ssim
+ - fastmri_nmse
+ regularizers: []
+ crop: header
+inference:
+ dataset:
+ name: FastMRI
+ transforms:
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 4.0
+ center_fractions:
+ - 0.08
+ mode: STATIC
+ cropping:
+ crop: null
+ sensitivity_map_estimation:
+ estimate_sensitivity_maps: true
+ normalization:
+ scaling_key: masked_kspace
+ scale_percentile: 0.995
+ use_seed: true
+ text_description: inference-4x
+ batch_size: 1
+ crop: header
+logging:
+ log_as_image: null
+ tensorboard:
+ num_images: 4
diff --git a/projects/modulated_convolution/configs/vsharp/prostate/vsharp_triang.yaml b/projects/modulated_convolution/configs/vsharp/prostate/vsharp_triang.yaml
new file mode 100644
index 00000000..53ff3760
--- /dev/null
+++ b/projects/modulated_convolution/configs/vsharp/prostate/vsharp_triang.yaml
@@ -0,0 +1,192 @@
+model:
+ model_name: vsharp.vsharp.VSharpNet
+ num_steps: 12
+ num_steps_dc_gd: 10
+ image_init: SENSE
+ no_parameter_sharing: true
+ auxiliary_steps: -1
+ image_model_architecture: UNET
+ initializer_channels:
+ - 32
+ - 32
+ - 64
+ - 64
+ initializer_dilations:
+ - 1
+ - 1
+ - 2
+ - 4
+ initializer_multiscale: 1
+ initializer_activation: PRELU
+ conv_modulation: NONE
+ aux_in_features: 2
+ log_aux: true
+ fc_hidden_features:
+ - 32
+ - 32
+ fc_groups: 1
+ fc_activation: SOFTPLUS
+ num_weights: null
+ modulation_at_input: false
+ image_resnet_hidden_channels: 128
+ image_resnet_num_blocks: 15
+ image_resnet_batchnorm: true
+ image_resnet_scale: 0.1
+ image_unet_num_filters: 32
+ image_unet_num_pool_layers: 4
+ image_unet_dropout: 0.0
+ image_unet_norm_type: INSTANCE
+ image_unet_adain_hidden_features: null
+ image_didn_hidden_channels: 16
+ image_didn_num_dubs: 6
+ image_didn_num_convs_recon: 9
+ image_conv_hidden_channels: 64
+ image_conv_n_convs: 15
+ image_conv_activation: ActivationType.RELU
+ image_conv_batchnorm: false
+additional_models:
+ sensitivity_model:
+ model_name: unet.unet_2d.UnetModel2d
+ in_channels: 2
+ out_channels: 2
+ num_filters: 16
+ num_pool_layers: 4
+ dropout_probability: 0.0
+ modulation: NONE
+ aux_in_features: null
+ fc_hidden_features: null
+ fc_groups: 1
+ fc_activation: SIGMOID
+ num_weights: null
+physics:
+ forward_operator: fft2
+ backward_operator: ifft2
+ use_noise_matrix: false
+ noise_matrix_scaling: 1.0
+training:
+ datasets:
+ - name: FastMRI
+ transforms:
+ crop: reconstruction_size
+ estimate_sensitivity_maps: true
+ scaling_key: masked_kspace
+ image_center_crop: false
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 4
+ - 16
+ center_fractions:
+ - 0.08
+ - 0.02
+ range_mode: LINEAR
+ scale_percentile: 0.995
+ use_seed: false
+ delete_kspace: false
+ model_checkpoint: null
+ optimizer: Adam
+ lr: 0.002
+ weight_decay: 0.0
+ batch_size: 1
+ lr_step_size: 30000
+ lr_gamma: 0.8
+ lr_warmup_iter: 1000
+ swa_start_iter: null
+ num_iterations: 150000
+ validation_steps: 4000
+ gradient_steps: 1
+ gradient_clipping: 0.0
+ gradient_debug: false
+ loss:
+ crop: header
+ losses:
+ - function: l1_loss
+ multiplier: 1.0
+ - function: ssim_loss
+ multiplier: 1.0
+ - function: hfen_l2_norm_loss
+ multiplier: 1.0
+ - function: hfen_l1_norm_loss
+ multiplier: 1.0
+ - function: kspace_nmae_loss
+ multiplier: 1.0
+ - function: kspace_nmse_loss
+ multiplier: 1.0
+ checkpointer:
+ checkpoint_steps: 4000
+ metrics: []
+ regularizers: []
+validation:
+ datasets:
+ - name: FastMRI
+ transforms:
+ estimate_sensitivity_maps: true
+ scaling_key: masked_kspace
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 4
+ center_fractions:
+ - 0.08
+ scale_percentile: 0.995
+ use_seed: true
+ text_description: 4x
+ - name: FastMRI
+ transforms:
+ estimate_sensitivity_maps: true
+ scaling_key: masked_kspace
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 8
+ center_fractions:
+ - 0.04
+ scale_percentile: 0.995
+ use_seed: true
+ text_description: 8x
+ - name: FastMRI
+ transforms:
+ estimate_sensitivity_maps: true
+ scaling_key: masked_kspace
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 16
+ center_fractions:
+ - 0.02
+ scale_percentile: 0.995
+ use_seed: true
+ text_description: 16x
+ batch_size: 20
+ metrics:
+ - fastmri_psnr
+ - fastmri_ssim
+ - fastmri_nmse
+ regularizers: []
+ crop: header
+inference:
+ dataset:
+ name: FastMRI
+ transforms:
+ masking:
+ name: FastMRIEquispaced
+ accelerations:
+ - 4.0
+ center_fractions:
+ - 0.08
+ mode: STATIC
+ cropping:
+ crop: null
+ sensitivity_map_estimation:
+ estimate_sensitivity_maps: true
+ normalization:
+ scaling_key: masked_kspace
+ scale_percentile: 0.995
+ use_seed: true
+ text_description: inference-4x
+ batch_size: 1
+ crop: header
+logging:
+ log_as_image: null
+ tensorboard:
+ num_images: 4
diff --git a/projects/modulated_convolution/figures/knee_reconstruction.png b/projects/modulated_convolution/figures/knee_reconstruction.png
new file mode 100644
index 00000000..0440f771
Binary files /dev/null and b/projects/modulated_convolution/figures/knee_reconstruction.png differ
diff --git a/projects/modulated_convolution/figures/modconv_architecture.png b/projects/modulated_convolution/figures/modconv_architecture.png
new file mode 100644
index 00000000..523d8746
Binary files /dev/null and b/projects/modulated_convolution/figures/modconv_architecture.png differ
diff --git a/projects/modulated_convolution/figures/modulation_mlp.png b/projects/modulated_convolution/figures/modulation_mlp.png
new file mode 100644
index 00000000..49c361f9
Binary files /dev/null and b/projects/modulated_convolution/figures/modulation_mlp.png differ
diff --git a/tests/checkpointer_test.py b/tests/checkpointer_test.py
index f08b0fe9..88fa8528 100644
--- a/tests/checkpointer_test.py
+++ b/tests/checkpointer_test.py
@@ -60,7 +60,14 @@ def create_checkpointables(*keys):
[],
["sensitivity_model", "optimizer", "something_which_is_not_stored"],
["sensitivity_model", "optimizer"],
- ["sensitivity_model", "optimizer", "__author__", "__version__", "__datetime__", "__mixed_precision__"],
+ [
+ "sensitivity_model",
+ "optimizer",
+ "__author__",
+ "__version__",
+ "__datetime__",
+ "__mixed_precision__",
+ ],
],
)
def test_checkpointer(checkpoint_ids, checkpointables_keys):
@@ -68,7 +75,11 @@ def test_checkpointer(checkpoint_ids, checkpointables_keys):
for checkpoint_id in checkpoint_ids:
checkpointables = create_checkpointables(checkpointables_keys)
- checkpointer = Checkpointer(save_directory=pathlib.Path(tempdir), save_to_disk=True, **checkpointables)
+ checkpointer = Checkpointer(
+ save_directory=pathlib.Path(tempdir),
+ save_to_disk=True,
+ **checkpointables,
+ )
# Test save function
checkpointer.save(iteration=checkpoint_id)
# Test load function
diff --git a/tests/tests_common/subsample_test.py b/tests/tests_common/subsample_test.py
index 4cf399ff..fb075346 100644
--- a/tests/tests_common/subsample_test.py
+++ b/tests/tests_common/subsample_test.py
@@ -22,6 +22,7 @@
import torch
from direct.common.subsample import *
+from direct.utils.distributions import triangular_distribution
@pytest.mark.parametrize(
@@ -136,7 +137,12 @@ def test_mask_reuse_kt(mask_func, center_fracs, accelerations, batch_size, shape
@pytest.mark.parametrize(
"mask_func",
- [FastMRIRandomMaskFunc, FastMRIEquispacedMaskFunc, FastMRIMagicMaskFunc, Gaussian1DMaskFunc],
+ [
+ FastMRIRandomMaskFunc,
+ FastMRIEquispacedMaskFunc,
+ FastMRIMagicMaskFunc,
+ Gaussian1DMaskFunc,
+ ],
)
@pytest.mark.parametrize(
"center_fracs, accelerations, batch_size, dim",
@@ -171,7 +177,12 @@ def test_cartesian_mask_low_freqs(mask_func, center_fracs, accelerations, batch_
@pytest.mark.parametrize(
"mask_func",
- [FastMRIRandomMaskFunc, FastMRIEquispacedMaskFunc, FastMRIMagicMaskFunc, Gaussian1DMaskFunc],
+ [
+ FastMRIRandomMaskFunc,
+ FastMRIEquispacedMaskFunc,
+ FastMRIMagicMaskFunc,
+ Gaussian1DMaskFunc,
+ ],
)
@pytest.mark.parametrize(
"shape, center_fractions, accelerations, mode",
@@ -199,7 +210,12 @@ def test_apply_mask_cartesian(mask_func, shape, center_fractions, accelerations,
@pytest.mark.parametrize(
"mask_func",
- [FastMRIRandomMaskFunc, FastMRIEquispacedMaskFunc, FastMRIMagicMaskFunc, Gaussian1DMaskFunc],
+ [
+ FastMRIRandomMaskFunc,
+ FastMRIEquispacedMaskFunc,
+ FastMRIMagicMaskFunc,
+ Gaussian1DMaskFunc,
+ ],
)
@pytest.mark.parametrize(
"shape, center_fractions, accelerations",
@@ -534,3 +550,60 @@ def test_apply_kt_mask(mask_func, shape, accelerations, center_fractions):
assert all(not np.allclose(mask[:, _], mask[:, _ + 1]) for _ in range(shape[1] - 1))
assert all(np.allclose(acs_mask[:, _], acs_mask[:, _ + 1]) for _ in range(shape[1] - 1))
+
+
+def test_linear_range_triangular_sampling_bias():
+ mask_func = FastMRIRandomMaskFunc(
+ center_fractions=[0.04, 0.08],
+ accelerations=[4.0, 8.0],
+ range_mode=RangeMode.LINEAR,
+ )
+ samples = []
+ for seed in range(500):
+ _, acceleration, _ = mask_func((1, 320, 320, 2), seed=seed, return_acceleration=True)
+ samples.append(acceleration)
+
+ mean_accel = float(np.mean(samples))
+ assert mean_accel > 5.5
+
+
+def test_return_acceleration_on_poisson_mask():
+ mask_func = VariableDensityPoissonMaskFunc(
+ center_fractions=[0.08, 0.04],
+ accelerations=[4.0, 8.0],
+ )
+ mask, acceleration, center_fraction = mask_func((1, 320, 320, 2), seed=42, return_acceleration=True)
+ assert mask.dtype == torch.bool
+ assert acceleration in (4.0, 8.0)
+ assert 0.0 < center_fraction <= 0.25
+
+
+def test_build_masking_function_linear_range():
+ mask_func = build_masking_function(
+ name="FastMRIRandom",
+ center_fractions=[0.04, 0.08],
+ accelerations=[4.0, 8.0],
+ range_mode=RangeMode.LINEAR,
+ )
+ assert mask_func.range_mode == RangeMode.LINEAR
+
+
+def test_equispaced_linear_range_no_invalid_randint():
+ mask_func = FastMRIEquispacedMaskFunc(
+ center_fractions=[0.08, 0.02],
+ accelerations=[4, 16],
+ range_mode=RangeMode.LINEAR,
+ )
+ shape = (1, 640, 368, 2)
+ for seed in range(1000):
+ mask, acceleration, center_fraction = mask_func(shape, seed=seed, return_acceleration=True)
+ assert mask.dtype == torch.bool
+ assert acceleration > 0
+ assert center_fraction > 0
+
+
+def test_triangular_distribution_default_rng():
+ samples = triangular_distribution(4.0, 8.0, n=100)
+ assert samples.shape == (100,)
+ assert np.all(samples >= 4.0)
+ assert np.all(samples <= 8.0)
diff --git a/tests/tests_data/mri_transforms_test.py b/tests/tests_data/mri_transforms_test.py
index 1ce75bd6..9cdf7bc6 100644
--- a/tests/tests_data/mri_transforms_test.py
+++ b/tests/tests_data/mri_transforms_test.py
@@ -91,7 +91,12 @@ def _mask_func(shape, seed=None, return_acs=False):
[
(["key1"], [True], {}, {"key1": True}),
(["key1", "key2"], [True, False], {}, {"key1": True, "key2": False}),
- (["key1"], [True], {"existing_key": "existing_value"}, {"existing_key": "existing_value", "key1": True}),
+ (
+ ["key1"],
+ [True],
+ {"existing_key": "existing_value"},
+ {"existing_key": "existing_value", "key1": True},
+ ),
],
)
def test_add_boolean_keys_module(keys, values, sample, expected):
@@ -183,7 +188,9 @@ def test_CreateSamplingMask(shape, return_acs, use_shape):
sample = create_sample(shape)
transform = CreateSamplingMask(
- mask_func=_mask_func, shape=shape[1:-1] if use_shape else None, return_acs=return_acs
+ mask_func=_mask_func,
+ shape=shape[1:-1] if use_shape else None,
+ return_acs=return_acs,
)
sample = transform(sample)
print(sample["kspace"].shape, sample["sampling_mask"].shape)
@@ -198,6 +205,25 @@ def test_CreateSamplingMask(shape, return_acs, use_shape):
assert "acs_mask" in sample
+def test_CreateSamplingMask_returns_acceleration():
+ from direct.common.subsample import FastMRIEquispacedMaskFunc, RangeMode
+
+ shape = (4, 32, 32, 2)
+ sample = create_sample(shape)
+ mask_func = FastMRIEquispacedMaskFunc(
+ center_fractions=[0.08, 0.04],
+ accelerations=[4.0, 8.0],
+ range_mode=RangeMode.UNIFORM,
+ )
+ transform = CreateSamplingMask(mask_func=mask_func)
+ sample = transform(sample)
+
+ assert "acceleration" in sample
+ assert "center_fraction" in sample
+ assert sample["acceleration"].numel() == 1
+ assert sample["center_fraction"].numel() == 1
+
+
@pytest.mark.parametrize(
"shape",
[(4, 32, 32), (3, 10, 16)],
@@ -222,7 +248,14 @@ def test_ApplyMask(shape):
)
@pytest.mark.parametrize(
"crop",
- [(10, 5, 6), "reconstruction_size", "[10, 5, 6]", "(10, 5, 6)", None, "invalid_key"],
+ [
+ (10, 5, 6),
+ "reconstruction_size",
+ "[10, 5, 6]",
+ "(10, 5, 6)",
+ None,
+ "invalid_key",
+ ],
)
@pytest.mark.parametrize(
"image_space_center_crop",
@@ -323,7 +356,14 @@ def test_PadKspace(shape, pad_shape):
)
@pytest.mark.parametrize(
"crop",
- [(10, 5, 6), "reconstruction_size", "[10, 5, 6]", "(10, 5, 6)", None, "invalid_key"],
+ [
+ (10, 5, 6),
+ "reconstruction_size",
+ "[10, 5, 6]",
+ "(10, 5, 6)",
+ None,
+ "invalid_key",
+ ],
)
@pytest.mark.parametrize(
"image_space_center_crop",
@@ -630,7 +670,14 @@ def test_CompressCoil(shape, compress_coils):
@pytest.mark.parametrize(
"shape, pad_coils",
- [[(3, 10, 16), 5], [(5, 7, 6), 5], [(4, 5, 5), 2], [(4, 5, 5), None], [(3, 4, 6, 4), 4], [(5, 3, 3, 4), 3]],
+ [
+ [(3, 10, 16), 5],
+ [(5, 7, 6), 5],
+ [(4, 5, 5), 2],
+ [(4, 5, 5), None],
+ [(3, 4, 6, 4), 4],
+ [(5, 3, 3, 4), 3],
+ ],
)
@pytest.mark.parametrize(
"key",
@@ -775,7 +822,14 @@ def test_build_mri_transforms(shape, spatial_dims, estimate_body_coil_image, ima
assert all(
key in sample.keys()
- for key in ["sampling_mask", "sensitivity_map", "target", "masked_kspace", "scaling_diff", "scaling_factor"]
+ for key in [
+ "sampling_mask",
+ "sensitivity_map",
+ "target",
+ "masked_kspace",
+ "scaling_diff",
+ "scaling_factor",
+ ]
)
assert sample["masked_kspace"].shape == shape + (2,)
assert sample["sensitivity_map"].shape == shape + (2,)
diff --git a/tests/tests_data/transforms_test.py b/tests/tests_data/transforms_test.py
index d1589d68..1936b5f4 100644
--- a/tests/tests_data/transforms_test.py
+++ b/tests/tests_data/transforms_test.py
@@ -22,7 +22,7 @@
def create_input(shape):
- data = np.random.randn(*shape).copy()
+ data = np.random.RandomState(0).randn(*shape).copy()
data = torch.from_numpy(data).float()
return data
@@ -491,7 +491,11 @@ def test_complex_random_crop(shapes, crop_shape, sampler, sigma, expect_error, c
if expect_error:
with pytest.raises(ValueError):
samples = transforms.complex_random_crop(
- data_list, crop_shape, sampler=sampler, sigma=sigma, contiguous=contiguous
+ data_list,
+ crop_shape,
+ sampler=sampler,
+ sigma=sigma,
+ contiguous=contiguous,
)
else:
data_list = transforms.complex_random_crop(
diff --git a/tests/tests_functionals/gradloss_test.py b/tests/tests_functionals/gradloss_test.py
index cf14f818..47cba474 100644
--- a/tests/tests_functionals/gradloss_test.py
+++ b/tests/tests_functionals/gradloss_test.py
@@ -36,8 +36,8 @@ def test_nmse(image):
image_batch.append(image)
image_noise_batch.append(image_noise)
- image_batch_torch = torch.tensor(image_batch)
- image_noise_batch_torch = torch.tensor(image_noise_batch)
+ image_batch_torch = torch.tensor(np.array(image_batch))
+ image_noise_batch_torch = torch.tensor(np.array(image_noise_batch))
grad_loss_l1 = SobelGradL1Loss(image_batch_torch, image_noise_batch_torch)
grad_loss_l2 = SobelGradL2Loss(image_batch_torch, image_noise_batch_torch)
diff --git a/tests/tests_nn/auxiliary_data_test.py b/tests/tests_nn/auxiliary_data_test.py
new file mode 100644
index 00000000..d57d98c1
--- /dev/null
+++ b/tests/tests_nn/auxiliary_data_test.py
@@ -0,0 +1,202 @@
+# Copyright 2026 AI for Oncology Research Group. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from dataclasses import dataclass
+from typing import Optional
+
+import pytest
+import torch
+
+from direct.nn.conv.modulated import (
+ AuxiliaryFeature,
+ ModConvType,
+ prepare_auxiliary_data,
+ register_auxiliary_feature,
+ resolve_auxiliary_features,
+)
+
+
+@dataclass
+class _Cfg:
+ conv_modulation: ModConvType
+ aux_in_features: int
+ log_aux: bool
+ auxiliary_features: Optional[tuple[str, ...]] = None
+
+
+def _batch_data() -> dict[str, torch.Tensor]:
+ return {
+ "acceleration": torch.tensor([4.0, 8.0]),
+ "center_fraction": torch.tensor([0.08, 0.04]),
+ "field_strength": torch.tensor([3.0, 1.5]),
+ }
+
+
+def test_prepare_auxiliary_data_returns_none_without_modulation_config():
+ cfg = object()
+ assert prepare_auxiliary_data(_batch_data(), cfg) is None # type: ignore[arg-type]
+
+
+def test_prepare_auxiliary_data_returns_none_without_modulation():
+ cfg = _Cfg(conv_modulation=ModConvType.NONE, aux_in_features=2, log_aux=False)
+ assert prepare_auxiliary_data(_batch_data(), cfg) is None
+
+
+def test_prepare_auxiliary_data_for_adain_without_modulation():
+ @dataclass
+ class _AdaINCfg:
+ conv_modulation: ModConvType = ModConvType.NONE
+ aux_in_features: int = 2
+ log_aux: bool = True
+ auxiliary_features: Optional[tuple[str, ...]] = None
+ image_unet_norm_type: str = "ADAIN"
+
+ auxiliary_data = prepare_auxiliary_data(_batch_data(), _AdaINCfg())
+ assert auxiliary_data is not None
+ assert auxiliary_data.shape == (2, 2)
+
+
+def test_prepare_auxiliary_data_two_features_without_log():
+ cfg = _Cfg(conv_modulation=ModConvType.FEATURES, aux_in_features=2, log_aux=False)
+ auxiliary_data = prepare_auxiliary_data(_batch_data(), cfg)
+
+ assert auxiliary_data is not None
+ assert auxiliary_data.shape == (2, 2)
+ torch.testing.assert_close(
+ auxiliary_data,
+ torch.tensor([[4.0, 0.08], [8.0, 0.04]]),
+ )
+
+
+def test_prepare_auxiliary_data_two_features_with_log():
+ cfg = _Cfg(conv_modulation=ModConvType.FEATURES, aux_in_features=2, log_aux=True)
+ auxiliary_data = prepare_auxiliary_data(_batch_data(), cfg)
+
+ assert auxiliary_data is not None
+ torch.testing.assert_close(
+ auxiliary_data,
+ torch.log(torch.tensor([[4.0, 8.0], [8.0, 4.0]])),
+ )
+
+
+def test_prepare_auxiliary_data_three_features():
+ cfg = _Cfg(conv_modulation=ModConvType.FEATURES, aux_in_features=3, log_aux=False)
+ auxiliary_data = prepare_auxiliary_data(_batch_data(), cfg)
+
+ assert auxiliary_data is not None
+ assert auxiliary_data.shape == (2, 3)
+ torch.testing.assert_close(
+ auxiliary_data,
+ torch.tensor([[4.0, 0.08, 3.0], [8.0, 0.04, 1.5]]),
+ )
+
+
+def test_prepare_auxiliary_data_explicit_feature_list():
+ cfg = _Cfg(
+ conv_modulation=ModConvType.FEATURES,
+ aux_in_features=2,
+ log_aux=False,
+ auxiliary_features=("field_strength", "acceleration"),
+ )
+ auxiliary_data = prepare_auxiliary_data(_batch_data(), cfg)
+
+ assert auxiliary_data is not None
+ torch.testing.assert_close(
+ auxiliary_data,
+ torch.tensor([[3.0, 4.0], [1.5, 8.0]]),
+ )
+
+
+def test_prepare_auxiliary_data_missing_feature_raises():
+ cfg = _Cfg(conv_modulation=ModConvType.FEATURES, aux_in_features=2, log_aux=False)
+ data = {"acceleration": torch.tensor([4.0])}
+
+ with pytest.raises(KeyError, match="center_fraction"):
+ prepare_auxiliary_data(data, cfg)
+
+
+def test_prepare_auxiliary_data_invalid_aux_in_features_raises():
+ cfg = _Cfg(conv_modulation=ModConvType.FEATURES, aux_in_features=4, log_aux=False)
+
+ with pytest.raises(ValueError, match="exceeds the number of default"):
+ prepare_auxiliary_data(_batch_data(), cfg)
+
+
+def test_resolve_auxiliary_features_unknown_name_raises():
+ with pytest.raises(ValueError, match="Unknown auxiliary feature"):
+ resolve_auxiliary_features(("acceleration", "unknown_feature"), aux_in_features=2)
+
+
+def test_resolve_auxiliary_features_length_mismatch_raises():
+ with pytest.raises(ValueError, match="auxiliary_features has length"):
+ resolve_auxiliary_features(("acceleration",), aux_in_features=2)
+
+
+def test_resolve_auxiliary_features_non_positive_raises():
+ with pytest.raises(ValueError, match="aux_in_features must be positive"):
+ resolve_auxiliary_features(None, aux_in_features=0)
+
+
+def test_register_auxiliary_feature():
+ feature = AuxiliaryFeature("custom_feature")
+ register_auxiliary_feature(feature)
+ resolved = resolve_auxiliary_features(("custom_feature",), aux_in_features=1)
+ assert resolved[0].key == "custom_feature"
+
+
+def test_prepare_auxiliary_data_string_modulation():
+ cfg = _Cfg(conv_modulation="features", aux_in_features=2, log_aux=False) # type: ignore[arg-type]
+ auxiliary_data = prepare_auxiliary_data(_batch_data(), cfg)
+ assert auxiliary_data is not None
+ assert auxiliary_data.shape == (2, 2)
+
+
+def test_prepare_auxiliary_data_missing_aux_in_features():
+ class _CfgNoAux:
+ conv_modulation = ModConvType.FEATURES
+ aux_in_features = None
+ log_aux = False
+ auxiliary_features = None
+
+ assert prepare_auxiliary_data(_batch_data(), _CfgNoAux()) is None
+
+
+def test_prepare_auxiliary_data_scalar_feature():
+ cfg = _Cfg(conv_modulation=ModConvType.FEATURES, aux_in_features=1, log_aux=False)
+ data = {"acceleration": torch.tensor(4.0)}
+ auxiliary_data = prepare_auxiliary_data(data, cfg)
+ assert auxiliary_data is not None
+ assert auxiliary_data.shape == (1, 1)
+
+
+def test_prepare_auxiliary_data_column_feature():
+ cfg = _Cfg(conv_modulation=ModConvType.FEATURES, aux_in_features=1, log_aux=False)
+ data = {"acceleration": torch.tensor([[4.0], [8.0]])}
+ auxiliary_data = prepare_auxiliary_data(data, cfg)
+ assert auxiliary_data is not None
+ torch.testing.assert_close(auxiliary_data, torch.tensor([[4.0], [8.0]]))
+
+
+def test_prepare_auxiliary_data_invalid_feature_shape_raises():
+ cfg = _Cfg(conv_modulation=ModConvType.FEATURES, aux_in_features=1, log_aux=False)
+ data = {"acceleration": torch.randn(2, 4, 4)}
+ with pytest.raises(ValueError, match="Expected auxiliary feature"):
+ prepare_auxiliary_data(data, cfg)
+
+
+def test_prepare_auxiliary_data_explicit_features_override():
+ cfg = _Cfg(conv_modulation=ModConvType.FEATURES, aux_in_features=1, log_aux=False)
+ features = (AuxiliaryFeature("acceleration"),)
+ auxiliary_data = prepare_auxiliary_data(_batch_data(), cfg, features=features)
+ assert auxiliary_data is not None
+ torch.testing.assert_close(auxiliary_data, torch.tensor([[4.0], [8.0]]))
diff --git a/tests/tests_nn/modulated_conv_test.py b/tests/tests_nn/modulated_conv_test.py
new file mode 100644
index 00000000..c4c9e27f
--- /dev/null
+++ b/tests/tests_nn/modulated_conv_test.py
@@ -0,0 +1,412 @@
+# Copyright 2026 AI for Oncology Research Group. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import pytest
+import torch
+
+from direct.nn.conv.conv import Conv2d
+from direct.nn.conv.modulated import (
+ ModConv2d,
+ ModConv2dBias,
+ ModConv3d,
+ ModConvActivation,
+ ModConvTranspose2d,
+ ModConvTranspose3d,
+ ModConvType,
+ ModulationParams,
+ mod_conv2d,
+ mod_conv3d,
+ mod_conv_transpose2d,
+ mod_conv_transpose3d,
+)
+from direct.nn.didn.didn import DIDN
+from direct.nn.mwcnn.mwcnn import MWCNN
+from direct.nn.unet.unet_2d import UnetModel2d
+
+MODULATION_TYPES = [
+ ModConvType.NONE,
+ ModConvType.FEATURES,
+ ModConvType.PARTIAL_IN,
+ ModConvType.PARTIAL_OUT,
+ ModConvType.SUM,
+]
+
+
+@pytest.mark.parametrize("modulation", MODULATION_TYPES)
+@pytest.mark.parametrize("bias", [ModConv2dBias.PARAM, ModConv2dBias.NONE])
+def test_modconv2d(modulation, bias):
+ batch, in_ch, out_ch, h, w = 2, 4, 8, 16, 16
+ aux_feat = 3
+ x = torch.randn(batch, in_ch, h, w)
+ y = torch.randn(batch, aux_feat)
+
+ kwargs = dict(
+ in_channels=in_ch,
+ out_channels=out_ch,
+ kernel_size=3,
+ padding=1,
+ bias=bias,
+ modulation=modulation,
+ )
+ if modulation != ModConvType.NONE:
+ kwargs.update(aux_in_features=aux_feat, fc_hidden_features=(16,), fc_groups=1)
+ if modulation == ModConvType.SUM:
+ kwargs["num_weights"] = 3
+
+ model = ModConv2d(**kwargs)
+ if modulation == ModConvType.NONE:
+ out = model(x)
+ else:
+ out = model(x, y)
+
+ assert out.shape == (batch, out_ch, h, w)
+
+
+@pytest.mark.parametrize("modulation", MODULATION_TYPES)
+def test_modconv_transpose2d(modulation):
+ batch, in_ch, out_ch, h, w = 2, 8, 4, 8, 8
+ aux_feat = 3
+ x = torch.randn(batch, in_ch, h, w)
+ y = torch.randn(batch, aux_feat)
+
+ kwargs = dict(
+ in_channels=in_ch,
+ out_channels=out_ch,
+ kernel_size=2,
+ stride=2,
+ modulation=modulation,
+ )
+ if modulation != ModConvType.NONE:
+ kwargs.update(aux_in_features=aux_feat, fc_hidden_features=(16,), fc_groups=1)
+ if modulation == ModConvType.SUM:
+ kwargs["num_weights"] = 3
+
+ model = ModConvTranspose2d(**kwargs)
+ if modulation == ModConvType.NONE:
+ out = model(x)
+ else:
+ out = model(x, y)
+
+ assert out.shape == (batch, out_ch, h * 2, w * 2)
+
+
+@pytest.mark.parametrize("modulation", [ModConvType.NONE, ModConvType.FEATURES, ModConvType.SUM])
+def test_modconv3d(modulation):
+ batch, in_ch, out_ch, d, h, w = 2, 4, 8, 4, 8, 8
+ aux_feat = 3
+ x = torch.randn(batch, in_ch, d, h, w)
+ y = torch.randn(batch, aux_feat)
+
+ kwargs = dict(
+ in_channels=in_ch,
+ out_channels=out_ch,
+ kernel_size=3,
+ padding=1,
+ modulation=modulation,
+ )
+ if modulation != ModConvType.NONE:
+ kwargs.update(aux_in_features=aux_feat, fc_hidden_features=(16,), fc_groups=1)
+ if modulation == ModConvType.SUM:
+ kwargs["num_weights"] = 3
+
+ model = ModConv3d(**kwargs)
+ if modulation == ModConvType.NONE:
+ out = model(x)
+ else:
+ out = model(x, y)
+
+ assert out.shape == (batch, out_ch, d, h, w)
+
+
+@pytest.mark.parametrize("modulation", [ModConvType.NONE, ModConvType.FEATURES, ModConvType.SUM])
+def test_modconv_transpose3d(modulation):
+ batch, in_ch, out_ch, d, h, w = 2, 8, 4, 4, 4, 4
+ aux_feat = 3
+ x = torch.randn(batch, in_ch, d, h, w)
+ y = torch.randn(batch, aux_feat)
+
+ kwargs = dict(
+ in_channels=in_ch,
+ out_channels=out_ch,
+ kernel_size=2,
+ stride=2,
+ modulation=modulation,
+ )
+ if modulation != ModConvType.NONE:
+ kwargs.update(aux_in_features=aux_feat, fc_hidden_features=(16,), fc_groups=1)
+ if modulation == ModConvType.SUM:
+ kwargs["num_weights"] = 3
+
+ model = ModConvTranspose3d(**kwargs)
+ if modulation == ModConvType.NONE:
+ out = model(x)
+ else:
+ out = model(x, y)
+
+ assert out.shape == (batch, out_ch, d * 2, h * 2, w * 2)
+
+
+@pytest.mark.parametrize("modulation", [ModConvType.NONE, ModConvType.FEATURES])
+def test_conv2d_cascade(modulation):
+ batch, in_ch, out_ch, h, w = 2, 2, 4, 16, 16
+ aux_feat = 3
+ x = torch.randn(batch, in_ch, h, w)
+ y = torch.randn(batch, aux_feat)
+
+ kwargs = dict(
+ in_channels=in_ch,
+ out_channels=out_ch,
+ hidden_channels=8,
+ n_convs=3,
+ modulation=modulation,
+ )
+ if modulation != ModConvType.NONE:
+ kwargs.update(aux_in_features=aux_feat, fc_hidden_features=(16,), fc_groups=1)
+
+ model = Conv2d(**kwargs)
+ if modulation == ModConvType.NONE:
+ out = model(x)
+ else:
+ out = model(x, y)
+
+ assert out.shape == (batch, out_ch, h, w)
+
+
+@pytest.mark.parametrize("modulation", [ModConvType.NONE, ModConvType.FEATURES])
+def test_didn(modulation):
+ batch, in_ch, out_ch, h, w = 1, 2, 2, 32, 32
+ aux_feat = 3
+ x = torch.randn(batch, in_ch, h, w)
+ y = torch.randn(batch, aux_feat)
+
+ kwargs = dict(
+ in_channels=in_ch,
+ out_channels=out_ch,
+ hidden_channels=8,
+ num_dubs=2,
+ num_convs_recon=3,
+ modulation=modulation,
+ )
+ if modulation != ModConvType.NONE:
+ kwargs.update(aux_in_features=aux_feat, fc_hidden_features=(8,), fc_groups=1)
+
+ model = DIDN(**kwargs)
+ if modulation == ModConvType.NONE:
+ out = model(x)
+ else:
+ out = model(x, y)
+
+ assert out.shape == (batch, out_ch, h, w)
+
+
+@pytest.mark.parametrize("modulation", [ModConvType.NONE, ModConvType.FEATURES])
+def test_mwcnn(modulation):
+ batch, in_ch, h, w = 1, 2, 32, 32
+ aux_feat = 3
+ x = torch.randn(batch, in_ch, h, w)
+ y = torch.randn(batch, aux_feat)
+
+ kwargs = dict(
+ input_channels=in_ch,
+ first_conv_hidden_channels=8,
+ num_scales=2,
+ modulation=modulation,
+ )
+ if modulation != ModConvType.NONE:
+ kwargs.update(aux_in_features=aux_feat, fc_hidden_features=(8,), fc_groups=1)
+
+ model = MWCNN(**kwargs)
+ if modulation == ModConvType.NONE:
+ out = model(x)
+ else:
+ out = model(x, y)
+
+ assert out.shape == (batch, in_ch, h, w)
+
+
+@pytest.mark.parametrize("modulation", [ModConvType.NONE, ModConvType.FEATURES])
+def test_modconv2d_gradient_flow(modulation):
+ batch, in_ch, out_ch, h, w = 2, 4, 8, 8, 8
+ aux_feat = 3
+ x = torch.randn(batch, in_ch, h, w, requires_grad=True)
+ y = torch.randn(batch, aux_feat, requires_grad=True)
+
+ kwargs = dict(
+ in_channels=in_ch,
+ out_channels=out_ch,
+ kernel_size=3,
+ padding=1,
+ modulation=modulation,
+ )
+ if modulation != ModConvType.NONE:
+ kwargs.update(aux_in_features=aux_feat, fc_hidden_features=(16,), fc_groups=1)
+
+ model = ModConv2d(**kwargs)
+ if modulation == ModConvType.NONE:
+ out = model(x)
+ else:
+ out = model(x, y)
+
+ loss = out.sum()
+ loss.backward()
+
+ assert x.grad is not None
+ if modulation != ModConvType.NONE:
+ assert y.grad is not None
+
+
+@pytest.mark.parametrize("fc_groups", [1, 2])
+def test_modconv2d_fc_groups(fc_groups):
+ batch, in_ch, out_ch, h, w = 2, 4, 8, 8, 8
+ aux_feat = 3
+ x = torch.randn(batch, in_ch, h, w)
+ y = torch.randn(batch, aux_feat)
+
+ model = ModConv2d(
+ in_channels=in_ch,
+ out_channels=out_ch,
+ kernel_size=3,
+ padding=1,
+ modulation=ModConvType.FEATURES,
+ aux_in_features=aux_feat,
+ fc_hidden_features=(16,),
+ fc_groups=fc_groups,
+ )
+ out = model(x, y)
+ assert out.shape == (batch, out_ch, h, w)
+
+
+def test_modconv2d_learned_bias():
+ batch, in_ch, out_ch, h, w = 2, 4, 8, 8, 8
+ aux_feat = 3
+ x = torch.randn(batch, in_ch, h, w)
+ y = torch.randn(batch, aux_feat)
+
+ model = ModConv2d(
+ in_channels=in_ch,
+ out_channels=out_ch,
+ kernel_size=3,
+ padding=1,
+ modulation=ModConvType.FEATURES,
+ bias=ModConv2dBias.LEARNED,
+ aux_in_features=aux_feat,
+ fc_hidden_features=(16,),
+ fc_groups=1,
+ )
+ out = model(x, y)
+ assert out.shape == (batch, out_ch, h, w)
+
+
+@pytest.mark.parametrize(
+ "modulation",
+ [ModConvType.FULL, ModConvType.PARTIAL_IN, ModConvType.PARTIAL_OUT],
+)
+@pytest.mark.parametrize("fc_groups", [1, 2])
+def test_modconv2d_weight_modulation_types(modulation, fc_groups):
+ batch, in_ch, out_ch, h, w = 2, 4, 8, 16, 16
+ aux_feat = 3
+ x = torch.randn(batch, in_ch, h, w)
+ y = torch.randn(batch, aux_feat)
+
+ model = ModConv2d(
+ in_channels=in_ch,
+ out_channels=out_ch,
+ kernel_size=3,
+ padding=1,
+ modulation=modulation,
+ aux_in_features=aux_feat,
+ fc_hidden_features=(16, 8),
+ fc_groups=fc_groups,
+ fc_activation=ModConvActivation.SOFTPLUS,
+ )
+ out = model(x, y)
+ assert out.shape == (batch, out_ch, h, w)
+
+
+def test_mod_conv_factories():
+ params = ModulationParams(
+ modulation=ModConvType.FEATURES,
+ aux_in_features=2,
+ fc_hidden_features=(8,),
+ fc_activation=ModConvActivation.SOFTPLUS,
+ )
+ x2d = torch.randn(1, 4, 8, 8)
+ y = torch.randn(1, 2)
+ x3d = torch.randn(1, 4, 4, 8, 8)
+
+ conv2d = mod_conv2d(4, 8, kernel_size=3, padding=1, modulation_params=params)
+ assert conv2d(x2d, y).shape == (1, 8, 8, 8)
+
+ conv_t2d = mod_conv_transpose2d(8, 4, kernel_size=2, stride=2, modulation_params=params)
+ assert conv_t2d(torch.randn(1, 8, 8, 8), y).shape == (1, 4, 16, 16)
+
+ conv3d = mod_conv3d(4, 8, kernel_size=3, padding=1, modulation_params=params)
+ assert conv3d(x3d, y).shape == (1, 8, 4, 8, 8)
+
+ conv_t3d = mod_conv_transpose3d(8, 4, kernel_size=2, stride=2, modulation_params=params)
+ assert conv_t3d(torch.randn(1, 8, 4, 8, 8), y).shape == (1, 4, 8, 16, 16)
+
+
+def test_modconv2d_init_validation():
+ with pytest.raises(ValueError, match="aux_in_features"):
+ ModConv2d(4, 8, 3, modulation=ModConvType.FEATURES)
+ with pytest.raises(ValueError, match="fc_hidden_features"):
+ ModConv2d(4, 8, 3, modulation=ModConvType.FEATURES, aux_in_features=2)
+ with pytest.raises(ValueError, match="num_weights"):
+ ModConv2d(
+ 4,
+ 8,
+ 3,
+ modulation=ModConvType.SUM,
+ aux_in_features=2,
+ fc_hidden_features=(8,),
+ )
+ with pytest.raises(ValueError, match="LEARNED requires modulation"):
+ ModConv2d(4, 8, 3, bias=ModConv2dBias.LEARNED)
+
+
+def test_unet2d_modulated_forward():
+ model = UnetModel2d(
+ in_channels=2,
+ out_channels=2,
+ num_filters=8,
+ num_pool_layers=2,
+ dropout_probability=0.0,
+ modulation=ModConvType.FEATURES,
+ aux_in_features=2,
+ fc_hidden_features=(8,),
+ )
+ x = torch.randn(1, 2, 32, 32)
+ y = torch.randn(1, 2)
+ out = model(x, y)
+ assert out.shape == x.shape
+
+
+def test_didn_modulation_params():
+ params = ModulationParams(
+ modulation=ModConvType.FEATURES,
+ aux_in_features=2,
+ fc_hidden_features=(8,),
+ )
+ model = DIDN(
+ in_channels=2,
+ out_channels=2,
+ hidden_channels=8,
+ num_dubs=2,
+ num_convs_recon=3,
+ modulation_params=params,
+ )
+ x = torch.randn(1, 2, 32, 32)
+ y = torch.randn(1, 2)
+ out = model(x, y)
+ assert out.shape == x.shape
diff --git a/tests/tests_utils/dataset_field_strength_test.py b/tests/tests_utils/dataset_field_strength_test.py
new file mode 100644
index 00000000..0a8d39f9
--- /dev/null
+++ b/tests/tests_utils/dataset_field_strength_test.py
@@ -0,0 +1,50 @@
+# Copyright 2026 AI for Oncology Research Group. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tests for field-strength parsing from dataset filenames."""
+
+import numpy as np
+import pytest
+
+from direct.utils.dataset import maybe_attach_field_strength, parse_field_strength_tesla
+
+
+@pytest.mark.parametrize(
+ ("filename", "expected"),
+ [
+ ("2025_train_Center005_UIH_15T_umr670_P001_lge_lax_2ch.mat", 1.5),
+ ("2025_train_Center001_UIH_30T_umr780_P001_cine_sax.mat", 3.0),
+ ("subject_055T_scan.mat", 0.55),
+ ("scanner_70T_data.h5", 7.0),
+ ("prefix_7T_suffix.h5", 7.0),
+ ("/data/path/Center_15T_file.mat", 1.5),
+ ("file1000000.h5", None),
+ ("no_field_token.h5", None),
+ ("P015_cine.mat", None), # digits without trailing T
+ ],
+)
+def test_parse_field_strength_tesla(filename, expected):
+ assert parse_field_strength_tesla(filename) == expected
+
+
+def test_maybe_attach_field_strength_adds_key():
+ sample = {"filename": "2025_train_Center001_UIH_30T_umr780_P001.mat"}
+ maybe_attach_field_strength(sample)
+ assert "field_strength" in sample
+ np.testing.assert_allclose(sample["field_strength"], np.array([3.0]))
+
+
+def test_maybe_attach_field_strength_skips_when_absent():
+ sample = {"filename": "file1000000.h5"}
+ maybe_attach_field_strength(sample)
+ assert "field_strength" not in sample