Adding univariate processing steps

src/meteaudata/__init__.py

@@ -6,6 +6,10 @@
 )
 from meteaudata.processing_steps.univariate.replace import replace_ranges  # noqa: F401
 from meteaudata.processing_steps.univariate.resample import resample  # noqa: F401
+from meteaudata.processing_steps.univariate.select_time_range import select_time_range  # noqa: F401
+from meteaudata.processing_steps.univariate.check_missing_values import check_missing_values  # noqa: F401
+from meteaudata.processing_steps.univariate.remove_duplicates import remove_duplicates  # noqa: F401
+
 from meteaudata.processing_steps.univariate.subset import subset  # noqa: F401
 from meteaudata.types import (  # noqa: F401
     DataProvenance,

src/meteaudata/processing_steps/univariate/check_missing_values.py

@@ -0,0 +1,60 @@
+import datetime
+import pandas as pd
+from meteaudata.types import (
+    FunctionInfo,
+    Parameters,
+    ProcessingStep,
+    ProcessingType,
+)
+
+def check_missing_values(
+    input_series: list[pd.Series], *args, **kwargs
+) -> list[tuple[pd.Series, list[ProcessingStep]]]:
+    """
+    A processing function to check for missing values in time series data.
+
+    The function checks for any missing (NaN) values in the input series.
+
+    Args:
+        input_series (list[pd.Series]): List of input time series to be processed.
+
+    Returns:
+        list[tuple[pd.Series, list[ProcessingStep]]]: List of series with metadata, marking missing value detection.
+    """
+
+    func_info = FunctionInfo(
+        name="check_missing_values",
+        version="0.1",
+        author="Loes Verhaeghe",
+        reference="Loes Verhaeghe with the help of chat gpt",
+    )
+
+    processing_step = ProcessingStep(
+        type=ProcessingType.SORTING,
+        parameters=Parameters(),  # No specific parameters for missing value check
+        function_info=func_info,
+        description="A processing function to check for missing values in a time series",
+        run_datetime=datetime.datetime.now(),
+        requires_calibration=False,
+        input_series_names=[str(col.name) for col in input_series],
+        suffix="CheckedMissingValues",
+    )
+
+    outputs = []
+
+    for col in input_series:
+        col = col.copy()
+        col_name = col.name
+        signal, _ = str(col_name).split("_")
+
+        missing_count = col.isnull().sum()
+        print(f"Series '{col_name}' has {missing_count} missing values.")
+
+        # Update the series name with the processing step suffix
+        new_name = f"{signal}_{processing_step.suffix}"
+        col.name = new_name
+
+        # Append the series along with the processing step metadata
+        outputs.append((col, [processing_step]))
+
+    return outputs

src/meteaudata/processing_steps/univariate/remove_duplicates.py

@@ -0,0 +1,66 @@
+import datetime
+import pandas as pd
+from meteaudata.types import (
+    FunctionInfo,
+    Parameters,
+    ProcessingStep,
+    ProcessingType,
+)
+
+def remove_duplicates(
+    input_series: list[pd.Series], *args, **kwargs
+) -> list[tuple[pd.Series, list[ProcessingStep]]]:
+    """
+    A processing function to remove duplicate sample points from time series data.
+
+    The function checks for duplicates and retains only the first occurrence of each duplicate.
+
+    Args:
+        input_series (list[pd.Series]): List of input time series to be processed.
+
+    Returns:
+        list[tuple[pd.Series, list[ProcessingStep]]]: Time series with duplicates removed, including metadata about the processing steps.
+    """
+
+    func_info = FunctionInfo(
+        name="remove_duplicates",
+        version="0.1",
+        author="Loes Verhaeghe",
+        reference="Loes Verhaeghe with the help of chat gpt",
+    )
+
+    processing_step = ProcessingStep(
+        type=ProcessingType.RESAMPLING,
+        parameters=None,
+        function_info=func_info,
+        description="A processing function to remove duplicate sample points from time series",
+        run_datetime=datetime.datetime.now(),
+        requires_calibration=False,
+        input_series_names=[str(col.name) for col in input_series],
+        suffix="NoDuplicates",
+    )
+
+    outputs = []
+
+    for col in input_series:
+        col = col.copy()
+        col_name = col.name
+        signal, _ = str(col_name).split("_")
+
+        # Ensure the series has a proper datetime index
+        if not isinstance(col.index, pd.DatetimeIndex):
+            raise IndexError(
+                f"Series {col.name} has index type {type(col.index)}. Please provide pd.DatetimeIndex."
+            )
+
+        # Remove duplicate values while keeping the first occurrence
+        filtered_col = col.loc[~col.index.duplicated(keep='first')]
+
+        # Update the series name with the processing step suffix
+        new_name = f"{signal}_{processing_step.suffix}"
+        filtered_col.name = new_name
+
+        # Append the filtered series along with the processing step metadata
+        outputs.append((filtered_col, [processing_step]))
+
+    return outputs

src/meteaudata/processing_steps/univariate/resample.py

@@ -39,7 +39,7 @@ def resample(
     for col in input_series:
         col = col.copy()
         col_name = col.name
-        signal, _ = str(col_name).split("_")
+        signal = "_".join(str(col_name).split("_")[:-1])
         if not isinstance(col.index, (pd.DatetimeIndex, pd.TimedeltaIndex)):
             raise IndexError(
                 f"Series {col.name} has index type {type(col.index)}. Please provide either pd.DatetimeIndex or pd.TimedeltaIndex"

src/meteaudata/processing_steps/univariate/select_time_range.py

@@ -0,0 +1,76 @@
+import datetime
+import pandas as pd
+from meteaudata.types import (
+    FunctionInfo,
+    Parameters,
+    ProcessingStep,
+    ProcessingType,
+)
+
+def select_time_range(
+
+    input_series: list[pd.Series], start_time: str, end_time: str, *args, **kwargs
+
+) -> list[tuple[pd.Series, list[ProcessingStep]]]:
+
+    """
+    A processing function to filter time series data within a specified time range.
+
+    The function accepts a start and end time, and filters the data accordingly.
+
+    Args:
+        input_series (list[pd.Series]): List of input time series to be processed.
+        start_time (str): Start of the time range (e.g., "2023-10-01 00:00:00").
+        end_time (str): End of the time range (e.g., "2023-10-20 00:00:00").
+
+    Returns:
+        list[tuple[pd.Series, list[ProcessingStep]]]: Filtered time series with metadata about the processing steps.
+    """
+
+    func_info = FunctionInfo(
+        name="select_time_range",
+        version="0.1",
+        author="Loes Verhaeghe",
+        reference="Loes Verhaeghe with the help of chat gpt",
+    )
+
+    parameters = Parameters(start_time=start_time, end_time=end_time)
+
+    processing_step = ProcessingStep(
+        type=ProcessingType.SORTING,
+        parameters=parameters,
+        function_info=func_info,
+        description="A processing function to select data between a specific time range",
+        run_datetime=datetime.datetime.now(),
+        requires_calibration=False,
+        input_series_names=[str(col.name) for col in input_series],
+        suffix="SelectedTimeRange",
+    )
+
+    outputs = []
+
+    start_time = pd.to_datetime(start_time)
+    end_time = pd.to_datetime(end_time)
+
+    for col in input_series:
+        col = col.copy()
+        col_name = col.name
+        signal, _ = str(col_name).split("_")
+
+        # Ensure the series has a proper datetime index
+        if not isinstance(col.index, pd.DatetimeIndex):
+            raise IndexError(
+                f"Series {col.name} has index type {type(col.index)}. Please provide pd.DatetimeIndex."
+            )
+
+        # Filter the data based on the given time range
+        filtered_col = col[(col.index >= start_time) & (col.index <= end_time)]
+
+        # Update the series name with the processing step suffix
+        new_name = f"{signal}_{processing_step.suffix}"
+        filtered_col.name = new_name
+
+        # Append the filtered series along with the processing step metadata
+        outputs.append((filtered_col, [processing_step]))
+
+    return outputs

src/meteaudata/types.py

@@ -13,7 +13,9 @@
 import pandas as pd
 import plotly.graph_objects as go
 import plotly.io as pio
+pio.renderers.default = "vscode" 
 import yaml
+import matplotlib.pyplot as plt
 from plotly.subplots import make_subplots
 from pydantic import (
     BaseModel,
@@ -1011,6 +1013,44 @@ def plot(
             yaxis_title=y_axis,
         )
         return fig
+
+    def plot_fast(
+        self,
+        ts_names: list[str],
+        title: Optional[str] = None,
+        y_axis: Optional[str] = None,
+        x_axis: Optional[str] = None,
+    ) -> None:
+        # Set default titles if not provided
+        if not title:
+            title = f"Time series plot of {self.name}"
+        if not y_axis:
+            y_axis = f"{self.name} ({self.units})"
+        if not x_axis:
+            x_axis = "Time"
+
+        # Create a figure and axis
+        plt.figure(figsize=(12, 6))
+
+        # Loop through time series names and plot each one
+        for ts_name in ts_names:
+            ts = self.time_series[ts_name].series
+            # Assuming the time series `ts` has a pandas Series structure
+            plt.plot(ts, label=ts_name)
+
+        # Add title and labels
+        plt.title(title)
+        plt.xlabel(x_axis)
+        plt.ylabel(y_axis)
+
+        # Show legend
+        plt.legend()
+
+        # Display the plot
+        plt.tight_layout()
+        plt.show()        
+
+        return
 
     def build_dependency_graph(self, ts_name: str) -> list[dict[str, Any]]:
         """