Feature/add mdposit scraper

README.md

@@ -171,6 +171,26 @@ This command will:
 5. Save the extracted metadata to Parquet files
 
 
+## Scrape MDposit
+
+Have a look to the notes regarding [MDposit](docs/mdposit.md) and its API.
+
+Scrape MDposit to collect molecular dynamics (MD) datasets and files:
+
+```bash
+uv run scrape-mdposit --output-dir data
+```
+
+This command will:
+
+1. Search for molecular dynamics entries and files through the MDposit API.
+2. Parse metadata and validate them using the Pydantic models
+   `DatasetMetadata` and `FileMetadata`.
+3. Save validated files and datasets metadata.
+
+The scraping takes about 13 minutes.
+
+
 ## Analyze Gromacs mdp and gro files
 
 ### Download files

pyproject.toml

@@ -49,3 +49,5 @@ build-backend = "uv_build"
 scrape-zenodo = "mdverse_scrapers.scrapers.zenodo:main"
 scrape-figshare = "mdverse_scrapers.scrapers.figshare:main"
 scrape-nomad = "mdverse_scrapers.scrapers.nomad:main"
+scrape-mdposit = "mdverse_scrapers.scrapers.mdposit:main"
+

src/mdverse_scrapers/models/dataset.py

@@ -50,14 +50,14 @@ class DatasetMetadata(BaseModel):
         ...,
         description=(
             "Name of the source repository. "
-            "Allowed values: ZENODO, FIGSHARE, OSF, NOMAD, ATLAS, GPCRMD."
+            "Allowed values: ZENODO, FIGSHARE, OSF, NOMAD, ATLAS, GPCRMD, MDPOSIT."
         ),
     )
     dataset_project_name: DatasetProjectName | None = Field(
         None,
         description=(
             "Name of the project."
-            "Allowed values: ZENODO, FIGSHARE, OSF, NOMAD, ATLAS, GPCRMD."
+            "Allowed values: ZENODO, FIGSHARE, OSF, NOMAD, ATLAS, GPCRMD, MDDB."
         ),
     )
     dataset_id_in_repository: str = Field(
@@ -190,12 +190,12 @@ class DatasetMetadata(BaseModel):
     @field_validator(
         "date_created", "date_last_updated", "date_last_fetched", mode="before"
     )
-    def format_dates(cls, v: datetime | str) -> str:  # noqa: N805
+    def format_dates(cls, v: datetime | str | None) -> str | None:  # noqa: N805
         """Convert datetime objects or ISO strings to '%Y-%m-%dT%H:%M:%S' format.
 
         Parameters
         ----------
-        cls : type[BaseDataset]
+        cls : type[DatasetMetadata]
             The Pydantic model class being validated.
         v : str
             The input value of the 'date' field to validate.
@@ -205,6 +205,8 @@ def format_dates(cls, v: datetime | str) -> str:  # noqa: N805
         str:
             The date in '%Y-%m-%dT%H:%M:%S' format.
         """
+        if v is None:
+            return None
         if isinstance(v, datetime):
             return v.strftime("%Y-%m-%dT%H:%M:%S")
         return datetime.fromisoformat(v).strftime("%Y-%m-%dT%H:%M:%S")

src/mdverse_scrapers/models/enums.py

@@ -19,6 +19,7 @@ class DatasetRepositoryName(StrEnum):
     NOMAD = "nomad"
     ATLAS = "atlas"
     GPCRMD = "gpcrmd"
+    MDPOSIT = "mdposit"
 
 
 class DatasetProjectName(StrEnum):
@@ -31,3 +32,4 @@ class DatasetProjectName(StrEnum):
     ATLAS = "atlas"
     GPCRMD = "gpcrmd"
     NMRLIPIDS = "nmrlipids"
+    MDDB = "mddb"