Skip to content
Open
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
116 changes: 114 additions & 2 deletions medkit/text/deid.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
from __future__ import annotations

from typing import TYPE_CHECKING, Iterator
from typing import TYPE_CHECKING, ClassVar, Iterator

from presidio_analyzer import Pattern, PatternRecognizer

from medkit._import import import_optional
from medkit.core.text import Entity, span_utils
Expand All @@ -11,7 +13,7 @@

presidio_analyzer = import_optional("presidio_analyzer", extra="deid")

__all__ = ["PIIDetector"]
__all__ = ["PIIDetector", "FrDateRecognizer", "FrAgeRecognizer"]


class PIIDetector(NEROperation):
Expand All @@ -35,3 +37,113 @@ def _run_one(self, segment: Segment) -> Iterator[Entity]:
spans=spans,
metadata={"score": result.score},
)


class FrDateRecognizer(PatternRecognizer):
"""
Recognizes French Date using regex.

:param patterns: List of patterns to be used by this recognizer
:param supported_language: Language this recognizer supports
:param supported_entity: The entity this recognizer can detect
Comment thread
cpetresc marked this conversation as resolved.
Outdated
"""

PATTERNS: ClassVar[list[Pattern]] = [
Pattern(
"French dates with day month year",
r"\b(?i)(\d{1,2}|1er) "
r"((janvier|janv.|JAN)|(février|févr.|FÉV)|(mars|MAR)|"
r"(avril|avr.|AVR)|(mai|MAI)|(juin|JUN)|(juillet|juill.|JUL)|"
r"(août|AOÛ)|(septembre|sept.|SEP)|(octobre|oct.|OCT)|"
r"(novembre|nov.|NOV)|(décembre|déc|DÉC)) \d{4}\b",
0.9,
),
Pattern(
"French dates with month year",
r"\b(?i)((janvier|janv.|JAN)|(février|févr.|FÉV)|(mars|MAR)|"
r"(avril|avr.|AVR)|(mai|MAI)|(juin|JUN)|(juillet|juill.|JUL)|"
r"(août|AOÛ)|(septembre|sept.|SEP)|(octobre|oct.|OCT)|"
r"(novembre|nov.|NOV)|(décembre|déc|DÉC)) \d{4}\b",
0.9,
),
Pattern(
"French dates with day month",
r"\b(?i)(\d{1,2} |1er )"
r"((janvier|janv.|JAN)|(février|févr.|FÉV)|(mars|MAR)|"
r"(avril|avr.|AVR)|(mai|MAI)|(juin|JUN)|(juillet|juill.|JUL)|"
r"(août|AOÛ)|(septembre|sept.|SEP)|(octobre|oct.|OCT)|"
r"(novembre|nov.|NOV)|(décembre|déc|DÉC))\b",
0.9,
),
Pattern(
"French dates with month",
r"\b(?i)((janvier|janv.|JAN)|(février|févr.|FÉV)|(mars|MAR)|"
r"(avril|avr.|AVR)|(mai|MAI)|(juin|JUN)|(juillet|juill.|JUL)|"
r"(août|AOÛ)|(septembre|sept.|SEP)|(octobre|oct.|OCT)|"
r"(novembre|nov.|NOV)|(décembre|déc|DÉC))\b",
0.9,
),
]

def __init__(
self,
patterns: list[Pattern] | None = None,
supported_language: str = "fr",
supported_entity: str = "FR_DATE",
):
patterns = patterns if patterns else self.PATTERNS
Comment thread
cpetresc marked this conversation as resolved.
super().__init__(
supported_entity=supported_entity,
patterns=patterns,
supported_language=supported_language,
)


class FrAgeRecognizer(PatternRecognizer):
"""
Recognizes French Phone Age using regex.

:param patterns: List of patterns to be used by this recognizer
:param supported_language: Language this recognizer supports
:param supported_entity: The entity this recognizer can detect
Comment thread
cpetresc marked this conversation as resolved.
Outdated
"""

PATTERNS: ClassVar[list[Pattern]] = [
Pattern(
"French age written with 'number (ans|mois)' "
"ou 'number et number (ans|mois)' "
"ou 'number à number (ans|mois)' "
"ou ' number ou number (ans|mois)'",
r"\d+\s+(ans|mois)|\d+\s+et\s+\d+\s+(ans|mois)|" r"\d+\s+à\s+\d+\s+(ans|mois)|\d+\s+ou\s+\d+\s+(ans|mois)",
0.9,
),
Pattern(
"French age written with 'number ans et number mois' "
"ou 'number ans et number ou number mois' "
" ou 'number ans et number à number mois' ",
r"\d+\s+ans\s+et\s+\d+\s+mois|\d+\s+ans\s+et\s+\d+\s+à\s+\d+\s+mois|"
r"\d+\s+ans\s+et\s+\d+\s+ou\s+\d+\s+mois",
0.9,
),
Pattern(
"French age written with letters '.. (ans|mois)' "
"ou ' .. et .. ans' ou ' .. à .. ans' ou ' .. ou .. ans' ",
r"((dix|vingt|trente|quarante|cinquante|soixante|soixante-dix|quatre-vingt|quatre-vingt-dix|cent)"
r"(\s+|-|-et-)?)?(un|deux|trois|quatre|cinq|six|sept|huit|neuf|dix|onze|douze|treize|quatorze"
r"|quinze|seize)?\s+((et\s+|à\s+|ou\s+)\d+\s+)?(ans|mois)",
0.9,
),
]

def __init__(
self,
patterns: list[Pattern] | None = None,
supported_language: str = "fr",
supported_entity: str = "FR_AGE",
):
patterns = patterns if patterns else self.PATTERNS
super().__init__(
supported_entity=supported_entity,
patterns=patterns,
supported_language=supported_language,
)