diff --git a/autoprep/preprocessor.py b/autoprep/preprocessor.py new file mode 100644 index 0000000..0cb3337 --- /dev/null +++ b/autoprep/preprocessor.py @@ -0,0 +1,63 @@ +from scipy import stats +import pandas as pd + + +class Preprocessor: + def __init__(self, series: pd.Series): + print(series.dtype.name) + if series.dtype.name == 'int64': + self.prep = NumPreprocessor() + elif series.dtype.name == 'float64': + self.prep = NumPreprocessor() + elif series.dtype.name == 'bool': + self.prep = BoolPreprocessor() + elif series.dtype.name == 'datetime64': + self.prep = DatePreprocessor() + else: + self.prep = CatPreprocessor() + self.series = series + + def apply(self): + return self.prep.apply(self.series) + + +class NumPreprocessor: + @staticmethod + def _fill(series: pd.Series): + return series.fillna(series.median()) + + @staticmethod + def _normarize(series: pd.Series): + return pd.Series(stats.zscore(series)) + + def apply(self, series: pd.Series): + return self._normarize(self._fill(series)) + + +class BoolPreprocessor: + @staticmethod + def _fill(series): + pass + + def apply(self, series): + pass + + +class DatePreprocessor: + @staticmethod + def _fill(series): + pass + + def apply(self, series): + pass + + +class CatPreprocessor: + def _fill(self): + pass + + def _encoding(self): + pass + + def apply(self, series): + pass diff --git a/setup.py b/setup.py index 5888828..0c8a467 100644 --- a/setup.py +++ b/setup.py @@ -18,6 +18,6 @@ def read(filename): packages=['autoprep'], long_description=read('README.md'), long_description_content_type="text/markdown", - install_requires=['numpy', 'pandas'], + install_requires=['numpy', 'scipy', 'pandas'], classifiers=[] ) diff --git a/tests/test_preprocessor.py b/tests/test_preprocessor.py new file mode 100644 index 0000000..1f348b7 --- /dev/null +++ b/tests/test_preprocessor.py @@ -0,0 +1,18 @@ +import unittest +import pandas as pd +from autoprep.preprocessor import Preprocessor + + +class TestPreprocessor(unittest.TestCase): + + def test_num_preprocessor(self): + series = pd.Series([18, 18, 20, None, 20, 22, 22, 30]) + p = Preprocessor(series) + res = p.apply() + self.assertEqual(list(res), + [-0.9035624609139906, -0.9035624609139906, -0.34752402342845795, -0.34752402342845795, + -0.34752402342845795, 0.20851441405707474, 0.20851441405707474, 2.4326681639992054]) + + +if __name__ == '__main__': + unittest.main()