Numeric array as atom take 2

SYMBOLS_MANIFEST.txt

@@ -845,6 +845,7 @@ System`NumberQ
 System`NumberString
 System`Numerator
 System`NumericFunction
+System`NumericArray
 System`NumericQ
 System`O
 System`Octahedron

mathics/builtin/list/constructing.py

@@ -13,10 +13,11 @@
 from typing import Optional, Tuple
 
 from mathics.builtin.box.layout import RowBox
-from mathics.core.atoms import ByteArray, Integer, Integer1, is_integer_rational_or_real
+from mathics.core.atoms import ByteArray, Integer, Integer1, is_integer_rational_or_real, NumericArray
 from mathics.core.attributes import A_HOLD_FIRST, A_LISTABLE, A_LOCKED, A_PROTECTED
 from mathics.core.builtin import BasePattern, Builtin, IterationFunction
 from mathics.core.convert.expression import to_expression
+from mathics.core.convert.python import from_python
 from mathics.core.convert.sympy import from_sympy
 from mathics.core.element import ElementsProperties
 from mathics.core.evaluation import Evaluation
@@ -198,8 +199,14 @@ class Normal(Builtin):
     def eval_general(self, expr: Expression, evaluation: Evaluation):
         "Normal[expr_]"
         if isinstance(expr, Atom):
-            if isinstance(expr, ByteArray):
-                return ListExpression(*expr.items)
+            if hasattr(expr, "items"):
+                def normal(items):
+                    return ListExpression(*(
+                        normal(item.items) if isinstance(item, Atom) and hasattr(item, "items")
+                        else item
+                        for item in items
+                    ))
+                return normal(expr.items)
             return expr
         if expr.has_form("RootSum", 2):
             return from_sympy(expr.to_sympy().doit(roots=True))

mathics/builtin/list/eol.py

@@ -698,25 +698,24 @@ def eval(self, expr, evaluation: Evaluation, expression: Expression):
             if not hasattr(expr, "items"):
                 evaluation.message("First", "normal", Integer1, expression)
                 return
-            expr_len = len(expr.items)
+            parts = expr.items
         else:
-            expr_len = len(expr.elements)
+            parts = expr.elements
+
+        expr_len = len(parts)
         if expr_len == 0:
             evaluation.message("First", "nofirst", expr)
             return
 
-        if isinstance(expr, ByteArray):
-            return expr.items[0]
-
-        if expr_len > 2 and expr.head is SymbolSequence:
+        if expr_len > 2 and expr.get_head() is SymbolSequence:
             evaluation.message(
                 "First", "argt", SymbolFirst, Integer(expr_len), Integer1, Integer2
             )
             return
 
-        first_elem = expr.elements[0]
+        first_elem = parts[0]
 
-        if expr.head == SymbolSequence or (
+        if expr.get_head() == SymbolSequence or (
             not isinstance(expr, ListExpression)
             and len == 2
             and isinstance(first_elem, Atom)

mathics/builtin/numericarray.py

@@ -0,0 +1,63 @@
+# -*- coding: utf-8 -*-
+"""Rules for working with NumericArray atoms."""
+
+from typing import Optional, Tuple
+
+try:  # pragma: no cover - numpy is optional at runtime
+    import numpy
+except ImportError:  # pragma: no cover - handled via requires attribute
+    numpy = None
+
+from mathics.core.atoms import NumericArray, NUMERIC_ARRAY_TYPE_MAP, String
+from mathics.core.builtin import Builtin
+from mathics.core.convert.python import from_python
+from mathics.core.symbols import Symbol, strip_context
+from mathics.core.systemsymbols import SymbolAutomatic, SymbolFailed, SymbolNumericArray
+
+
+# class name modeled on Complex_ to avoid collision with NumericArray atom
+class NumericArray_(Builtin):
+
+    summary_text = "construct NumericArray"
+    name = "NumericArray"
+    rules = {
+        "NumericArray[list_List]": "NumericArray[list, Automatic]"
+    }
+    messages = {
+        "type": "The type specification `1` is not supported in NumericArray.",
+    }
+
+    # rule to convert NumericArray[...nested list...] expression to NumericArray atom
+    def eval_list(self, data, typespec, evaluation):
+        "System`NumericArray[data_List, typespec_]"
+
+        # get a string key from the typespec
+        if isinstance(typespec, Symbol):
+            key = strip_context(typespec.get_name())
+        elif isinstance(typespec, String):
+            key = typespec.value
+        else:
+            evaluation.message("NumericArray", "type", typespec)            
+            return SymbolFailed
+
+        # compute numpy dtype from key
+        if key == "Automatic":
+            dtype = None
+        else:
+            dtype = NUMERIC_ARRAY_TYPE_MAP.get(key, None)
+            if not dtype:
+                evaluation.message("NumericArray", "type", typespec)
+                return SymbolFailed
+
+        # compute array from data and dtype and wrap it in a NumericArray atom
+        python_value = data.to_python()
+        array = numpy.array(python_value, dtype=dtype)
+        atom = NumericArray(array, dtype)
+
+        return atom
+
+    # this doesn't work
+    # instead see Normal builtin in mathics/builtin/list/constructing.py
+    #def eval_normal(self, array, evaluation):
+    #    "System`Normal[array_NumericArray]"
+    #    return from_python(array.value.tolist())

mathics/core/atoms.py

@@ -10,10 +10,16 @@
 import sympy
 from sympy.core import numbers as sympy_numbers
 
+try:  # pragma: no cover - optional dependency handled at runtime
+    import numpy
+except ImportError:  # pragma: no cover - numpy is optional at import time
+    numpy = None
+
 from mathics.core.element import BoxElementMixin, ImmutableValueMixin
 from mathics.core.keycomparable import (
     BASIC_ATOM_BYTEARRAY_ELT_ORDER,
     BASIC_ATOM_NUMBER_ELT_ORDER,
+    BASIC_ATOM_NUMERICARRAY_ELT_ORDER,
     BASIC_ATOM_STRING_ELT_ORDER,
 )
 from mathics.core.number import (
@@ -1098,6 +1104,125 @@ def is_zero(self) -> bool:
 }
 
 
+#
+# NumericArray
+#
+
+if numpy is not None:
+    NUMERIC_ARRAY_TYPE_MAP = {
+        "UnsignedInteger8": numpy.dtype("uint8"),
+        "UnsignedInteger16": numpy.dtype("uint16"),
+        "UnsignedInteger32": numpy.dtype("uint32"),
+        "UnsignedInteger64": numpy.dtype("uint64"),
+        "Integer8": numpy.dtype("int8"),
+        "Integer16": numpy.dtype("int16"),
+        "Integer32": numpy.dtype("int32"),
+        "Integer64": numpy.dtype("int64"),
+        "Real32": numpy.dtype("float32"),
+        "Real64": numpy.dtype("float64"),
+        "ComplexReal32": numpy.dtype("complex64"),
+        "ComplexReal64": numpy.dtype("complex128"),
+    }
+    NUMERIC_ARRAY_DTYPE_TO_NAME = {
+        dtype: name for name, dtype in NUMERIC_ARRAY_TYPE_MAP.items()
+    }
+else:  # pragma: no cover - executed only when numpy is absent
+    NUMERIC_ARRAY_TYPE_MAP = {}
+    NUMERIC_ARRAY_DTYPE_TO_NAME = {}
+
+
+# TODO: would it be useful to follow the example of Complex and parameterize by type?
+# would that be array.dtype or the MMA type from the map above?
+class NumericArray(Atom, ImmutableValueMixin):
+    """
+    NumericArray provides compact storage and efficient access for machine-precision numeric arrays,
+    backed by NumPy arrays.
+    """
+
+    class_head_name = "NumericArray"
+
+    def __init__(self, value, dtype=None):
+        if numpy is None:
+            raise ImportError("numpy is required for NumericArray")
+
+        # compute value
+        if not isinstance(value, numpy.ndarray):
+            value = numpy.asarray(value, dtype=dtype)
+        elif dtype is not None:
+            value = value.astype(dtype)
+        self.value = value
+
+        # check type
+        self._type_name = NUMERIC_ARRAY_DTYPE_TO_NAME.get(self.value.dtype, None)
+        if not self._type_name:
+            allowed = ", ".join(str(dtype) for dtype in NUMERIC_ARRAY_TYPE_MAP.values())
+            message = f"Argument 'value' must be one of {allowed}; is {str(self.value.dtype)}."
+            raise ValueError(message)
+
+        # summary and hash
+        shape_string = "×".join(str(dim) for dim in self.value.shape) or "0"
+        self._summary_string = f"{self._type_name}, {shape_string}"
+        self._hash = None
+
+    # TODO: this is potentially expensive - what if we left it unimplemented? is hashing a numpy array reasonable?
+    # TODO: to make it less expensive only look at first 100 bytes - ok? needed?
+    def __hash__(self):
+        if not self._hash:
+            self._hash = hash(("NumericArray", self.value.shape, self.value.tobytes()[:100]))
+        return self._hash
+
+    def __str__(self) -> str:
+        return f"NumericArray[{self._summary_string}]"
+
+    def atom_to_boxes(self, f, evaluation):
+        return String(f"<{self._summary_string}>")
+
+    def do_copy(self) -> "NumericArray":
+        return NumericArray(self.value.copy())
+
+    def default_format(self, evaluation, form) -> str:
+        return f"NumericArray[<{self._summary_string}>]"
+
+    @property
+    def items(self) -> Tuple:
+        from mathics.core.convert.python import from_python
+        if len(self.value.shape) == 1:
+            return tuple(from_python(item.item()) for item in self.value)
+        else:
+            return tuple(NumericArray(array) for array in self.value)
+
+    @property
+    def element_order(self) -> tuple:
+        return (
+            BASIC_ATOM_NUMERICARRAY_ELT_ORDER,
+            self.value.shape,
+            self.value.dtype,
+            self.value.tobytes()
+        )
+
+    @property
+    def pattern_precedence(self) -> tuple:
+        return super().pattern_precedence
+
+    def sameQ(self, rhs) -> bool:
+        return isinstance(rhs, NumericArray) and numpy.array_equal(self.value, rhs.value)
+
+    def to_sympy(self, **kwargs):
+        return None
+
+    # TODO: note that this returns a simple python list (of lists),
+    # not the numpy array - ok?
+    def to_python(self, *args, **kwargs):
+        return self.value.tolist()
+
+    # TODO: what is this? is it right?
+    def user_hash(self, update):
+        update(self._summary[2])
+
+    def __getnewargs__(self):
+        return (self.value, self.value.dtype)
+
+
 class String(Atom, BoxElementMixin):
     value: str
     class_head_name = "System`String"

mathics/core/convert/python.py

@@ -5,7 +5,12 @@
 
 from typing import Any
 
-from mathics.core.atoms import Complex, Integer, Rational, Real, String
+try:  # pragma: no cover - numpy is optional
+    import numpy
+except ImportError:  # pragma: no cover - optional dependency missing
+    numpy = None
+
+from mathics.core.atoms import Complex, Integer, NumericArray, Rational, Real, String
 from mathics.core.number import get_type
 from mathics.core.symbols import (
     BaseElement,
@@ -113,5 +118,7 @@ def from_python(arg: Any) -> BaseElement:
         from mathics.builtin.binary.bytearray import ByteArray
 
         return Expression(SymbolByteArray, ByteArray(arg))
+    elif numpy is not None and isinstance(arg, numpy.ndarray):
+        return NumericArray(arg)
     else:
         raise NotImplementedError

mathics/core/keycomparable.py

@@ -279,6 +279,7 @@ def __ne__(self, other) -> bool:
 BASIC_ATOM_STRING_ELT_ORDER = 0x01
 BASIC_ATOM_BYTEARRAY_ELT_ORDER = 0x02
 LITERAL_EXPRESSION_ELT_ORDER = 0x03
+BASIC_ATOM_NUMERICARRAY_ELT_ORDER = 0x04
 
 BASIC_NUMERIC_EXPRESSION_ELT_ORDER = 0x12
 GENERAL_NUMERIC_EXPRESSION_ELT_ORDER = 0x13

mathics/core/systemsymbols.py

@@ -179,6 +179,7 @@
 SymbolNumberForm = Symbol("System`NumberForm")
 SymbolNumberString = Symbol("System`NumberString")
 SymbolNumberQ = Symbol("System`NumberQ")
+SymbolNumericArray = Symbol("System`NumericArray")
 SymbolNumericQ = Symbol("System`NumericQ")
 SymbolO = Symbol("System`O")
 SymbolOpacity = Symbol("System`Opacity")

test/builtin/test_numericarray.py

@@ -0,0 +1,74 @@
+from mathics.core.atoms import Integer, NumericArray, String
+from mathics.core.convert.python import from_python
+from test.helper import check_evaluation, evaluate
+
+import numpy as np
+import pytest
+
+#
+# Python API tests
+#
+
+def test_numericarray_atom_preserves_array_reference():
+    array = np.array([1, 2, 3], dtype=np.int64)
+    atom = NumericArray(array)
+    assert atom.value is array
+
+def test_numericarray_atom_preserves_equality():
+    array = np.array([1, 2, 3], dtype=np.int64)
+    atom = NumericArray(array, dtype=np.float64)
+    np.testing.assert_array_equal(atom.value, array)
+
+
+def test_numericarray_expression_from_python_array():
+    array = np.array([[1.0, 2.0], [3.0, 4.0]], dtype=np.float32)
+    atom = from_python(array)
+    assert isinstance(atom, NumericArray)
+    assert atom.value is array
+
+def test_numericarray_hash():
+    a = [[1, 2], [3, 4]]
+    array1a = np.array(a, dtype=np.float32)
+    atom1a = from_python(array1a)
+    array1b = np.array(a, dtype=np.float32)
+    atom1b = from_python(array1a)
+    array2 = np.array(a, dtype=np.float64)
+    atom2 = from_python(array2)
+    assert hash(atom1a) == hash(atom1b), "hashes of different arrays with same value should be same"
+    assert hash(atom1a) != hash(atom2), "hashes of arrays with different values should be different"
+
+
+#
+# WL tests
+#
+
+@pytest.mark.parametrize(
+    ("str_expr", "str_expected"),
+    [
+        # TODO: remove this - was temp to see if Normal still worked on ByteArray after changes
+        #("Normal[ByteArray[{1,2}]]", "{1, 2}"),
+        ("NumericArray[{{1,2},{3,4}}]", "<Integer64, 2×2>"),
+        ("ToString[NumericArray[{{1,2},{3,4}}]]", "<Integer64, 2×2>"),
+        ("Head[NumericArray[{1,2}]]", "NumericArray"),
+        ("AtomQ[NumericArray[{1,2}]]", "True"),
+        ("First[NumericArray[{1,2,3}]]", "1"),
+        ("First[NumericArray[{{1,2}, {3,4}}]]", "<Integer64, 2>"),
+        # TODO: these do not work yet
+        # change was applied to First to make that work,
+        # but did not want to change Last because it is awaiting DRYing
+        #("Last[NumericArray[{1,2,3}]]", "3"),
+        #("Last[NumericArray[{{1,2}, {3,4}}]]", "<Integer64, 2>"),
+        ("Normal[NumericArray[{{1,2}, {3,4}}]]", "{{1, 2}, {3, 4}}"),
+    ]
+)
+def test_basics(str_expr, str_expected):
+    check_evaluation(str_expr, str_expected, hold_expected=True)
+
+def test_type_conversion():
+    expr = evaluate("NumericArray[{1,2}]")
+    assert isinstance(expr, NumericArray)
+    assert expr.value.dtype == np.int64
+    expr = evaluate('NumericArray[{1,2}, "ComplexReal32"]')
+    assert expr.value.dtype == np.complex64
+
+