diff --git a/DESCRIPTION b/DESCRIPTION index ec0a77966..e0f09e145 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -45,6 +45,7 @@ Imports: methods, png, rappdirs, + bit64, utils, rlang, withr diff --git a/src/libpython.h b/src/libpython.h index aad3d6aac..cdfe7090e 100644 --- a/src/libpython.h +++ b/src/libpython.h @@ -352,6 +352,12 @@ LIBPYTHON_EXTERN PyObject* (*PyInt_FromLong)(long); LIBPYTHON_EXTERN long (*PyInt_AsLong)(PyObject *); LIBPYTHON_EXTERN PyObject* (*PyLong_FromLong)(long); LIBPYTHON_EXTERN long (*PyLong_AsLong)(PyObject *); +LIBPYTHON_EXTERN PyObject* (*PyLong_FromUnsignedLong)(long); +LIBPYTHON_EXTERN unsigned long (*PyLong_AsUnsignedLong)(PyObject *); +LIBPYTHON_EXTERN long (*PyLong_AsLongAndOverflow)(PyObject *, int*); +LIBPYTHON_EXTERN PyObject* (*PyInt_FromUnsignedLong)(long); +LIBPYTHON_EXTERN unsigned long (*PyInt_AsUnsignedLong)(PyObject *); +LIBPYTHON_EXTERN long (*PyInt_AsLongAndOverflow)(PyObject *, int*); LIBPYTHON_EXTERN PyObject* (*PyBool_FromLong)(long); @@ -440,6 +446,7 @@ typedef struct tagPyArrayObject { typedef unsigned char npy_bool; typedef long npy_long; +typedef unsigned long npy_ulong; typedef double npy_double; typedef struct { double real, imag; } npy_cdouble; typedef npy_cdouble npy_complex128; diff --git a/src/python.cpp b/src/python.cpp index f844956b8..332a06e7a 100644 --- a/src/python.cpp +++ b/src/python.cpp @@ -82,6 +82,27 @@ std::wstring s_python_v3; std::string s_pythonhome; std::wstring s_pythonhome_v3; +const std::string CONFIG_LONG_AS_BIT64="reticulate.long_as_bit64"; +const std::string CONFIG_ULONG_AS_BIT64="reticulate.ulong_as_bit64"; + +template +T getConfig(std::string config, T defValue) { + Environment base( "package:base" ) ; + Function getOption = base["getOption"]; + SEXP s = getOption(config, defValue); + if(s == NULL) { + return defValue; + } + return as(s); +} + +bool convertLongToBit64() { + return getConfig(CONFIG_LONG_AS_BIT64, false); +} + +bool convertULongToBit64() { + return getConfig(CONFIG_ULONG_AS_BIT64, false); +} // helper to convert std::string to std::wstring @@ -294,12 +315,19 @@ int narrow_array_typenum(int typenum) { case NPY_INT: typenum = NPY_LONG; break; - // double - case NPY_UINT: - case NPY_ULONG: - case NPY_ULONGLONG: + case NPY_LONG: case NPY_LONGLONG: + typenum = convertLongToBit64() ? NPY_LONG : NPY_DOUBLE; + break; + + case NPY_ULONG: + case NPY_ULONGLONG: + typenum = convertULongToBit64() ? NPY_ULONG : NPY_DOUBLE; + break; + + // double + case NPY_UINT: case NPY_HALF: case NPY_FLOAT: case NPY_DOUBLE: @@ -328,12 +356,24 @@ int narrow_array_typenum(int typenum) { return typenum; } +int typenum(PyArrayObject* array) { + return PyArray_TYPE(array); +} + +int typenum(PyArray_Descr* descr) { + return descr->type_num; +} + +int typenum(int typeenum) { + return typeenum; +} + int narrow_array_typenum(PyArrayObject* array) { - return narrow_array_typenum(PyArray_TYPE(array)); + return narrow_array_typenum(typenum(array)); } int narrow_array_typenum(PyArray_Descr* descr) { - return narrow_array_typenum(descr->type_num); + return narrow_array_typenum(typenum(descr->type_num)); } bool is_numpy_str(PyObject* x) { @@ -911,10 +951,8 @@ bool py_is_callable(PyObjectRef x) { return py_is_callable(x.get()); } - // convert a python object to an R object SEXP py_to_r(PyObject* x, bool convert) { - // NULL for Python None if (py_is_none(x)) return R_NilValue; @@ -928,9 +966,18 @@ SEXP py_to_r(PyObject* x, bool convert) { return LogicalVector::create(x == Py_True); // integer - else if (scalarType == INTSXP) - return IntegerVector::create(PyInt_AsLong(x)); - + else if (scalarType == INTSXP) { + long val = PyLong_AsLong(x); + if((val > std::numeric_limits::max() || val < std::numeric_limits::min()) && convertLongToBit64()) { + Rcpp::NumericVector vec(1); + std::memcpy(&(vec[0]), &(val), sizeof(double)); + vec.attr("class") = "integer64"; + return vec; + } + else{ + return IntegerVector::create(val); + } + } // double else if (scalarType == REALSXP) return NumericVector::create(PyFloat_AsDouble(x)); @@ -963,9 +1010,23 @@ SEXP py_to_r(PyObject* x, bool convert) { return vec; } else if (scalarType == INTSXP) { Rcpp::IntegerVector vec(len); - for (Py_ssize_t i = 0; i std::numeric_limits::max() || num < std::numeric_limits::min()) && convertLongToBit64()) { + //We need to start over an interpret as 64 bit int + Rcpp::NumericVector nVec(len); + long long* res_ptr = (long long*) dataptr(nVec); + for (Py_ssize_t j = 0; j(value)); + } else if(isUint64) { + return PyLong_FromUnsignedLong(reinterpret_cast(value)); + } else { + return PyFloat_FromDouble(value); + } + } else { + PyObjectPtr list(PyList_New(LENGTH(sexp))); + for (R_xlen_t i = 0; i(value)); + } else if(isUint64) { + obj = PyLong_FromUnsignedLong(reinterpret_cast(value)); + } else { + obj = PyFloat_FromDouble(value); + } + int res = PyList_SetItem(list, i, obj); + if (res != 0) + throw PythonException(py_fetch_error()); + } + return list.detach(); } - - return list.detach(); - } // complex (pass length 1 vectors as scalars, otherwise pass list) @@ -2703,6 +2804,7 @@ SEXP py_dict_get_item(PyObjectRef dict, RObject key) { return py_ref(Py_None, false); } + Py_IncRef(item); return py_ref(item, dict.convert()); @@ -2765,6 +2867,7 @@ CharacterVector py_dict_get_keys_as_str(PyObjectRef dict) { std::vector keys; PyObjectPtr it(PyObject_GetIter(py_keys)); + if (it.is_null()) throw PythonException(py_fetch_error()); @@ -3397,4 +3500,5 @@ PyObjectRef py_capsule(SEXP x) { ensure_python_initialized(); return py_ref(py_capsule_new(x), false); + } diff --git a/tests/testthat/test-python-numpy.R b/tests/testthat/test-python-numpy.R index ec1153f51..b039ce4d5 100644 --- a/tests/testthat/test-python-numpy.R +++ b/tests/testthat/test-python-numpy.R @@ -29,11 +29,38 @@ test_that("Character arrays are handled correctly", { expect_equal(a1, py_to_r(r_to_py(a1))) }) + +test_that("Long integer types are converted to bit64", { + skip_if_no_numpy() + np <- import("numpy", convert = FALSE) + dtypes <- c(np$int64, np$long, np$uint64, np$longlong, np$ulonglong) + require(bit64) + #First run with the default mode + lapply(dtypes, function(dtype) { + a1 <- np$array(c(as.integer64("12345"), as.integer64("1567447722123456786")), dtype = dtype) + expect_equal(class(as.vector(py_to_r(a1))), "numeric") + }) + + #Now with the options set + options(reticulate.long_as_bit64=TRUE) + options(reticulate.ulong_as_bit64=TRUE) + lapply(dtypes, function(dtype) { + a1 <- np$array(c(as.integer64("12345"), as.integer64("1567447722123456786")), dtype = dtype) + res = c('integer64') + if(dtype == np$uint64 || dtype == np$ulonglong) { + res = c(res, 'np.ulong') + } + expect_setequal(class(py_to_r(a1)), res) + }) + options(reticulate.long_as_bit64=F) + options(reticulate.ulong_as_bit64=F) +}) + test_that("Long integer types are converted to R numeric", { skip_if_no_numpy() np <- import("numpy", convert = FALSE) - dtypes <- c(np$int64, np$int32, - np$uint64, np$uint32) + dtypes <- c(np$uint32) + lapply(dtypes, function(dtype) { a1 <- np$array(c(1L:30L), dtype = dtype) a1 <- as.vector(py_to_r(a1)) @@ -86,3 +113,4 @@ test_that("boolean matrices are converted appropriately", { A <- matrix(TRUE, nrow = 2, ncol = 2) expect_equal(A, py_to_r(r_to_py(A))) }) + diff --git a/tests/testthat/test-python-pandas.R b/tests/testthat/test-python-pandas.R index 0aa9e0410..ee70cc72a 100644 --- a/tests/testthat/test-python-pandas.R +++ b/tests/testthat/test-python-pandas.R @@ -166,6 +166,23 @@ test_that("NaT is converted to NA", { expect_equal(py_to_r(before), py_to_r(after)) + +}) + +test_that("Large ints are handled correctly", { + skip_if_no_pandas() + require(bit64) + + options(reticulate.long_as_bit64=TRUE) + options(reticulate.ulong_as_bit64=TRUE) + A <- data.frame(val=c(as.integer64("1567447722123456785"), as.integer64("1567447722123456786"))) + expect_equal(A, py_to_r(r_to_py(A))) + + options(reticulate.long_as_bit64=F) + options(reticulate.ulong_as_bit64=F) + A <- data.frame(val=c(as.integer64("1567447722123456785"), as.integer64("1567447722123456786"))) + expect_equal(A, py_to_r(r_to_py(A))) + }) test_that("pandas NAs are converted to R NAs", { @@ -218,4 +235,5 @@ df = pd.DataFrame({"FCT": pd.Categorical(["No", "Yes"]), expect_identical(p_df, r_df) + })