feat: Add concat(..., how="*_relaxed"})

narwhals/_duckdb/namespace.py

@@ -26,7 +26,7 @@
     combine_evaluate_output_names,
 )
 from narwhals._sql.namespace import SQLNamespace
-from narwhals._utils import Implementation
+from narwhals._utils import Implementation, safe_cast
 from narwhals.schema import Schema, merge_schemas, to_supertype
 
 if TYPE_CHECKING:
@@ -97,10 +97,7 @@ def concat(
             schemas: Iterable[Schema] = (Schema(df.collect_schema()) for df in items)
             out_schema = reduce(to_supertype, schemas)
             native_items = (
-                item.select(
-                    *(self.col(name).cast(dtype) for name, dtype in out_schema.items())
-                ).native
-                for item in items
+                item.select(*safe_cast(self, out_schema)).native for item in items
             )
             res = reduce(DuckDBPyRelation.union, native_items)
             return first._with_native(res)
@@ -117,22 +114,20 @@ def concat(
         if how == "diagonal_relaxed":
             schemas = [Schema(df.collect_schema()) for df in items]
             out_schema = reduce(merge_schemas, schemas)
-            res, *others = (
+            native_items = (
                 item.select(
                     *(
-                        self.col(name).cast(dtype)
+                        self.col(name)
                         if name in schema
                         else self.lit(None, dtype=dtype).alias(name)
                         for name, dtype in out_schema.items()
                     )
-                ).native
+                )
+                .select(*safe_cast(self, out_schema))
+                .native
                 for item, schema in zip(items, schemas)
             )
-            for _item in others:
-                # TODO(unassigned): use relational API when available https://github.com/duckdb/duckdb/discussions/16996
-                res = duckdb.sql("""
-                    from res select * union all by name from _item select *
-                """)
+            res = reduce(DuckDBPyRelation.union, native_items)
             return first._with_native(res)
         raise NotImplementedError
 

narwhals/_ibis/namespace.py

@@ -18,7 +18,7 @@
 from narwhals._ibis.selectors import IbisSelectorNamespace
 from narwhals._ibis.utils import function, lit, narwhals_to_native_dtype
 from narwhals._sql.namespace import SQLNamespace
-from narwhals._utils import Implementation
+from narwhals._utils import Implementation, safe_cast
 from narwhals.schema import Schema, to_supertype
 
 if TYPE_CHECKING:
@@ -74,11 +74,8 @@ def concat(
 
         if how.endswith("relaxed"):
             schemas = (Schema(frame.collect_schema()) for frame in frames)
-            out_schema = reduce(to_supertype, schemas).items()
-            frames = [
-                frame.select(*(self.col(name).cast(dtype) for name, dtype in out_schema))
-                for frame in frames
-            ]
+            out_schema = reduce(to_supertype, schemas)
+            frames = [frame.select(*safe_cast(self, out_schema)) for frame in frames]
         try:
             result = ibis.union(*(lf.native for lf in frames))
         except ibis.IbisError:

narwhals/_pandas_like/utils.py

@@ -196,6 +196,20 @@ def rename(
     return cast("NativeNDFrameT", result)  # type: ignore[redundant-cast]
 
 
+@lru_cache(maxsize=16)
+def is_dtype_non_pyarrow_string(native_dtype: Any) -> bool:
+    """*There is no problem which can't be solved by adding an extra string type* pandas."""
+    # TODO @dangotbanned: Investigate how we could handle `cudf` without `str(native_dtype)`
+    # https://github.com/rapidsai/cudf/blob/a32b8cf62c9b086b645b0825b78b99f065b1887f/python/cudf/cudf/utils/dtypes.py#L646-L670
+    return isinstance(native_dtype, pd.StringDtype) or str(native_dtype) in {
+        "string",
+        "string[python]",
+        "string[pyarrow_numpy]",
+        "<StringDtype(na_value=nan)>",  # why? why? why?
+        "str",
+    }
+
+
 @lru_cache(maxsize=16)
 def non_object_native_to_narwhals_dtype(native_dtype: Any, version: Version) -> DType:  # noqa: C901, PLR0912
     dtype = str(native_dtype)

narwhals/_spark_like/namespace.py

@@ -19,6 +19,7 @@
     true_divide,
 )
 from narwhals._sql.namespace import SQLNamespace
+from narwhals._utils import safe_cast
 from narwhals.schema import Schema, merge_schemas, to_supertype
 
 if TYPE_CHECKING:
@@ -181,10 +182,7 @@ def concat(
             schemas: Iterable[Schema] = (Schema(df.collect_schema()) for df in items)
             out_schema = reduce(to_supertype, schemas)
             native_items = (
-                item.select(
-                    *(self.col(name).cast(dtype) for name, dtype in out_schema.items())
-                ).native
-                for item in items
+                item.select(*safe_cast(self, out_schema)).native for item in items
             )
             union = items[0].native.__class__.union
             return SparkLikeLazyFrame(
@@ -199,12 +197,14 @@ def concat(
             native_items = (
                 item.select(
                     *(
-                        self.col(name).cast(dtype)
+                        self.col(name)
                         if name in schema
                         else self.lit(None, dtype=dtype).alias(name)
                         for name, dtype in out_schema.items()
                     )
-                ).native
+                )
+                .select(*safe_cast(self, out_schema))
+                .native
                 for item, schema in zip(items, schemas)
             )
             union = items[0].native.__class__.union

narwhals/_utils.py

@@ -68,7 +68,14 @@
         TypeIs,
     )
 
-    from narwhals._compliant import CompliantExprT, CompliantSeriesT, NativeSeriesT_co
+    from narwhals import dtypes
+    from narwhals._compliant import (
+        CompliantExprT,
+        CompliantFrameT,
+        CompliantNamespace,
+        CompliantSeriesT,
+        NativeSeriesT_co,
+    )
     from narwhals._compliant.any_namespace import NamespaceAccessor
     from narwhals._compliant.typing import (
         Accessor,
@@ -2148,3 +2155,14 @@ def __repr__(self) -> str:  # pragma: no cover
 
 # Can be imported from types in Python 3.10
 NoneType = type(None)
+
+
+def safe_cast(
+    ns: CompliantNamespace[CompliantFrameT, CompliantExprT],
+    mapping: Mapping[str, dtypes.DType],
+) -> Iterable[CompliantExprT]:
+    Unknown = ns._version.dtypes.Unknown()  # noqa: N806
+    return (
+        ns.col(name) if dtype == Unknown else ns.col(name).cast(dtype)
+        for name, dtype in mapping.items()
+    )

tests/frame/schema_test.py

@@ -744,11 +744,11 @@ def test_to_supertype(left: IntoSchema, right: IntoSchema, expected: IntoSchema)
             pytest.raises(ComputeError, match="schema names differ: got b, expected a"),
         ),
         (
-            {"a": nw.String()},
+            {"a": nw.Binary()},
             {"a": nw.Int64()},
             pytest.raises(
                 SchemaMismatchError,
-                match="failed to determine supertype of String and Int64",
+                match="failed to determine supertype of Binary and Int64",
             ),
         ),
     ],