Merge remote-tracking branch 'upstream/2.3.x' into backport-60985

Author: jorisvandenbossche

pandas/core/arrays/base.py

@@ -2386,7 +2386,14 @@ def _groupby_op(
             if op.how not in ["any", "all"]:
                 # Fail early to avoid conversion to object
                 op._get_cython_function(op.kind, op.how, np.dtype(object), False)
-            npvalues = self.to_numpy(object, na_value=np.nan)
+
+            arr = self
+            if op.how == "sum":
+                # https://github.com/pandas-dev/pandas/issues/60229
+                # All NA should result in the empty string.
+                if min_count == 0:
+                    arr = arr.fillna("")
+            npvalues = arr.to_numpy(object, na_value=np.nan)
         else:
             raise NotImplementedError(
                 f"function is not implemented for this dtype: {self.dtype}"

pandas/core/indexes/base.py

@@ -6387,6 +6387,36 @@ def _find_common_type_compat(self, target) -> DtypeObj:
         """
         target_dtype, _ = infer_dtype_from(target)
 
+        if using_string_dtype():
+            # special case: if left or right is a zero-length RangeIndex or
+            # Index[object], those can be created by the default empty constructors
+            # -> for that case ignore this dtype and always return the other
+            # (https://github.com/pandas-dev/pandas/pull/60797)
+            from pandas.core.indexes.range import RangeIndex
+
+            if len(self) == 0 and (
+                isinstance(self, RangeIndex) or self.dtype == np.object_
+            ):
+                if target_dtype.kind == "M":
+                    if hasattr(target_dtype, "tz"):
+                        target_dtype_ns = DatetimeTZDtype("ns", tz=target_dtype.tz)
+                    else:
+                        target_dtype_ns = np.dtype("datetime64[ns]")  # type: ignore[assignment]
+                    try:
+                        Index(target, dtype=target_dtype_ns, copy=False)
+                    except OutOfBoundsDatetime:
+                        return np.dtype(object)
+                    except Exception:
+                        pass
+                    return target_dtype_ns
+                return target_dtype
+            if (
+                isinstance(target, Index)
+                and len(target) == 0
+                and (isinstance(target, RangeIndex) or target_dtype == np.object_)
+            ):
+                return self.dtype
+
         # special case: if one dtype is uint64 and the other a signed int, return object
         # See https://github.com/pandas-dev/pandas/issues/26778 for discussion
         # Now it's:
@@ -7005,6 +7035,14 @@ def insert(self, loc: int, item) -> Index:
 
         arr = self._values
 
+        if using_string_dtype() and len(self) == 0 and self.dtype == np.object_:
+            # special case: if we are an empty object-dtype Index, also
+            # take into account the inserted item for the resulting dtype
+            # (https://github.com/pandas-dev/pandas/pull/60797)
+            dtype = self._find_common_type_compat(item)
+            if dtype != self.dtype:
+                return self.astype(dtype).insert(loc, item)
+
         try:
             if isinstance(arr, ExtensionArray):
                 res_values = arr.insert(loc, item)

pandas/tests/frame/constructors/test_from_dict.py

@@ -3,8 +3,6 @@
 import numpy as np
 import pytest
 
-from pandas._config import using_string_dtype
-
 from pandas import (
     DataFrame,
     Index,
@@ -44,7 +42,6 @@ def test_constructor_single_row(self):
         )
         tm.assert_frame_equal(result, expected)
 
-    @pytest.mark.xfail(using_string_dtype(), reason="columns inferring logic broken")
     def test_constructor_list_of_series(self):
         data = [
             OrderedDict([["a", 1.5], ["b", 3.0], ["c", 4.0]]),

pandas/tests/frame/indexing/test_coercion.py

@@ -103,12 +103,7 @@ def test_26395(indexer_al):
     df["D"] = 0
 
     indexer_al(df)["C", "D"] = 2
-    expected = DataFrame(
-        {"D": [0, 0, 2]},
-        index=["A", "B", "C"],
-        columns=pd.Index(["D"], dtype=object),
-        dtype=np.int64,
-    )
+    expected = DataFrame({"D": [0, 0, 2]}, index=["A", "B", "C"], dtype=np.int64)
     tm.assert_frame_equal(df, expected)
 
     with tm.assert_produces_warning(
@@ -118,7 +113,7 @@ def test_26395(indexer_al):
     expected = DataFrame(
         {"D": [0, 0, 44.5]},
         index=["A", "B", "C"],
-        columns=pd.Index(["D"], dtype=object),
+        columns=["D"],
         dtype=np.float64,
     )
     tm.assert_frame_equal(df, expected)
@@ -130,7 +125,7 @@ def test_26395(indexer_al):
     expected = DataFrame(
         {"D": [0, 0, "hello"]},
         index=["A", "B", "C"],
-        columns=pd.Index(["D"], dtype=object),
+        columns=["D"],
         dtype=object,
     )
     tm.assert_frame_equal(df, expected)

pandas/tests/frame/indexing/test_indexing.py

@@ -1206,7 +1206,7 @@ def test_loc_setitem_datetimelike_with_inference(self):
         result = df.dtypes
         expected = Series(
             [np.dtype("timedelta64[ns]")] * 6 + [np.dtype("datetime64[ns]")] * 2,
-            index=Index(list("ABCDEFGH"), dtype=object),
+            index=list("ABCDEFGH"),
         )
         tm.assert_series_equal(result, expected)
 

pandas/tests/frame/indexing/test_insert.py

@@ -67,8 +67,7 @@ def test_insert_with_columns_dups(self):
         df.insert(0, "A", ["d", "e", "f"], allow_duplicates=True)
         df.insert(0, "A", ["a", "b", "c"], allow_duplicates=True)
         exp = DataFrame(
-            [["a", "d", "g"], ["b", "e", "h"], ["c", "f", "i"]],
-            columns=Index(["A", "A", "A"], dtype=object),
+            [["a", "d", "g"], ["b", "e", "h"], ["c", "f", "i"]], columns=["A", "A", "A"]
         )
         tm.assert_frame_equal(df, exp)
 

pandas/tests/frame/indexing/test_setitem.py

@@ -146,18 +146,32 @@ def test_setitem_different_dtype(self):
         )
         tm.assert_series_equal(result, expected)
 
-    def test_setitem_empty_columns(self):
-        # GH 13522
+    def test_setitem_overwrite_index(self):
+        # GH 13522 - assign the index as a column and then overwrite the values
+        # -> should not affect the index
         df = DataFrame(index=["A", "B", "C"])
         df["X"] = df.index
         df["X"] = ["x", "y", "z"]
         exp = DataFrame(
-            data={"X": ["x", "y", "z"]},
-            index=["A", "B", "C"],
-            columns=Index(["X"], dtype=object),
+            data={"X": ["x", "y", "z"]}, index=["A", "B", "C"], columns=["X"]
         )
         tm.assert_frame_equal(df, exp)
 
+    def test_setitem_empty_columns(self):
+        # Starting from an empty DataFrame and setting a column should result
+        # in a default string dtype for the columns' Index
+        # https://github.com/pandas-dev/pandas/issues/60338
+
+        df = DataFrame()
+        df["foo"] = [1, 2, 3]
+        expected = DataFrame({"foo": [1, 2, 3]})
+        tm.assert_frame_equal(df, expected)
+
+        df = DataFrame(columns=Index([]))
+        df["foo"] = [1, 2, 3]
+        expected = DataFrame({"foo": [1, 2, 3]})
+        tm.assert_frame_equal(df, expected)
+
     def test_setitem_dt64_index_empty_columns(self):
         rng = date_range("1/1/2000 00:00:00", "1/1/2000 1:59:50", freq="10s")
         df = DataFrame(index=np.arange(len(rng)))
@@ -171,9 +185,7 @@ def test_setitem_timestamp_empty_columns(self):
         df["now"] = Timestamp("20130101", tz="UTC").as_unit("ns")
 
         expected = DataFrame(
-            [[Timestamp("20130101", tz="UTC")]] * 3,
-            index=range(3),
-            columns=Index(["now"], dtype=object),
+            [[Timestamp("20130101", tz="UTC")]] * 3, index=range(3), columns=["now"]
         )
         tm.assert_frame_equal(df, expected)
 
@@ -212,7 +224,7 @@ def test_setitem_period_preserves_dtype(self):
         result = DataFrame([])
         result["a"] = data
 
-        expected = DataFrame({"a": data}, columns=Index(["a"], dtype=object))
+        expected = DataFrame({"a": data}, columns=["a"])
 
         tm.assert_frame_equal(result, expected)
 
@@ -939,7 +951,7 @@ def test_setitem_scalars_no_index(self):
         # GH#16823 / GH#17894
         df = DataFrame()
         df["foo"] = 1
-        expected = DataFrame(columns=Index(["foo"], dtype=object)).astype(np.int64)
+        expected = DataFrame(columns=["foo"]).astype(np.int64)
         tm.assert_frame_equal(df, expected)
 
     def test_setitem_newcol_tuple_key(self, float_frame):

pandas/tests/frame/methods/test_dropna.py

@@ -182,12 +182,9 @@ def test_dropna_multiple_axes(self):
         with pytest.raises(TypeError, match="supplying multiple axes"):
             inp.dropna(how="all", axis=(0, 1), inplace=True)
 
-    def test_dropna_tz_aware_datetime(self, using_infer_string):
+    def test_dropna_tz_aware_datetime(self):
         # GH13407
-
         df = DataFrame()
-        if using_infer_string:
-            df.columns = df.columns.astype("str")
         dt1 = datetime.datetime(2015, 1, 1, tzinfo=dateutil.tz.tzutc())
         dt2 = datetime.datetime(2015, 2, 2, tzinfo=dateutil.tz.tzutc())
         df["Time"] = [dt1]

pandas/tests/frame/methods/test_reset_index.py

@@ -4,8 +4,6 @@
 import numpy as np
 import pytest
 
-from pandas._config import using_string_dtype
-
 from pandas.core.dtypes.common import (
     is_float_dtype,
     is_integer_dtype,
@@ -646,7 +644,6 @@ def test_rest_index_multiindex_categorical_with_missing_values(self, codes):
         tm.assert_frame_equal(res, expected)
 
 
-@pytest.mark.xfail(using_string_dtype(), reason="TODO(infer_string) - GH#60338")
 @pytest.mark.parametrize(
     "array, dtype",
     [
@@ -783,3 +780,34 @@ def test_reset_index_false_index_name():
     result_frame.reset_index()
     expected_frame = DataFrame(range(5, 10), RangeIndex(range(5), name=False))
     tm.assert_frame_equal(result_frame, expected_frame)
+
+
+@pytest.mark.parametrize("columns", [None, Index([])])
+def test_reset_index_with_empty_frame(columns):
+    # Currently empty DataFrame has RangeIndex or object dtype Index, but when
+    # resetting the index we still want to end up with the default string dtype
+    # https://github.com/pandas-dev/pandas/issues/60338
+
+    index = Index([], name="foo")
+    df = DataFrame(index=index, columns=columns)
+    result = df.reset_index()
+    expected = DataFrame(columns=["foo"])
+    tm.assert_frame_equal(result, expected)
+
+    index = Index([1, 2, 3], name="foo")
+    df = DataFrame(index=index, columns=columns)
+    result = df.reset_index()
+    expected = DataFrame({"foo": [1, 2, 3]})
+    tm.assert_frame_equal(result, expected)
+
+    index = MultiIndex.from_tuples([], names=["foo", "bar"])
+    df = DataFrame(index=index, columns=columns)
+    result = df.reset_index()
+    expected = DataFrame(columns=["foo", "bar"])
+    tm.assert_frame_equal(result, expected)
+
+    index = MultiIndex.from_tuples([(1, 2), (2, 3)], names=["foo", "bar"])
+    df = DataFrame(index=index, columns=columns)
+    result = df.reset_index()
+    expected = DataFrame({"foo": [1, 2], "bar": [2, 3]})
+    tm.assert_frame_equal(result, expected)

pandas/tests/frame/test_constructors.py

@@ -21,8 +21,6 @@
 import pytest
 import pytz
 
-from pandas._config import using_string_dtype
-
 from pandas._libs import lib
 from pandas.compat.numpy import np_version_gt2
 from pandas.errors import IntCastingNaNError
@@ -2002,7 +2000,6 @@ def test_constructor_with_datetimes4(self):
         df = DataFrame({"value": dr})
         assert str(df.iat[0, 0].tz) == "US/Eastern"
 
-    @pytest.mark.xfail(using_string_dtype(), reason="TODO(infer_string)")
     def test_constructor_with_datetimes5(self):
         # GH 7822
         # preserver an index with a tz on dict construction

pandas/tests/frame/test_query_eval.py

@@ -757,7 +757,6 @@ def test_check_tz_aware_index_query(self, tz_aware_fixture):
         tm.assert_frame_equal(result, expected)
 
         expected = DataFrame(df_index)
-        expected.columns = expected.columns.astype(object)
         result = df.reset_index().query('"2018-01-03 00:00:00+00" < time')
         tm.assert_frame_equal(result, expected)
 

pandas/tests/frame/test_reductions.py

@@ -846,6 +846,16 @@ def test_axis_1_empty(self, all_reductions, index):
         expected = Series([], index=index, dtype=expected_dtype)
         tm.assert_series_equal(result, expected)
 
+    @pytest.mark.parametrize("min_count", [0, 1])
+    def test_axis_1_sum_na(self, string_dtype_no_object, skipna, min_count):
+        # https://github.com/pandas-dev/pandas/issues/60229
+        dtype = string_dtype_no_object
+        df = DataFrame({"a": [pd.NA]}, dtype=dtype)
+        result = df.sum(axis=1, skipna=skipna, min_count=min_count)
+        value = "" if skipna and min_count == 0 else pd.NA
+        expected = Series([value], dtype=dtype)
+        tm.assert_series_equal(result, expected)
+
     @pytest.mark.parametrize("method, unit", [("sum", 0), ("prod", 1)])
     @pytest.mark.parametrize("numeric_only", [None, True, False])
     def test_sum_prod_nanops(self, method, unit, numeric_only):

pandas/tests/groupby/test_groupby.py

@@ -1623,7 +1623,7 @@ def test_groupby_2d_malformed():
     d["label"] = ["l1", "l2"]
     tmp = d.groupby(["group"]).mean(numeric_only=True)
     res_values = np.array([[0.0, 1.0], [0.0, 1.0]])
-    tm.assert_index_equal(tmp.columns, Index(["zeros", "ones"], dtype=object))
+    tm.assert_index_equal(tmp.columns, Index(["zeros", "ones"]))
     tm.assert_numpy_array_equal(tmp.values, res_values)
 
 

pandas/tests/groupby/test_reductions.py

@@ -798,6 +798,20 @@ def test_string_dtype_all_na(
     tm.assert_equal(result, expected)
 
 
+@pytest.mark.parametrize("min_count", [0, 1])
+def test_string_dtype_empty_sum(string_dtype_no_object, min_count):
+    # https://github.com/pandas-dev/pandas/issues/60229
+    dtype = string_dtype_no_object
+    df = DataFrame({"a": ["x"], "b": [pd.NA]}, dtype=dtype)
+    gb = df.groupby("a")
+    result = gb.sum(min_count=min_count)
+    value = "" if min_count == 0 else pd.NA
+    expected = DataFrame(
+        {"b": value}, index=pd.Index(["x"], name="a", dtype=dtype), dtype=dtype
+    )
+    tm.assert_frame_equal(result, expected)
+
+
 def test_max_nan_bug():
     df = DataFrame(
         {

pandas/tests/indexes/base_class/test_reshape.py

@@ -34,7 +34,7 @@ def test_insert(self):
 
         # test empty
         null_index = Index([])
-        tm.assert_index_equal(Index(["a"], dtype=object), null_index.insert(0, "a"))
+        tm.assert_index_equal(Index(["a"]), null_index.insert(0, "a"))
 
     def test_insert_missing(self, request, nulls_fixture, using_infer_string):
         if using_infer_string and nulls_fixture is pd.NA:

pandas/tests/indexes/base_class/test_setops.py

@@ -240,7 +240,6 @@ def test_tuple_union_bug(self, method, expected, sort):
     def test_union_name_preservation(
         self, first_list, second_list, first_name, second_name, expected_name, sort
     ):
-        expected_dtype = object if not first_list or not second_list else "str"
         first = Index(first_list, name=first_name)
         second = Index(second_list, name=second_name)
         union = first.union(second, sort=sort)
@@ -251,7 +250,7 @@ def test_union_name_preservation(
             expected = Index(sorted(vals), name=expected_name)
             tm.assert_index_equal(union, expected)
         else:
-            expected = Index(vals, name=expected_name, dtype=expected_dtype)
+            expected = Index(vals, name=expected_name)
             tm.assert_index_equal(union.sort_values(), expected.sort_values())
 
     @pytest.mark.parametrize(

pandas/tests/indexes/datetimes/test_join.py

@@ -70,13 +70,17 @@ def test_join_utc_convert(self, join_type):
         assert isinstance(result, DatetimeIndex)
         assert result.tz is timezone.utc
 
-    def test_datetimeindex_union_join_empty(self, sort):
+    def test_datetimeindex_union_join_empty(self, sort, using_infer_string):
         dti = date_range(start="1/1/2001", end="2/1/2001", freq="D")
         empty = Index([])
 
         result = dti.union(empty, sort=sort)
-        expected = dti.astype("O")
-        tm.assert_index_equal(result, expected)
+        if using_infer_string:
+            assert isinstance(result, DatetimeIndex)
+            tm.assert_index_equal(result, dti)
+        else:
+            expected = dti.astype("O")
+            tm.assert_index_equal(result, expected)
 
         result = dti.join(empty)
         assert isinstance(result, DatetimeIndex)