BUG: DataFrame.diff(axis=1) with mixed (or EA) dtypes

doc/source/whatsnew/v1.1.0.rst

@@ -276,6 +276,7 @@ Numeric
 - Bug in :meth:`to_numeric` with string argument ``"uint64"`` and ``errors="coerce"`` silently fails (:issue:`32394`)
 - Bug in :meth:`to_numeric` with ``downcast="unsigned"`` fails for empty data (:issue:`32493`)
 - Bug in :meth:`DataFrame.mean` with ``numeric_only=False`` and either ``datetime64`` dtype or ``PeriodDtype`` column incorrectly raising ``TypeError`` (:issue:`32426`)
+- Bug in :meth:`DataFrame.diff` with ``axis=1`` returning incorrect results with mixed dtypes; this will now raise ``NotImplementedError`` (:issue:`32995`)
 -
 
 Conversion

pandas/core/frame.py

@@ -6667,6 +6667,30 @@ def diff(self, periods: int = 1, axis: Axis = 0) -> "DataFrame":
         5  NaN  NaN   NaN
         """
         bm_axis = self._get_block_manager_axis(axis)
+        self._consolidate_inplace()
+        if bm_axis == 0 and len(self._data.blocks) > 1 and periods != 0:
+            # i.e. axis == 1 and we have multiple blocks
+            # We need to consolidate before this check otherwise we risk
+            #  getting unexpected dtypes in our result.
+
+            def get_na_ser(ser):
+                # get a Series with an appropriate diff dtype
+                return ser.to_frame().diff(1, axis=1).iloc[:, 0]
+
+            if periods < 0:
+                return self.iloc[:, ::-1].diff(-periods, axis).iloc[:, ::-1]
+
+            ncols = min(periods, len(self.columns))
+            na_results = [get_na_ser(self.iloc[:, n]) for n in range(ncols)]
+            results = [
+                self.iloc[:, n] - self.iloc[:, n - periods]
+                for n in range(ncols, len(self.columns))
+            ]
+
+            all_results = na_results + results
+            dresults = {n: all_results[n] for n in range(len(all_results))}
+            return self._construct_result(dresults)
+
         new_data = self._data.diff(n=periods, axis=bm_axis)
         return self._constructor(new_data)
 

pandas/core/internals/blocks.py

@@ -1834,7 +1834,14 @@ def interpolate(
         )
 
     def diff(self, n: int, axis: int = 1) -> List["Block"]:
+        if axis == 0 and n != 0:
+            # n==0 case will be a no-op so let is fall through
+            # Since we only have one column, the result will be all-NA.
+            #  Create this result by shifting along axis=0 past the length of
+            #  our values.
+            return super().diff(len(self.values), axis=0)
         if axis == 1:
+            # TODO(EA2D): unnecessary with 2D EAs
             # we are by definition 1D.
             axis = 0
         return super().diff(n, axis)

pandas/tests/frame/methods/test_diff.py

@@ -118,3 +118,42 @@ def test_diff_axis(self):
         tm.assert_frame_equal(
             df.diff(axis=0), DataFrame([[np.nan, np.nan], [2.0, 2.0]])
         )
+
+    def test_diff_period(self):
+        # GH#32995 Don't pass an incorrect axis
+        #  TODO(EA2D): this bug wouldn't have happened with 2D EA
+        pi = pd.date_range("2016-01-01", periods=3).to_period("D")
+        df = pd.DataFrame({"A": pi})
+
+        result = df.diff(1, axis=1)
+
+        # TODO: should we make Block.diff do type inference?  or maybe algos.diff?
+        expected = (df - pd.NaT).astype(object)
+        tm.assert_frame_equal(result, expected)
+
+    def test_diff_axis1_mixed_dtypes(self):
+        # GH#32995 operate column-wise when we have mixed dtypes and axis=1
+        df = pd.DataFrame({"A": range(3), "B": 2 * np.arange(3, dtype=np.float64)})
+
+        expected = pd.DataFrame({"A": [np.nan, np.nan, np.nan], "B": df["B"] / 2})
+
+        result = df.diff(axis=1)
+        tm.assert_frame_equal(result, expected)
+
+    def test_diff_axis1_mixed_dtypes_large_periods(self):
+        # GH#32995 operate column-wise when we have mixed dtypes and axis=1
+        df = pd.DataFrame({"A": range(3), "B": 2 * np.arange(3, dtype=np.float64)})
+
+        expected = df * np.nan
+
+        result = df.diff(axis=1, periods=3)
+        tm.assert_frame_equal(result, expected)
+
+    def test_diff_axis1_mixed_dtypes_negative_periods(self):
+        # GH#32995 operate column-wise when we have mixed dtypes and axis=1
+        df = pd.DataFrame({"A": range(3), "B": 2 * np.arange(3, dtype=np.float64)})
+
+        expected = pd.DataFrame({"A": -1.0 * df["A"], "B": df["B"] * np.nan})
+
+        result = df.diff(axis=1, periods=-1)
+        tm.assert_frame_equal(result, expected)