Merge remote-tracking branch 'github/main' into fix_read_pandas

Author: TrevorBergeron

bigframes/ml/metrics/__init__.py

@@ -18,6 +18,7 @@
     auc,
     confusion_matrix,
     f1_score,
+    mean_squared_error,
     precision_score,
     r2_score,
     recall_score,
@@ -35,5 +36,6 @@
     "confusion_matrix",
     "precision_score",
     "f1_score",
+    "mean_squared_error",
     "pairwise",
 ]

bigframes/ml/metrics/_metrics.py

@@ -161,14 +161,10 @@ def roc_auc_score(
 
     fpr, tpr, _ = roc_curve(y_true_series, y_score_series, drop_intermediate=False)
 
-    # TODO(bmil): remove this once bigframes supports the necessary operations
-    pd_fpr = fpr.to_pandas()
-    pd_tpr = tpr.to_pandas()
-
     # Use the trapezoid rule to compute the area under the ROC curve
-    width_diff = pd_fpr.diff().iloc[1:].reset_index(drop=True)
-    height_avg = (pd_tpr.iloc[:-1] + pd_tpr.iloc[1:].reset_index(drop=True)) / 2
-    return (width_diff * height_avg).sum()
+    width_diff = fpr.diff().iloc[1:].reset_index(drop=True)
+    height_avg = (tpr.iloc[:-1] + tpr.iloc[1:].reset_index(drop=True)) / 2
+    return typing.cast(float, (width_diff * height_avg).sum())
 
 
 roc_auc_score.__doc__ = inspect.getdoc(vendored_metrics_ranking.roc_auc_score)
@@ -335,3 +331,17 @@ def f1_score(
 
 
 f1_score.__doc__ = inspect.getdoc(vendored_metrics_classification.f1_score)
+
+
+def mean_squared_error(
+    y_true: Union[bpd.DataFrame, bpd.Series],
+    y_pred: Union[bpd.DataFrame, bpd.Series],
+) -> float:
+    y_true_series, y_pred_series = utils.convert_to_series(y_true, y_pred)
+
+    return (y_pred_series - y_true_series).pow(2).sum() / len(y_true_series)
+
+
+mean_squared_error.__doc__ = inspect.getdoc(
+    vendored_metrics_regression.mean_squared_error
+)

bigframes/operations/_matplotlib/core.py

@@ -14,7 +14,6 @@
 
 import abc
 import typing
-import uuid
 
 import pandas as pd
 
@@ -115,6 +114,18 @@ def _compute_plot_data(self):
         if self._is_column_name(c, sample) and sample[c].dtype == dtypes.STRING_DTYPE:
             sample[c] = sample[c].astype("object")
 
+        # To avoid Matplotlib's automatic conversion of `Float64` or `Int64` columns
+        # to `object` types (which breaks float-like behavior), this code proactively
+        # converts the column to a compatible format.
+        s = self.kwargs.get("s", None)
+        if pd.core.dtypes.common.is_integer(s):
+            s = self.data.columns[s]
+        if self._is_column_name(s, sample):
+            if sample[s].dtype == dtypes.INT_DTYPE:
+                sample[s] = sample[s].astype("int64")
+            elif sample[s].dtype == dtypes.FLOAT_DTYPE:
+                sample[s] = sample[s].astype("float64")
+
         return sample
 
     def _is_sequence_arg(self, arg):
@@ -130,9 +141,3 @@ def _is_column_name(self, arg, data):
             and pd.core.dtypes.common.is_hashable(arg)
             and arg in data.columns
         )
-
-    def _generate_new_column_name(self, data):
-        col_name = None
-        while col_name is None or col_name in data.columns:
-            col_name = f"plot_temp_{str(uuid.uuid4())[:8]}"
-        return col_name

tests/system/large/ml/test_compose.py

@@ -45,14 +45,8 @@ def test_columntransformer_standalone_fit_and_transform(
     )
     result = transformer.transform(new_penguins_df).to_pandas()
 
-    # TODO: bug? feature columns seem to be in nondeterministic random order
-    # workaround: sort columns by name. Can't repro it in pantheon, so could
-    # be a bigframes issue...
-    result = result.reindex(sorted(result.columns), axis=1)
-
     expected = pandas.DataFrame(
         {
-            "min_max_scaled_culmen_length_mm": [0.269, 0.232, 0.210],
             "onehotencoded_species": [
                 [{"index": 1, "value": 1.0}],
                 [{"index": 1, "value": 1.0}],
@@ -63,6 +57,7 @@ def test_columntransformer_standalone_fit_and_transform(
                 -0.9945520581113803,
                 -1.104611490204711,
             ],
+            "min_max_scaled_culmen_length_mm": [0.269, 0.232, 0.210],
             "standard_scaled_flipper_length_mm": [-0.350044, -1.418336, -0.9198],
         },
         index=pandas.Index([1633, 1672, 1690], dtype="Int64", name="tag_number"),
@@ -91,11 +86,6 @@ def test_columntransformer_standalone_fit_transform(new_penguins_df):
         new_penguins_df[["species", "culmen_length_mm", "flipper_length_mm"]]
     ).to_pandas()
 
-    # TODO: bug? feature columns seem to be in nondeterministic random order
-    # workaround: sort columns by name. Can't repro it in pantheon, so could
-    # be a bigframes issue...
-    result = result.reindex(sorted(result.columns), axis=1)
-
     expected = pandas.DataFrame(
         {
             "onehotencoded_species": [