star

venv/lib/python3.9/site-packages/mlflow/models/evaluation/__init__.py

@@ -0,0 +1,23 @@
+from mlflow.data.evaluation_dataset import EvaluationDataset
+from mlflow.models.evaluation.base import (
+    EvaluationArtifact,
+    EvaluationMetric,
+    EvaluationResult,
+    ModelEvaluator,
+    evaluate,
+    list_evaluators,
+    make_metric,
+)
+from mlflow.models.evaluation.validation import MetricThreshold
+
+__all__ = [
+    "ModelEvaluator",
+    "EvaluationDataset",
+    "EvaluationResult",
+    "EvaluationMetric",
+    "EvaluationArtifact",
+    "make_metric",
+    "evaluate",
+    "list_evaluators",
+    "MetricThreshold",
+]

venv/lib/python3.9/site-packages/mlflow/models/evaluation/_shap_patch.py

@@ -0,0 +1,64 @@
+import pickle
+
+import numpy as np
+import shap
+from shap._serializable import Deserializer, Serializable, Serializer
+
+
+class _PatchedKernelExplainer(shap.KernelExplainer):
+    @staticmethod
+    def not_equal(i, j):
+        # `shap.KernelExplainer.not_equal` method fails on some special types such as
+        # timestamp, this breaks the kernel explainer routine.
+        # `PatchedKernelExplainer` fixes this issue.
+        # See https://github.com/slundberg/shap/pull/2586
+        number_types = (int, float, np.number)
+        if isinstance(i, number_types) and isinstance(j, number_types):
+            return 0 if np.isclose(i, j, equal_nan=True) else 1
+        else:
+            return 0 if i == j else 1
+
+    def save(self, out_file, model_saver=None, masker_saver=None):
+        """
+        This patched `save` method fix `KernelExplainer.save`.
+        Issues in original `KernelExplainer.save`:
+         - It saves model by calling model.save, but shap.utils._legacy.Model has no save method
+         - It tries to save "masker", but there's no "masker" in KernelExplainer
+         - It does not save "KernelExplainer.data" attribute, the attribute is required when
+           loading back
+        Note: `model_saver` and `masker_saver` are meaningless argument for `KernelExplainer.save`,
+        the model in "KernelExplainer" is an instance of `shap.utils._legacy.Model`
+        (it wraps the predict function), we can only use pickle to dump it.
+        and no `masker` for KernelExplainer so `masker_saver` is meaningless.
+        but I preserve the 2 argument for overridden API compatibility.
+        """
+        pickle.dump(type(self), out_file)
+        with Serializer(out_file, "shap.Explainer", version=0) as s:
+            s.save("model", self.model)
+            s.save("link", self.link)
+            s.save("data", self.data)
+
+    @classmethod
+    def load(cls, in_file, model_loader=None, masker_loader=None, instantiate=True):
+        """
+        This patched `load` method fix `KernelExplainer.load`.
+        Issues in original KernelExplainer.load:
+         - Use mismatched model loader to load model
+         - Try to load non-existent "masker" attribute
+         - Does not load "data" attribute and then cause calling " KernelExplainer"
+           constructor lack of "data" argument.
+        Note: `model_loader` and `masker_loader` are meaningless argument for
+        `KernelExplainer.save`, because the `model` object is saved by pickle dump,
+        we must use pickle load to load it.
+        and no `masker` for KernelExplainer so `masker_loader` is meaningless.
+        but I preserve the 2 argument for overridden API compatibility.
+        """
+        if instantiate:
+            return cls._instantiated_load(in_file, model_loader=None, masker_loader=None)
+
+        kwargs = Serializable.load(in_file, instantiate=False)
+        with Deserializer(in_file, "shap.Explainer", min_version=0, max_version=0) as s:
+            kwargs["model"] = s.load("model")
+            kwargs["link"] = s.load("link")
+            kwargs["data"] = s.load("data")
+        return kwargs

venv/lib/python3.9/site-packages/mlflow/models/evaluation/artifacts.py

@@ -0,0 +1,194 @@
+import json
+import pathlib
+import pickle
+from collections import namedtuple
+from json import JSONDecodeError
+
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+
+from mlflow.exceptions import MlflowException
+from mlflow.models.evaluation.base import EvaluationArtifact
+from mlflow.utils.annotations import developer_stable
+from mlflow.utils.proto_json_utils import NumpyEncoder
+
+
+@developer_stable
+class ImageEvaluationArtifact(EvaluationArtifact):
+    def _save(self, output_artifact_path):
+        self._content.save(output_artifact_path)
+
+    def _load_content_from_file(self, local_artifact_path):
+        from PIL.Image import open as open_image
+
+        self._content = open_image(local_artifact_path)
+        self._content.load()  # Load image and close the file descriptor.
+        return self._content
+
+
+@developer_stable
+class CsvEvaluationArtifact(EvaluationArtifact):
+    def _save(self, output_artifact_path):
+        self._content.to_csv(output_artifact_path, index=False)
+
+    def _load_content_from_file(self, local_artifact_path):
+        self._content = pd.read_csv(local_artifact_path)
+        return self._content
+
+
+@developer_stable
+class ParquetEvaluationArtifact(EvaluationArtifact):
+    def _save(self, output_artifact_path):
+        self._content.to_parquet(output_artifact_path, compression="brotli")
+
+    def _load_content_from_file(self, local_artifact_path):
+        self._content = pd.read_parquet(local_artifact_path)
+        return self._content
+
+
+@developer_stable
+class NumpyEvaluationArtifact(EvaluationArtifact):
+    def _save(self, output_artifact_path):
+        np.save(output_artifact_path, self._content, allow_pickle=False)
+
+    def _load_content_from_file(self, local_artifact_path):
+        self._content = np.load(local_artifact_path, allow_pickle=False)
+        return self._content
+
+
+@developer_stable
+class JsonEvaluationArtifact(EvaluationArtifact):
+    def _save(self, output_artifact_path):
+        with open(output_artifact_path, "w") as f:
+            json.dump(self._content, f)
+
+    def _load_content_from_file(self, local_artifact_path):
+        with open(local_artifact_path) as f:
+            self._content = json.load(f)
+        return self._content
+
+
+@developer_stable
+class TextEvaluationArtifact(EvaluationArtifact):
+    def _save(self, output_artifact_path):
+        with open(output_artifact_path, "w") as f:
+            f.write(self._content)
+
+    def _load_content_from_file(self, local_artifact_path):
+        with open(local_artifact_path) as f:
+            self._content = f.read()
+        return self._content
+
+
+@developer_stable
+class PickleEvaluationArtifact(EvaluationArtifact):
+    def _save(self, output_artifact_path):
+        with open(output_artifact_path, "wb") as f:
+            pickle.dump(self._content, f)
+
+    def _load_content_from_file(self, local_artifact_path):
+        with open(local_artifact_path, "rb") as f:
+            self._content = pickle.load(f)
+        return self._content
+
+
+_EXT_TO_ARTIFACT_MAP = {
+    ".png": ImageEvaluationArtifact,
+    ".jpg": ImageEvaluationArtifact,
+    ".jpeg": ImageEvaluationArtifact,
+    ".json": JsonEvaluationArtifact,
+    ".npy": NumpyEvaluationArtifact,
+    ".csv": CsvEvaluationArtifact,
+    ".parquet": ParquetEvaluationArtifact,
+    ".txt": TextEvaluationArtifact,
+}
+
+_TYPE_TO_EXT_MAP = {
+    pd.DataFrame: ".csv",
+    np.ndarray: ".npy",
+    plt.Figure: ".png",
+}
+
+_TYPE_TO_ARTIFACT_MAP = {
+    pd.DataFrame: CsvEvaluationArtifact,
+    np.ndarray: NumpyEvaluationArtifact,
+    plt.Figure: ImageEvaluationArtifact,
+}
+
+_InferredArtifactProperties = namedtuple(
+    "_InferredArtifactProperties", ["from_path", "type", "ext"]
+)
+
+
+def _infer_artifact_type_and_ext(artifact_name, raw_artifact, custom_metric_tuple):
+    """
+    This function performs type and file extension inference on the provided artifact
+
+    Args:
+        artifact_name: The name of the provided artifact
+        raw_artifact: The artifact object
+        custom_metric_tuple: Containing a user provided function and its index in the
+            ``custom_metrics`` parameter of ``mlflow.evaluate``
+
+    Returns:
+        InferredArtifactProperties namedtuple
+    """
+
+    exception_header = (
+        f"Custom metric function '{custom_metric_tuple.name}' at index "
+        f"{custom_metric_tuple.index} in the `custom_metrics` parameter produced an "
+        f"artifact '{artifact_name}'"
+    )
+
+    # Given a string, first see if it is a path. Otherwise, check if it is a JsonEvaluationArtifact
+    if isinstance(raw_artifact, str):
+        potential_path = pathlib.Path(raw_artifact)
+        if potential_path.exists():
+            raw_artifact = potential_path
+        else:
+            try:
+                json.loads(raw_artifact)
+                return _InferredArtifactProperties(
+                    from_path=False, type=JsonEvaluationArtifact, ext=".json"
+                )
+            except JSONDecodeError:
+                raise MlflowException(
+                    f"{exception_header} with string representation '{raw_artifact}' that is "
+                    f"neither a valid path to a file nor a JSON string."
+                )
+
+    # Type inference based on the file extension
+    if isinstance(raw_artifact, pathlib.Path):
+        if not raw_artifact.exists():
+            raise MlflowException(f"{exception_header} with path '{raw_artifact}' does not exist.")
+        if not raw_artifact.is_file():
+            raise MlflowException(f"{exception_header} with path '{raw_artifact}' is not a file.")
+        if raw_artifact.suffix not in _EXT_TO_ARTIFACT_MAP:
+            raise MlflowException(
+                f"{exception_header} with path '{raw_artifact}' does not match any of the supported"
+                f" file extensions: {', '.join(_EXT_TO_ARTIFACT_MAP.keys())}."
+            )
+        return _InferredArtifactProperties(
+            from_path=True, type=_EXT_TO_ARTIFACT_MAP[raw_artifact.suffix], ext=raw_artifact.suffix
+        )
+
+    # Type inference based on object type
+    if type(raw_artifact) in _TYPE_TO_ARTIFACT_MAP:
+        return _InferredArtifactProperties(
+            from_path=False,
+            type=_TYPE_TO_ARTIFACT_MAP[type(raw_artifact)],
+            ext=_TYPE_TO_EXT_MAP[type(raw_artifact)],
+        )
+
+    # Given as other python object, we first attempt to infer as JsonEvaluationArtifact. If that
+    # fails, we store it as PickleEvaluationArtifact
+    try:
+        json.dumps(raw_artifact, cls=NumpyEncoder)
+        return _InferredArtifactProperties(
+            from_path=False, type=JsonEvaluationArtifact, ext=".json"
+        )
+    except TypeError:
+        return _InferredArtifactProperties(
+            from_path=False, type=PickleEvaluationArtifact, ext=".pickle"
+        )

venv/lib/python3.9/site-packages/mlflow/models/evaluation/evaluator_registry.py

@@ -0,0 +1,80 @@
+import warnings
+
+from mlflow.exceptions import MlflowException
+from mlflow.utils.import_hooks import register_post_import_hook
+from mlflow.utils.plugins import get_entry_points
+
+
+class ModelEvaluatorRegistry:
+    """
+    Scheme-based registry for model evaluator implementations
+    """
+
+    def __init__(self):
+        self._registry = {}
+        self._builtin_evaluators = {}
+
+    def register(self, scheme, evaluator):
+        """Register model evaluator provided by other packages"""
+        self._registry[scheme] = evaluator
+
+    def register_builtin(self, scheme, evaluator):
+        """Register built-in model evaluator"""
+        self._registry[scheme] = evaluator
+        self._builtin_evaluators[scheme] = evaluator
+
+    def register_entrypoints(self):
+        # Register ModelEvaluator implementation provided by other packages
+        for entrypoint in get_entry_points("mlflow.model_evaluator"):
+            try:
+                self.register(entrypoint.name, entrypoint.load())
+            except (AttributeError, ImportError) as exc:
+                warnings.warn(
+                    'Failure attempting to register model evaluator for scheme "{}": {}'.format(
+                        entrypoint.name, str(exc)
+                    ),
+                    stacklevel=2,
+                )
+
+    def get_evaluator(self, evaluator_name):
+        """
+        Get an evaluator instance from the registry based on the name of evaluator
+        """
+        evaluator_cls = self._registry.get(evaluator_name)
+        if evaluator_cls is None:
+            raise MlflowException(
+                f"Could not find a registered model evaluator for: {evaluator_name}. "
+                f"Currently registered evaluator names are: {list(self._registry.keys())}"
+            )
+        return evaluator_cls()
+
+    def is_builtin(self, name):
+        return name in self._builtin_evaluators
+
+    def is_registered(self, name):
+        return name in self._registry
+
+
+_model_evaluation_registry = ModelEvaluatorRegistry()
+
+
+def register_evaluators(module):
+    from mlflow.models.evaluation.evaluators.classifier import ClassifierEvaluator
+    from mlflow.models.evaluation.evaluators.default import DefaultEvaluator
+    from mlflow.models.evaluation.evaluators.regressor import RegressorEvaluator
+    from mlflow.models.evaluation.evaluators.shap import ShapEvaluator
+
+    # Built-in evaluators
+    module._model_evaluation_registry.register_builtin(DefaultEvaluator.name, DefaultEvaluator)
+    module._model_evaluation_registry.register_builtin(
+        ClassifierEvaluator.name, ClassifierEvaluator
+    )
+    module._model_evaluation_registry.register_builtin(RegressorEvaluator.name, RegressorEvaluator)
+    module._model_evaluation_registry.register_builtin(ShapEvaluator.name, ShapEvaluator)
+
+    # Plugin evaluators
+    module._model_evaluation_registry.register_entrypoints()
+
+
+# Put it in post-importing hook to avoid circuit importing
+register_post_import_hook(register_evaluators, __name__, overwrite=True)

venv/lib/python3.9/site-packages/mlflow/models/evaluation/evaluators/classifier.py

@@ -0,0 +1,680 @@
+import logging
+import math
+from collections import namedtuple
+from contextlib import contextmanager
+from typing import Optional
+
+import numpy as np
+import pandas as pd
+from sklearn import metrics as sk_metrics
+
+import mlflow
+from mlflow import MlflowException
+from mlflow.environment_variables import _MLFLOW_EVALUATE_SUPPRESS_CLASSIFICATION_ERRORS
+from mlflow.models.evaluation.artifacts import CsvEvaluationArtifact
+from mlflow.models.evaluation.base import EvaluationMetric, EvaluationResult, _ModelType
+from mlflow.models.evaluation.default_evaluator import (
+    BuiltInEvaluator,
+    _extract_raw_model,
+    _get_aggregate_metrics_values,
+)
+from mlflow.models.utils import plot_lines
+
+_logger = logging.getLogger(__name__)
+
+
+_Curve = namedtuple("_Curve", ["plot_fn", "plot_fn_args", "auc"])
+
+
+class ClassifierEvaluator(BuiltInEvaluator):
+    """
+    A built-in evaluator for classifier models.
+    """
+
+    name = "classifier"
+
+    @classmethod
+    def can_evaluate(cls, *, model_type, evaluator_config, **kwargs):
+        # TODO: Also the model needs to be pyfunc model, not function or endpoint URI
+        return model_type == _ModelType.CLASSIFIER
+
+    def _evaluate(
+        self,
+        model: Optional["mlflow.pyfunc.PyFuncModel"],
+        extra_metrics: list[EvaluationMetric],
+        custom_artifacts=None,
+        **kwargs,
+    ) -> Optional[EvaluationResult]:
+        # Get classification config
+        self.y_true = self.dataset.labels_data
+        self.label_list = self.evaluator_config.get("label_list")
+        self.pos_label = self.evaluator_config.get("pos_label")
+        self.sample_weights = self.evaluator_config.get("sample_weights")
+        if self.pos_label and self.label_list and self.pos_label not in self.label_list:
+            raise MlflowException.invalid_parameter_value(
+                f"'pos_label' {self.pos_label} must exist in 'label_list' {self.label_list}."
+            )
+
+        # Check if the model_type is consistent with ground truth labels
+        inferred_model_type = _infer_model_type_by_labels(self.y_true)
+        if _ModelType.CLASSIFIER != inferred_model_type:
+            _logger.warning(
+                f"According to the evaluation dataset label values, the model type looks like "
+                f"{inferred_model_type}, but you specified model type 'classifier'. Please "
+                f"verify that you set the `model_type` and `dataset` arguments correctly."
+            )
+
+        # Run model prediction
+        input_df = self.X.copy_to_avoid_mutation()
+        self.y_pred, self.y_probs = self._generate_model_predictions(model, input_df)
+
+        self._validate_label_list()
+
+        self._compute_builtin_metrics(model)
+        self.evaluate_metrics(extra_metrics, prediction=self.y_pred, target=self.y_true)
+        self.evaluate_and_log_custom_artifacts(
+            custom_artifacts, prediction=self.y_pred, target=self.y_true
+        )
+
+        # Log metrics and artifacts
+        self.log_metrics()
+        self.log_eval_table(self.y_pred)
+
+        if len(self.label_list) == 2:
+            self._log_binary_classifier_artifacts()
+        else:
+            self._log_multiclass_classifier_artifacts()
+        self._log_confusion_matrix()
+
+        return EvaluationResult(
+            metrics=self.aggregate_metrics, artifacts=self.artifacts, run_id=self.run_id
+        )
+
+    def _generate_model_predictions(self, model, input_df):
+        predict_fn, predict_proba_fn = _extract_predict_fn_and_prodict_proba_fn(model)
+        # Classifier model is guaranteed to output single column of predictions
+        y_pred = self.dataset.predictions_data if model is None else predict_fn(input_df)
+
+        # Predict class probabilities if the model supports it
+        y_probs = predict_proba_fn(input_df) if predict_proba_fn is not None else None
+        return y_pred, y_probs
+
+    def _validate_label_list(self):
+        if self.label_list is None:
+            # If label list is not specified, infer label list from model output
+            self.label_list = np.unique(np.concatenate([self.y_true, self.y_pred]))
+        else:
+            # np.where only works for numpy array, not list
+            self.label_list = np.array(self.label_list)
+
+        # sort label_list ASC, for binary classification it makes sure the last one is pos label
+        self.label_list.sort()
+
+        is_binomial = len(self.label_list) <= 2
+        if is_binomial:
+            if self.pos_label is None:
+                self.pos_label = self.label_list[-1]
+            else:
+                if self.pos_label in self.label_list:
+                    self.label_list = np.delete(
+                        self.label_list, np.where(self.label_list == self.pos_label)
+                    )
+                self.label_list = np.append(self.label_list, self.pos_label)
+            if len(self.label_list) < 2:
+                raise MlflowException(
+                    "Evaluation dataset for classification must contain at least two unique "
+                    f"labels, but only {len(self.label_list)} unique labels were found.",
+                )
+            with _suppress_class_imbalance_errors(IndexError, log_warning=False):
+                _logger.info(
+                    "The evaluation dataset is inferred as binary dataset, positive label is "
+                    f"{self.label_list[1]}, negative label is {self.label_list[0]}."
+                )
+        else:
+            _logger.info(
+                "The evaluation dataset is inferred as multiclass dataset, number of classes "
+                f"is inferred as {len(self.label_list)}. If this is incorrect, please specify the "
+                "`label_list` parameter in `evaluator_config`."
+            )
+
+    def _compute_builtin_metrics(self, model):
+        self._evaluate_sklearn_model_score_if_scorable(model, self.y_true, self.sample_weights)
+
+        if len(self.label_list) <= 2:
+            metrics = _get_binary_classifier_metrics(
+                y_true=self.y_true,
+                y_pred=self.y_pred,
+                y_proba=self.y_probs,
+                labels=self.label_list,
+                pos_label=self.pos_label,
+                sample_weights=self.sample_weights,
+            )
+            if metrics:
+                self.metrics_values.update(_get_aggregate_metrics_values(metrics))
+                self._compute_roc_and_pr_curve()
+        else:
+            average = self.evaluator_config.get("average", "weighted")
+            metrics = _get_multiclass_classifier_metrics(
+                y_true=self.y_true,
+                y_pred=self.y_pred,
+                y_proba=self.y_probs,
+                labels=self.label_list,
+                average=average,
+                sample_weights=self.sample_weights,
+            )
+            if metrics:
+                self.metrics_values.update(_get_aggregate_metrics_values(metrics))
+
+    def _compute_roc_and_pr_curve(self):
+        if self.y_probs is not None:
+            with _suppress_class_imbalance_errors(ValueError, log_warning=False):
+                self.roc_curve = _gen_classifier_curve(
+                    is_binomial=True,
+                    y=self.y_true,
+                    y_probs=self.y_probs[:, 1],
+                    labels=self.label_list,
+                    pos_label=self.pos_label,
+                    curve_type="roc",
+                    sample_weights=self.sample_weights,
+                )
+
+                self.metrics_values.update(
+                    _get_aggregate_metrics_values({"roc_auc": self.roc_curve.auc})
+                )
+            with _suppress_class_imbalance_errors(ValueError, log_warning=False):
+                self.pr_curve = _gen_classifier_curve(
+                    is_binomial=True,
+                    y=self.y_true,
+                    y_probs=self.y_probs[:, 1],
+                    labels=self.label_list,
+                    pos_label=self.pos_label,
+                    curve_type="pr",
+                    sample_weights=self.sample_weights,
+                )
+
+                self.metrics_values.update(
+                    _get_aggregate_metrics_values({"precision_recall_auc": self.pr_curve.auc})
+                )
+
+    def _log_pandas_df_artifact(self, pandas_df, artifact_name):
+        artifact_file_name = f"{artifact_name}.csv"
+        artifact_file_local_path = self.temp_dir.path(artifact_file_name)
+        pandas_df.to_csv(artifact_file_local_path, index=False)
+        mlflow.log_artifact(artifact_file_local_path)
+        artifact = CsvEvaluationArtifact(
+            uri=mlflow.get_artifact_uri(artifact_file_name),
+            content=pandas_df,
+        )
+        artifact._load(artifact_file_local_path)
+        self.artifacts[artifact_name] = artifact
+
+    def _log_multiclass_classifier_artifacts(self):
+        per_class_metrics_collection_df = _get_classifier_per_class_metrics_collection_df(
+            y=self.y_true,
+            y_pred=self.y_pred,
+            labels=self.label_list,
+            sample_weights=self.sample_weights,
+        )
+
+        log_roc_pr_curve = False
+        if self.y_probs is not None:
+            max_classes_for_multiclass_roc_pr = self.evaluator_config.get(
+                "max_classes_for_multiclass_roc_pr", 10
+            )
+            if len(self.label_list) <= max_classes_for_multiclass_roc_pr:
+                log_roc_pr_curve = True
+            else:
+                _logger.warning(
+                    f"The classifier num_classes > {max_classes_for_multiclass_roc_pr}, skip "
+                    f"logging ROC curve and Precision-Recall curve. You can add evaluator config "
+                    f"'max_classes_for_multiclass_roc_pr' to increase the threshold."
+                )
+
+        if log_roc_pr_curve:
+            roc_curve = _gen_classifier_curve(
+                is_binomial=False,
+                y=self.y_true,
+                y_probs=self.y_probs,
+                labels=self.label_list,
+                pos_label=self.pos_label,
+                curve_type="roc",
+                sample_weights=self.sample_weights,
+            )
+
+            def plot_roc_curve():
+                roc_curve.plot_fn(**roc_curve.plot_fn_args)
+
+            self._log_image_artifact(plot_roc_curve, "roc_curve_plot")
+            per_class_metrics_collection_df["roc_auc"] = roc_curve.auc
+
+            pr_curve = _gen_classifier_curve(
+                is_binomial=False,
+                y=self.y_true,
+                y_probs=self.y_probs,
+                labels=self.label_list,
+                pos_label=self.pos_label,
+                curve_type="pr",
+                sample_weights=self.sample_weights,
+            )
+
+            def plot_pr_curve():
+                pr_curve.plot_fn(**pr_curve.plot_fn_args)
+
+            self._log_image_artifact(plot_pr_curve, "precision_recall_curve_plot")
+            per_class_metrics_collection_df["precision_recall_auc"] = pr_curve.auc
+
+        self._log_pandas_df_artifact(per_class_metrics_collection_df, "per_class_metrics")
+
+    def _log_roc_curve(self):
+        def _plot_roc_curve():
+            self.roc_curve.plot_fn(**self.roc_curve.plot_fn_args)
+
+        self._log_image_artifact(_plot_roc_curve, "roc_curve_plot")
+
+    def _log_precision_recall_curve(self):
+        def _plot_pr_curve():
+            self.pr_curve.plot_fn(**self.pr_curve.plot_fn_args)
+
+        self._log_image_artifact(_plot_pr_curve, "precision_recall_curve_plot")
+
+    def _log_lift_curve(self):
+        from mlflow.models.evaluation.lift_curve import plot_lift_curve
+
+        def _plot_lift_curve():
+            return plot_lift_curve(self.y_true, self.y_probs, pos_label=self.pos_label)
+
+        self._log_image_artifact(_plot_lift_curve, "lift_curve_plot")
+
+    def _log_binary_classifier_artifacts(self):
+        if self.y_probs is not None:
+            with _suppress_class_imbalance_errors(log_warning=False):
+                self._log_roc_curve()
+            with _suppress_class_imbalance_errors(log_warning=False):
+                self._log_precision_recall_curve()
+            with _suppress_class_imbalance_errors(ValueError, log_warning=False):
+                self._log_lift_curve()
+
+    def _log_confusion_matrix(self):
+        """
+        Helper method for logging confusion matrix
+        """
+        # normalize the confusion matrix, keep consistent with sklearn autologging.
+        confusion_matrix = sk_metrics.confusion_matrix(
+            self.y_true,
+            self.y_pred,
+            labels=self.label_list,
+            normalize="true",
+            sample_weight=self.sample_weights,
+        )
+
+        def plot_confusion_matrix():
+            import matplotlib
+            import matplotlib.pyplot as plt
+
+            with matplotlib.rc_context(
+                {
+                    "font.size": min(8, math.ceil(50.0 / len(self.label_list))),
+                    "axes.labelsize": 8,
+                }
+            ):
+                _, ax = plt.subplots(1, 1, figsize=(6.0, 4.0), dpi=175)
+                disp = sk_metrics.ConfusionMatrixDisplay(
+                    confusion_matrix=confusion_matrix,
+                    display_labels=self.label_list,
+                ).plot(cmap="Blues", ax=ax)
+                disp.ax_.set_title("Normalized confusion matrix")
+
+        if hasattr(sk_metrics, "ConfusionMatrixDisplay"):
+            self._log_image_artifact(
+                plot_confusion_matrix,
+                "confusion_matrix",
+            )
+        return
+
+
+def _is_categorical(values):
+    """
+    Infer whether input values are categorical on best effort.
+    Return True represent they are categorical, return False represent we cannot determine result.
+    """
+    dtype_name = pd.Series(values).convert_dtypes().dtype.name.lower()
+    return dtype_name in ["category", "string", "boolean"]
+
+
+def _is_continuous(values):
+    """
+    Infer whether input values is continuous on best effort.
+    Return True represent they are continuous, return False represent we cannot determine result.
+    """
+    dtype_name = pd.Series(values).convert_dtypes().dtype.name.lower()
+    return dtype_name.startswith("float")
+
+
+def _infer_model_type_by_labels(labels):
+    """
+    Infer model type by target values.
+    """
+    if _is_categorical(labels):
+        return _ModelType.CLASSIFIER
+    elif _is_continuous(labels):
+        return _ModelType.REGRESSOR
+    else:
+        return None  # Unknown
+
+
+def _extract_predict_fn_and_prodict_proba_fn(model):
+    predict_fn = None
+    predict_proba_fn = None
+
+    _, raw_model = _extract_raw_model(model)
+
+    if raw_model is not None:
+        predict_fn = raw_model.predict
+        predict_proba_fn = getattr(raw_model, "predict_proba", None)
+        try:
+            from mlflow.xgboost import (
+                _wrapped_xgboost_model_predict_fn,
+                _wrapped_xgboost_model_predict_proba_fn,
+            )
+
+            # Because shap evaluation will pass evaluation data in ndarray format
+            # (without feature names), if set validate_features=True it will raise error.
+            predict_fn = _wrapped_xgboost_model_predict_fn(raw_model, validate_features=False)
+            predict_proba_fn = _wrapped_xgboost_model_predict_proba_fn(
+                raw_model, validate_features=False
+            )
+        except ImportError:
+            pass
+    elif model is not None:
+        predict_fn = model.predict
+
+    return predict_fn, predict_proba_fn
+
+
+@contextmanager
+def _suppress_class_imbalance_errors(exception_type=Exception, log_warning=True):
+    """
+    Exception handler context manager to suppress Exceptions if the private environment
+    variable `_MLFLOW_EVALUATE_SUPPRESS_CLASSIFICATION_ERRORS` is set to `True`.
+    The purpose of this handler is to prevent an evaluation call for a binary or multiclass
+    classification automl run from aborting due to an extreme minority class imbalance
+    encountered during iterative training cycles due to the non deterministic sampling
+    behavior of Spark's DataFrame.sample() API.
+    The Exceptions caught in the usage of this are broad and are designed purely to not
+    interrupt the iterative hyperparameter tuning process. Final evaluations are done
+    in a more deterministic (but expensive) fashion.
+    """
+    try:
+        yield
+    except exception_type as e:
+        if _MLFLOW_EVALUATE_SUPPRESS_CLASSIFICATION_ERRORS.get():
+            if log_warning:
+                _logger.warning(
+                    "Failed to calculate metrics due to class imbalance. "
+                    "This is expected when the dataset is imbalanced."
+                )
+        else:
+            raise e
+
+
+def _get_binary_sum_up_label_pred_prob(positive_class_index, positive_class, y, y_pred, y_probs):
+    y = np.array(y)
+    y_bin = np.where(y == positive_class, 1, 0)
+    y_pred_bin = None
+    y_prob_bin = None
+    if y_pred is not None:
+        y_pred = np.array(y_pred)
+        y_pred_bin = np.where(y_pred == positive_class, 1, 0)
+
+    if y_probs is not None:
+        y_probs = np.array(y_probs)
+        y_prob_bin = y_probs[:, positive_class_index]
+
+    return y_bin, y_pred_bin, y_prob_bin
+
+
+def _get_common_classifier_metrics(
+    *, y_true, y_pred, y_proba, labels, average, pos_label, sample_weights
+):
+    metrics = {
+        "example_count": len(y_true),
+        "accuracy_score": sk_metrics.accuracy_score(y_true, y_pred, sample_weight=sample_weights),
+        "recall_score": sk_metrics.recall_score(
+            y_true,
+            y_pred,
+            average=average,
+            pos_label=pos_label,
+            sample_weight=sample_weights,
+        ),
+        "precision_score": sk_metrics.precision_score(
+            y_true,
+            y_pred,
+            average=average,
+            pos_label=pos_label,
+            sample_weight=sample_weights,
+        ),
+        "f1_score": sk_metrics.f1_score(
+            y_true,
+            y_pred,
+            average=average,
+            pos_label=pos_label,
+            sample_weight=sample_weights,
+        ),
+    }
+
+    if y_proba is not None:
+        with _suppress_class_imbalance_errors(ValueError):
+            metrics["log_loss"] = sk_metrics.log_loss(
+                y_true, y_proba, labels=labels, sample_weight=sample_weights
+            )
+    return metrics
+
+
+def _get_binary_classifier_metrics(
+    *, y_true, y_pred, y_proba=None, labels=None, pos_label=1, sample_weights=None
+):
+    with _suppress_class_imbalance_errors(ValueError):
+        tn, fp, fn, tp = sk_metrics.confusion_matrix(y_true, y_pred).ravel()
+        return {
+            "true_negatives": tn,
+            "false_positives": fp,
+            "false_negatives": fn,
+            "true_positives": tp,
+            **_get_common_classifier_metrics(
+                y_true=y_true,
+                y_pred=y_pred,
+                y_proba=y_proba,
+                labels=labels,
+                average="binary",
+                pos_label=pos_label,
+                sample_weights=sample_weights,
+            ),
+        }
+
+
+def _get_multiclass_classifier_metrics(
+    *,
+    y_true,
+    y_pred,
+    y_proba=None,
+    labels=None,
+    average="weighted",
+    sample_weights=None,
+):
+    metrics = _get_common_classifier_metrics(
+        y_true=y_true,
+        y_pred=y_pred,
+        y_proba=y_proba,
+        labels=labels,
+        average=average,
+        pos_label=None,
+        sample_weights=sample_weights,
+    )
+    if average in ("macro", "weighted") and y_proba is not None:
+        metrics.update(
+            roc_auc=sk_metrics.roc_auc_score(
+                y_true=y_true,
+                y_score=y_proba,
+                sample_weight=sample_weights,
+                average=average,
+                multi_class="ovr",
+            )
+        )
+    return metrics
+
+
+def _get_classifier_per_class_metrics_collection_df(y, y_pred, labels, sample_weights):
+    per_class_metrics_list = []
+    for positive_class_index, positive_class in enumerate(labels):
+        (
+            y_bin,
+            y_pred_bin,
+            _,
+        ) = _get_binary_sum_up_label_pred_prob(
+            positive_class_index, positive_class, y, y_pred, None
+        )
+        per_class_metrics = {"positive_class": positive_class}
+        binary_classifier_metrics = _get_binary_classifier_metrics(
+            y_true=y_bin,
+            y_pred=y_pred_bin,
+            pos_label=1,
+            sample_weights=sample_weights,
+        )
+        if binary_classifier_metrics:
+            per_class_metrics.update(binary_classifier_metrics)
+        per_class_metrics_list.append(per_class_metrics)
+
+    return pd.DataFrame(per_class_metrics_list)
+
+
+_Curve = namedtuple("_Curve", ["plot_fn", "plot_fn_args", "auc"])
+
+
+def _gen_classifier_curve(
+    is_binomial,
+    y,
+    y_probs,
+    labels,
+    pos_label,
+    curve_type,
+    sample_weights,
+):
+    """
+    Generate precision-recall curve or ROC curve for classifier.
+
+    Args:
+        is_binomial: True if it is binary classifier otherwise False
+        y: True label values
+        y_probs: if binary classifier, the predicted probability for positive class.
+                  if multiclass classifier, the predicted probabilities for all classes.
+        labels: The set of labels.
+        pos_label: The label of the positive class.
+        curve_type: "pr" or "roc"
+        sample_weights: Optional sample weights.
+
+    Returns:
+        An instance of "_Curve" which includes attributes "plot_fn", "plot_fn_args", "auc".
+    """
+    if curve_type == "roc":
+
+        def gen_line_x_y_label_auc(_y, _y_prob, _pos_label):
+            fpr, tpr, _ = sk_metrics.roc_curve(
+                _y,
+                _y_prob,
+                sample_weight=sample_weights,
+                # For multiclass classification where a one-vs-rest ROC curve is produced for each
+                # class, the positive label is binarized and should not be included in the plot
+                # legend
+                pos_label=_pos_label if _pos_label == pos_label else None,
+            )
+
+            auc = sk_metrics.roc_auc_score(y_true=_y, y_score=_y_prob, sample_weight=sample_weights)
+            return fpr, tpr, f"AUC={auc:.3f}", auc
+
+        xlabel = "False Positive Rate"
+        ylabel = "True Positive Rate"
+        title = "ROC curve"
+        if pos_label:
+            xlabel = f"False Positive Rate (Positive label: {pos_label})"
+            ylabel = f"True Positive Rate (Positive label: {pos_label})"
+    elif curve_type == "pr":
+
+        def gen_line_x_y_label_auc(_y, _y_prob, _pos_label):
+            precision, recall, _ = sk_metrics.precision_recall_curve(
+                _y,
+                _y_prob,
+                sample_weight=sample_weights,
+                # For multiclass classification where a one-vs-rest precision-recall curve is
+                # produced for each class, the positive label is binarized and should not be
+                # included in the plot legend
+                pos_label=_pos_label if _pos_label == pos_label else None,
+            )
+            # NB: We return average precision score (AP) instead of AUC because AP is more
+            # appropriate for summarizing a precision-recall curve
+            ap = sk_metrics.average_precision_score(
+                y_true=_y, y_score=_y_prob, pos_label=_pos_label, sample_weight=sample_weights
+            )
+            return recall, precision, f"AP={ap:.3f}", ap
+
+        xlabel = "Recall"
+        ylabel = "Precision"
+        title = "Precision recall curve"
+        if pos_label:
+            xlabel = f"Recall (Positive label: {pos_label})"
+            ylabel = f"Precision (Positive label: {pos_label})"
+    else:
+        assert False, "illegal curve type"
+
+    if is_binomial:
+        x_data, y_data, line_label, auc = gen_line_x_y_label_auc(y, y_probs, pos_label)
+        data_series = [(line_label, x_data, y_data)]
+    else:
+        curve_list = []
+        for positive_class_index, positive_class in enumerate(labels):
+            y_bin, _, y_prob_bin = _get_binary_sum_up_label_pred_prob(
+                positive_class_index, positive_class, y, labels, y_probs
+            )
+
+            x_data, y_data, line_label, auc = gen_line_x_y_label_auc(
+                y_bin, y_prob_bin, _pos_label=1
+            )
+            curve_list.append((positive_class, x_data, y_data, line_label, auc))
+
+        data_series = [
+            (f"label={positive_class},{line_label}", x_data, y_data)
+            for positive_class, x_data, y_data, line_label, _ in curve_list
+        ]
+        auc = [auc for _, _, _, _, auc in curve_list]
+
+    def _do_plot(**kwargs):
+        from matplotlib import pyplot
+
+        _, ax = plot_lines(**kwargs)
+        dash_line_args = {
+            "color": "gray",
+            "alpha": 0.3,
+            "drawstyle": "default",
+            "linestyle": "dashed",
+        }
+        if curve_type == "pr":
+            ax.plot([0, 1], [1, 0], **dash_line_args)
+        elif curve_type == "roc":
+            ax.plot([0, 1], [0, 1], **dash_line_args)
+
+        if is_binomial:
+            ax.legend(loc="best")
+        else:
+            ax.legend(loc="center left", bbox_to_anchor=(1, 0.5))
+            pyplot.subplots_adjust(right=0.6, bottom=0.25)
+
+    return _Curve(
+        plot_fn=_do_plot,
+        plot_fn_args={
+            "data_series": data_series,
+            "xlabel": xlabel,
+            "ylabel": ylabel,
+            "line_kwargs": {"drawstyle": "steps-post", "linewidth": 1},
+            "title": title,
+        },
+        auc=auc,
+    )

venv/lib/python3.9/site-packages/mlflow/models/evaluation/evaluators/default.py

@@ -0,0 +1,233 @@
+import logging
+import os
+import time
+from typing import Optional
+
+import numpy as np
+import pandas as pd
+
+import mlflow
+from mlflow.entities.metric import Metric
+from mlflow.exceptions import MlflowException
+from mlflow.metrics import (
+    MetricValue,
+    ari_grade_level,
+    exact_match,
+    flesch_kincaid_grade_level,
+    ndcg_at_k,
+    precision_at_k,
+    recall_at_k,
+    rouge1,
+    rouge2,
+    rougeL,
+    rougeLsum,
+    token_count,
+    toxicity,
+)
+from mlflow.metrics.genai.genai_metric import _GENAI_CUSTOM_METRICS_FILE_NAME
+from mlflow.models.evaluation.artifacts import JsonEvaluationArtifact
+from mlflow.models.evaluation.base import EvaluationMetric, EvaluationResult, _ModelType
+from mlflow.models.evaluation.default_evaluator import (
+    _LATENCY_METRIC_NAME,
+    BuiltInEvaluator,
+    _extract_output_and_other_columns,
+    _extract_predict_fn,
+)
+from mlflow.protos.databricks_pb2 import INVALID_PARAMETER_VALUE
+
+_logger = logging.getLogger(__name__)
+
+
+class DefaultEvaluator(BuiltInEvaluator):
+    """
+    The default built-in evaluator for any models that cannot be evaluated
+    by other built-in evaluators, such as question-answering.
+    """
+
+    name = "default"
+
+    @classmethod
+    def can_evaluate(cls, *, model_type, evaluator_config, **kwargs):
+        return model_type in _ModelType.values() or model_type is None
+
+    def _evaluate(
+        self,
+        model: Optional["mlflow.pyfunc.PyFuncModel"],
+        extra_metrics: list[EvaluationMetric],
+        custom_artifacts=None,
+        **kwargs,
+    ) -> Optional[EvaluationResult]:
+        compute_latency = False
+        for extra_metric in extra_metrics:
+            # If latency metric is specified, we will compute latency for the model
+            # during prediction, and we will remove the metric from the list of extra
+            # metrics to be computed after prediction.
+            if extra_metric.name == _LATENCY_METRIC_NAME:
+                compute_latency = True
+                extra_metrics.remove(extra_metric)
+        self._log_genai_custom_metrics(extra_metrics)
+
+        # Generate model predictions and evaluate metrics
+        y_pred, other_model_outputs, self.predictions = self._generate_model_predictions(
+            model, input_df=self.X.copy_to_avoid_mutation(), compute_latency=compute_latency
+        )
+        y_true = self.dataset.labels_data
+
+        metrics = self._builtin_metrics() + extra_metrics
+        self.evaluate_metrics(
+            metrics,
+            prediction=y_pred,
+            target=self.dataset.labels_data,
+            other_output_df=other_model_outputs,
+        )
+        self.evaluate_and_log_custom_artifacts(custom_artifacts, prediction=y_pred, target=y_true)
+
+        # Log metrics and artifacts
+        self.log_metrics()
+        self.log_eval_table(y_pred, other_model_outputs)
+        return EvaluationResult(
+            metrics=self.aggregate_metrics, artifacts=self.artifacts, run_id=self.run_id
+        )
+
+    def _builtin_metrics(self) -> list[Metric]:
+        """
+        Get a list of builtin metrics for the model type.
+        """
+        text_metrics = [
+            token_count(),
+            toxicity(),
+            flesch_kincaid_grade_level(),
+            ari_grade_level(),
+        ]
+        builtin_metrics = []
+
+        # NB: Classifier and Regressor are handled by dedicated built-in evaluators,
+        if self.model_type == _ModelType.QUESTION_ANSWERING:
+            builtin_metrics = [*text_metrics, exact_match()]
+        elif self.model_type == _ModelType.TEXT_SUMMARIZATION:
+            builtin_metrics = [
+                *text_metrics,
+                rouge1(),
+                rouge2(),
+                rougeL(),
+                rougeLsum(),
+            ]
+        elif self.model_type == _ModelType.TEXT:
+            builtin_metrics = text_metrics
+        elif self.model_type == _ModelType.RETRIEVER:
+            # default k to 3 if not specified
+            retriever_k = self.evaluator_config.pop("retriever_k", 3)
+            builtin_metrics = [
+                precision_at_k(retriever_k),
+                recall_at_k(retriever_k),
+                ndcg_at_k(retriever_k),
+            ]
+
+        return builtin_metrics
+
+    def _generate_model_predictions(
+        self,
+        model: Optional["mlflow.pyfunc.PyFuncModel"],
+        input_df: pd.DataFrame,
+        compute_latency=False,
+    ):
+        """
+        Helper method for generating model predictions
+        """
+        predict_fn = _extract_predict_fn(model)
+
+        def predict_with_latency(X_copy):
+            y_pred_list = []
+            pred_latencies = []
+            if len(X_copy) == 0:
+                raise ValueError("Empty input data")
+
+            is_dataframe = isinstance(X_copy, pd.DataFrame)
+
+            for row in X_copy.iterrows() if is_dataframe else enumerate(X_copy):
+                i, row_data = row
+                single_input = row_data.to_frame().T if is_dataframe else row_data
+                start_time = time.time()
+                y_pred = predict_fn(single_input)
+                end_time = time.time()
+                pred_latencies.append(end_time - start_time)
+                y_pred_list.append(y_pred)
+
+            # Update latency metric
+            self.metrics_values.update({_LATENCY_METRIC_NAME: MetricValue(scores=pred_latencies)})
+
+            # Aggregate all predictions into model_predictions
+            sample_pred = y_pred_list[0]
+            if isinstance(sample_pred, pd.DataFrame):
+                return pd.concat(y_pred_list)
+            elif isinstance(sample_pred, np.ndarray):
+                return np.concatenate(y_pred_list, axis=0)
+            elif isinstance(sample_pred, list):
+                return sum(y_pred_list, [])
+            elif isinstance(sample_pred, pd.Series):
+                return pd.concat(y_pred_list, ignore_index=True)
+            elif isinstance(sample_pred, str):
+                return y_pred_list
+            else:
+                raise MlflowException(
+                    message=f"Unsupported prediction type {type(sample_pred)} for model type "
+                    f"{self.model_type}.",
+                    error_code=INVALID_PARAMETER_VALUE,
+                )
+
+        if model is not None:
+            _logger.info("Computing model predictions.")
+
+            if compute_latency:
+                model_predictions = predict_with_latency(input_df)
+            else:
+                model_predictions = predict_fn(input_df)
+        else:
+            if compute_latency:
+                _logger.warning(
+                    "Setting the latency to 0 for all entries because the model is not provided."
+                )
+                self.metrics_values.update(
+                    {_LATENCY_METRIC_NAME: MetricValue(scores=[0.0] * len(input_df))}
+                )
+            model_predictions = self.dataset.predictions_data
+
+        output_column_name = self.predictions
+        (
+            y_pred,
+            other_output_df,
+            predictions_column_name,
+        ) = _extract_output_and_other_columns(model_predictions, output_column_name)
+
+        return y_pred, other_output_df, predictions_column_name
+
+    def _log_genai_custom_metrics(self, extra_metrics: list[EvaluationMetric]):
+        genai_custom_metrics = [
+            extra_metric.genai_metric_args
+            for extra_metric in extra_metrics
+            # When the field is present, the metric is created from either make_genai_metric
+            # or make_genai_metric_from_prompt. We will log the metric definition.
+            if extra_metric.genai_metric_args is not None
+        ]
+
+        if len(genai_custom_metrics) == 0:
+            return
+
+        names = []
+        versions = []
+        metric_args_list = []
+
+        for metric_args in genai_custom_metrics:
+            names.append(metric_args["name"])
+            # Custom metrics created from make_genai_metric_from_prompt don't have version
+            versions.append(metric_args.get("version", ""))
+            metric_args_list.append(metric_args)
+
+        data = {"name": names, "version": versions, "metric_args": metric_args_list}
+
+        mlflow.log_table(data, artifact_file=_GENAI_CUSTOM_METRICS_FILE_NAME)
+
+        artifact_name = os.path.splitext(_GENAI_CUSTOM_METRICS_FILE_NAME)[0]
+        self.artifacts[artifact_name] = JsonEvaluationArtifact(
+            uri=mlflow.get_artifact_uri(_GENAI_CUSTOM_METRICS_FILE_NAME)
+        )

venv/lib/python3.9/site-packages/mlflow/models/evaluation/evaluators/regressor.py

@@ -0,0 +1,96 @@
+from typing import Optional
+
+import numpy as np
+from sklearn import metrics as sk_metrics
+
+import mlflow
+from mlflow.models.evaluation.base import EvaluationMetric, EvaluationResult, _ModelType
+from mlflow.models.evaluation.default_evaluator import (
+    BuiltInEvaluator,
+    _extract_output_and_other_columns,
+    _extract_predict_fn,
+    _get_aggregate_metrics_values,
+)
+
+
+class RegressorEvaluator(BuiltInEvaluator):
+    """
+    A built-in evaluator for regressor models.
+    """
+
+    name = "regressor"
+
+    @classmethod
+    def can_evaluate(cls, *, model_type, evaluator_config, **kwargs):
+        return model_type == _ModelType.REGRESSOR
+
+    def _evaluate(
+        self,
+        model: Optional["mlflow.pyfunc.PyFuncModel"],
+        extra_metrics: list[EvaluationMetric],
+        custom_artifacts=None,
+        **kwargs,
+    ) -> Optional[EvaluationResult]:
+        self.y_true = self.dataset.labels_data
+        self.sample_weights = self.evaluator_config.get("sample_weights", None)
+
+        input_df = self.X.copy_to_avoid_mutation()
+        self.y_pred = self._generate_model_predictions(model, input_df)
+        self._compute_buildin_metrics(model)
+
+        self.evaluate_metrics(extra_metrics, prediction=self.y_pred, target=self.y_true)
+        self.evaluate_and_log_custom_artifacts(
+            custom_artifacts, prediction=self.y_pred, target=self.y_true
+        )
+
+        self.log_metrics()
+        self.log_eval_table(self.y_pred)
+
+        return EvaluationResult(
+            metrics=self.aggregate_metrics, artifacts=self.artifacts, run_id=self.run_id
+        )
+
+    def _generate_model_predictions(self, model, input_df):
+        if predict_fn := _extract_predict_fn(model):
+            preds = predict_fn(input_df)
+            y_pred, _, _ = _extract_output_and_other_columns(preds, self.predictions)
+            return y_pred
+        else:
+            return self.dataset.predictions_data
+
+    def _compute_buildin_metrics(self, model):
+        self._evaluate_sklearn_model_score_if_scorable(model, self.y_true, self.sample_weights)
+        self.metrics_values.update(
+            _get_aggregate_metrics_values(
+                _get_regressor_metrics(self.y_true, self.y_pred, self.sample_weights)
+            )
+        )
+
+
+def _get_regressor_metrics(y, y_pred, sample_weights):
+    from mlflow.metrics.metric_definitions import _root_mean_squared_error
+
+    sum_on_target = (
+        (np.array(y) * np.array(sample_weights)).sum() if sample_weights is not None else sum(y)
+    )
+    return {
+        "example_count": len(y),
+        "mean_absolute_error": sk_metrics.mean_absolute_error(
+            y, y_pred, sample_weight=sample_weights
+        ),
+        "mean_squared_error": sk_metrics.mean_squared_error(
+            y, y_pred, sample_weight=sample_weights
+        ),
+        "root_mean_squared_error": _root_mean_squared_error(
+            y_true=y,
+            y_pred=y_pred,
+            sample_weight=sample_weights,
+        ),
+        "sum_on_target": sum_on_target,
+        "mean_on_target": sum_on_target / len(y),
+        "r2_score": sk_metrics.r2_score(y, y_pred, sample_weight=sample_weights),
+        "max_error": sk_metrics.max_error(y, y_pred),
+        "mean_absolute_percentage_error": sk_metrics.mean_absolute_percentage_error(
+            y, y_pred, sample_weight=sample_weights
+        ),
+    }

venv/lib/python3.9/site-packages/mlflow/models/evaluation/evaluators/shap.py

@@ -0,0 +1,293 @@
+import functools
+import logging
+from typing import Optional
+
+import numpy as np
+from packaging.version import Version
+from sklearn.pipeline import Pipeline as sk_Pipeline
+
+import mlflow
+from mlflow import MlflowException
+from mlflow.models.evaluation.base import EvaluationMetric, EvaluationResult, _ModelType
+from mlflow.models.evaluation.default_evaluator import (
+    BuiltInEvaluator,
+    _extract_predict_fn,
+    _extract_raw_model,
+    _get_dataframe_with_renamed_columns,
+)
+from mlflow.models.evaluation.evaluators.classifier import (
+    _is_continuous,
+    _suppress_class_imbalance_errors,
+)
+from mlflow.protos.databricks_pb2 import INVALID_PARAMETER_VALUE
+from mlflow.pyfunc import _ServedPyFuncModel
+
+_logger = logging.getLogger(__name__)
+
+
+_SUPPORTED_SHAP_ALGORITHMS = ("exact", "permutation", "partition", "kernel")
+_DEFAULT_SAMPLE_ROWS_FOR_SHAP = 2000
+
+
+def _shap_predict_fn(x, predict_fn, feature_names):
+    return predict_fn(_get_dataframe_with_renamed_columns(x, feature_names))
+
+
+class ShapEvaluator(BuiltInEvaluator):
+    """
+    A built-in evaluator to get SHAP explainability insights for classifier and regressor models.
+
+    This evaluator often run with the main evaluator for the model like ClassifierEvaluator.
+    """
+
+    name = "shap"
+
+    @classmethod
+    def can_evaluate(cls, *, model_type, evaluator_config, **kwargs):
+        return model_type in (_ModelType.CLASSIFIER, _ModelType.REGRESSOR) and evaluator_config.get(
+            "log_model_explainability", True
+        )
+
+    def _evaluate(
+        self,
+        model: Optional["mlflow.pyfunc.PyFuncModel"],
+        extra_metrics: list[EvaluationMetric],
+        custom_artifacts=None,
+        **kwargs,
+    ) -> Optional[EvaluationResult]:
+        if isinstance(model, _ServedPyFuncModel):
+            _logger.warning(
+                "Skipping model explainability because a model server is used for environment "
+                "restoration."
+            )
+            return
+
+        model_loader_module, raw_model = _extract_raw_model(model)
+        if model_loader_module == "mlflow.spark":
+            # TODO: Shap explainer need to manipulate on each feature values,
+            #  but spark model input dataframe contains Vector type feature column
+            #  which shap explainer does not support.
+            #  To support this, we need expand the Vector type feature column into
+            #  multiple scalar feature columns and pass it to shap explainer.
+            _logger.warning(
+                "Logging model explainability insights is not currently supported for PySpark "
+                "models."
+            )
+            return
+
+        self.y_true = self.dataset.labels_data
+        self.label_list = self.evaluator_config.get("label_list")
+        self.pos_label = self.evaluator_config.get("pos_label")
+
+        if not (np.issubdtype(self.y_true.dtype, np.number) or self.y_true.dtype == np.bool_):
+            # Note: python bool type inherits number type but np.bool_ does not inherit np.number.
+            _logger.warning(
+                "Skip logging model explainability insights because it requires all label "
+                "values to be numeric or boolean."
+            )
+            return
+
+        algorithm = self.evaluator_config.get("explainability_algorithm", None)
+        if algorithm is not None and algorithm not in _SUPPORTED_SHAP_ALGORITHMS:
+            raise MlflowException(
+                message=f"Specified explainer algorithm {algorithm} is unsupported. Currently only "
+                f"support {','.join(_SUPPORTED_SHAP_ALGORITHMS)} algorithms.",
+                error_code=INVALID_PARAMETER_VALUE,
+            )
+
+        if algorithm != "kernel":
+            feature_dtypes = list(self.X.get_original().dtypes)
+            for feature_dtype in feature_dtypes:
+                if not np.issubdtype(feature_dtype, np.number):
+                    _logger.warning(
+                        "Skip logging model explainability insights because the shap explainer "
+                        f"{algorithm} requires all feature values to be numeric, and each feature "
+                        "column must only contain scalar values."
+                    )
+                    return
+
+        try:
+            import shap
+            from matplotlib import pyplot
+        except ImportError:
+            _logger.warning(
+                "SHAP or matplotlib package is not installed, so model explainability insights "
+                "will not be logged."
+            )
+            return
+
+        if Version(shap.__version__) < Version("0.40"):
+            _logger.warning(
+                "Shap package version is lower than 0.40, Skip log model explainability."
+            )
+            return
+
+        sample_rows = self.evaluator_config.get(
+            "explainability_nsamples", _DEFAULT_SAMPLE_ROWS_FOR_SHAP
+        )
+
+        X_df = self.X.copy_to_avoid_mutation()
+
+        sampled_X = shap.sample(X_df, sample_rows, random_state=0)
+
+        mode_or_mean_dict = _compute_df_mode_or_mean(X_df)
+        sampled_X = sampled_X.fillna(mode_or_mean_dict)
+
+        # shap explainer might call provided `predict_fn` with a `numpy.ndarray` type
+        # argument, this might break some model inference, so convert the argument into
+        # a pandas dataframe.
+        # The `shap_predict_fn` calls model's predict function, we need to restore the input
+        # dataframe with original column names, because some model prediction routine uses
+        # the column name.
+
+        predict_fn = _extract_predict_fn(model)
+        shap_predict_fn = functools.partial(
+            _shap_predict_fn, predict_fn=predict_fn, feature_names=self.dataset.feature_names
+        )
+
+        if self.label_list is None:
+            # If label list is not specified, infer label list from model output.
+            # We need to copy the input data as the model might mutate the input data.
+            y_pred = predict_fn(X_df.copy()) if predict_fn else self.dataset.predictions_data
+            self.label_list = np.unique(np.concatenate([self.y_true, y_pred]))
+
+        try:
+            if algorithm:
+                if algorithm == "kernel":
+                    # We need to lazily import shap, so lazily import `_PatchedKernelExplainer`
+                    from mlflow.models.evaluation._shap_patch import _PatchedKernelExplainer
+
+                    kernel_link = self.evaluator_config.get(
+                        "explainability_kernel_link", "identity"
+                    )
+                    if kernel_link not in ["identity", "logit"]:
+                        raise ValueError(
+                            "explainability_kernel_link config can only be set to 'identity' or "
+                            f"'logit', but got '{kernel_link}'."
+                        )
+                    background_X = shap.sample(X_df, sample_rows, random_state=3)
+                    background_X = background_X.fillna(mode_or_mean_dict)
+
+                    explainer = _PatchedKernelExplainer(
+                        shap_predict_fn, background_X, link=kernel_link
+                    )
+                else:
+                    explainer = shap.Explainer(
+                        shap_predict_fn,
+                        sampled_X,
+                        feature_names=self.dataset.feature_names,
+                        algorithm=algorithm,
+                    )
+            else:
+                if (
+                    raw_model
+                    and not len(self.label_list) > 2
+                    and not isinstance(raw_model, sk_Pipeline)
+                ):
+                    # For mulitnomial classifier, shap.Explainer may choose Tree/Linear explainer
+                    # for raw model, this case shap plot doesn't support it well, so exclude the
+                    # multinomial_classifier case here.
+                    explainer = shap.Explainer(
+                        raw_model, sampled_X, feature_names=self.dataset.feature_names
+                    )
+                else:
+                    # fallback to default explainer
+                    explainer = shap.Explainer(
+                        shap_predict_fn, sampled_X, feature_names=self.dataset.feature_names
+                    )
+
+            _logger.info(f"Shap explainer {explainer.__class__.__name__} is used.")
+
+            if algorithm == "kernel":
+                shap_values = shap.Explanation(
+                    explainer.shap_values(sampled_X), feature_names=self.dataset.feature_names
+                )
+            else:
+                shap_values = explainer(sampled_X)
+        except Exception as e:
+            # Shap evaluation might fail on some edge cases, e.g., unsupported input data values
+            # or unsupported model on specific shap explainer. Catch exception to prevent it
+            # breaking the whole `evaluate` function.
+
+            if not self.evaluator_config.get("ignore_exceptions", True):
+                raise e
+
+            _logger.warning(
+                f"Shap evaluation failed. Reason: {e!r}. "
+                "Set logging level to DEBUG to see the full traceback."
+            )
+            _logger.debug("", exc_info=True)
+            return
+        try:
+            mlflow.shap.log_explainer(explainer, artifact_path="explainer")
+        except Exception as e:
+            # TODO: The explainer saver is buggy, if `get_underlying_model_flavor` return "unknown",
+            #   then fallback to shap explainer saver, and shap explainer will call `model.save`
+            #   for sklearn model, there is no `.save` method, so error will happen.
+            _logger.warning(
+                f"Logging explainer failed. Reason: {e!r}. "
+                "Set logging level to DEBUG to see the full traceback."
+            )
+            _logger.debug("", exc_info=True)
+
+        def _adjust_color_bar():
+            pyplot.gcf().axes[-1].set_aspect("auto")
+            pyplot.gcf().axes[-1].set_box_aspect(50)
+
+        def _adjust_axis_tick():
+            pyplot.xticks(fontsize=10)
+            pyplot.yticks(fontsize=10)
+
+        def plot_beeswarm():
+            shap.plots.beeswarm(shap_values, show=False, color_bar=True)
+            _adjust_color_bar()
+            _adjust_axis_tick()
+
+        with _suppress_class_imbalance_errors(ValueError, log_warning=False):
+            self._log_image_artifact(
+                plot_beeswarm,
+                "shap_beeswarm_plot",
+            )
+
+        def plot_summary():
+            shap.summary_plot(shap_values, show=False, color_bar=True)
+            _adjust_color_bar()
+            _adjust_axis_tick()
+
+        with _suppress_class_imbalance_errors(TypeError, log_warning=False):
+            self._log_image_artifact(
+                plot_summary,
+                "shap_summary_plot",
+            )
+
+        def plot_feature_importance():
+            shap.plots.bar(shap_values, show=False)
+            _adjust_axis_tick()
+
+        with _suppress_class_imbalance_errors(IndexError, log_warning=False):
+            self._log_image_artifact(
+                plot_feature_importance,
+                "shap_feature_importance_plot",
+            )
+
+        return EvaluationResult(
+            metrics=self.aggregate_metrics,
+            artifacts=self.artifacts,
+            run_id=self.run_id,
+        )
+
+
+def _compute_df_mode_or_mean(df):
+    """
+    Compute mean (for continuous columns) and compute mode (for other columns) for the
+    input dataframe, return a dict, key is column name, value is the corresponding mode or
+    mean value, this function calls `_is_continuous` to determine whether the
+    column is continuous column.
+    """
+    continuous_cols = [c for c in df.columns if _is_continuous(df[c])]
+    df_cont = df[continuous_cols]
+    df_non_cont = df.drop(continuous_cols, axis=1)
+
+    means = {} if df_cont.empty else df_cont.mean().to_dict()
+    modes = {} if df_non_cont.empty else df_non_cont.mode().loc[0].to_dict()
+    return {**means, **modes}

venv/lib/python3.9/site-packages/mlflow/models/evaluation/lift_curve.py

@@ -0,0 +1,177 @@
+import matplotlib.pyplot as plt
+import numpy as np
+
+
+def _cumulative_gain_curve(y_true, y_score, pos_label=None):
+    """
+    This method is copied from scikit-plot package.
+    See https://github.com/reiinakano/scikit-plot/blob/2dd3e6a76df77edcbd724c4db25575f70abb57cb/scikitplot/helpers.py#L157
+
+    This function generates the points necessary to plot the Cumulative Gain
+
+    Note: This implementation is restricted to the binary classification task.
+
+    Args:
+        y_true (array-like, shape (n_samples)): True labels of the data.
+
+        y_score (array-like, shape (n_samples)): Target scores, can either be
+            probability estimates of the positive class, confidence values, or
+            non-thresholded measure of decisions (as returned by
+            decision_function on some classifiers).
+
+        pos_label (int or str, default=None): Label considered as positive and
+            others are considered negative
+
+    Returns:
+        percentages (numpy.ndarray): An array containing the X-axis values for
+            plotting the Cumulative Gains chart.
+
+        gains (numpy.ndarray): An array containing the Y-axis values for one
+            curve of the Cumulative Gains chart.
+
+    Raises:
+        ValueError: If `y_true` is not composed of 2 classes. The Cumulative
+            Gain Chart is only relevant in binary classification.
+    """
+    y_true, y_score = np.asarray(y_true), np.asarray(y_score)
+
+    # ensure binary classification if pos_label is not specified
+    classes = np.unique(y_true)
+    if pos_label is None and not (
+        np.array_equal(classes, [0, 1])
+        or np.array_equal(classes, [-1, 1])
+        or np.array_equal(classes, [0])
+        or np.array_equal(classes, [-1])
+        or np.array_equal(classes, [1])
+    ):
+        raise ValueError("Data is not binary and pos_label is not specified")
+    elif pos_label is None:
+        pos_label = 1.0
+
+    # make y_true a boolean vector
+    y_true = y_true == pos_label
+
+    sorted_indices = np.argsort(y_score)[::-1]
+    y_true = y_true[sorted_indices]
+    gains = np.cumsum(y_true)
+
+    percentages = np.arange(start=1, stop=len(y_true) + 1)
+
+    gains = gains / float(np.sum(y_true))
+    percentages = percentages / float(len(y_true))
+
+    gains = np.insert(gains, 0, [0])
+    percentages = np.insert(percentages, 0, [0])
+
+    return percentages, gains
+
+
+def plot_lift_curve(
+    y_true,
+    y_probas,
+    title="Lift Curve",
+    ax=None,
+    figsize=None,
+    title_fontsize="large",
+    text_fontsize="medium",
+    pos_label=None,
+):
+    """
+    This method is copied from scikit-plot package.
+    See https://github.com/reiinakano/scikit-plot/blob/2dd3e6a76df77edcbd724c4db25575f70abb57cb/scikitplot/metrics.py#L1133
+
+    Generates the Lift Curve from labels and scores/probabilities
+
+    The lift curve is used to determine the effectiveness of a
+    binary classifier. A detailed explanation can be found at
+    http://www2.cs.uregina.ca/~dbd/cs831/notes/lift_chart/lift_chart.html.
+    The implementation here works only for binary classification.
+
+    Args:
+        y_true (array-like, shape (n_samples)):
+            Ground truth (correct) target values.
+
+        y_probas (array-like, shape (n_samples, n_classes)):
+            Prediction probabilities for each class returned by a classifier.
+
+        title (string, optional): Title of the generated plot. Defaults to
+            "Lift Curve".
+
+        ax (:class:`matplotlib.axes.Axes`, optional): The axes upon which to
+            plot the learning curve. If None, the plot is drawn on a new set of
+            axes.
+
+        figsize (2-tuple, optional): Tuple denoting figure size of the plot
+            e.g. (6, 6). Defaults to ``None``.
+
+        title_fontsize (string or int, optional): Matplotlib-style fontsizes.
+            Use e.g. "small", "medium", "large" or integer-values. Defaults to
+            "large".
+
+        text_fontsize (string or int, optional): Matplotlib-style fontsizes.
+            Use e.g. "small", "medium", "large" or integer-values. Defaults to
+            "medium".
+
+        pos_label (optional): Label for the positive class.
+
+    Returns:
+        ax (:class:`matplotlib.axes.Axes`): The axes on which the plot was
+            drawn.
+
+    Example:
+        >>> lr = LogisticRegression()
+        >>> lr = lr.fit(X_train, y_train)
+        >>> y_probas = lr.predict_proba(X_test)
+        >>> plot_lift_curve(y_test, y_probas)
+        <matplotlib.axes._subplots.AxesSubplot object at 0x7fe967d64490>
+        >>> plt.show()
+
+        .. image:: _static/examples/plot_lift_curve.png
+           :align: center
+           :alt: Lift Curve
+    """
+    y_true = np.array(y_true)
+    y_probas = np.array(y_probas)
+
+    classes = np.unique(y_true)
+    if len(classes) != 2:
+        raise ValueError(f"Cannot calculate Lift Curve for data with {len(classes)} category/ies")
+
+    # Compute Cumulative Gain Curves
+    percentages, gains1 = _cumulative_gain_curve(y_true, y_probas[:, 0], classes[0])
+    percentages, gains2 = _cumulative_gain_curve(y_true, y_probas[:, 1], classes[1])
+
+    percentages = percentages[1:]
+    gains1 = gains1[1:]
+    gains2 = gains2[1:]
+
+    gains1 = gains1 / percentages
+    gains2 = gains2 / percentages
+
+    if ax is None:
+        _, ax = plt.subplots(1, 1, figsize=figsize)
+
+    ax.set_title(title, fontsize=title_fontsize)
+
+    label0 = f"Class {classes[0]}"
+    label1 = f"Class {classes[1]}"
+    # show (positive) next to the positive class in the legend
+    if pos_label:
+        if pos_label == classes[0]:
+            label0 = f"Class {classes[0]} (positive)"
+        elif pos_label == classes[1]:
+            label1 = f"Class {classes[1]} (positive)"
+        # do not mark positive class if pos_label is not in classes
+
+    ax.plot(percentages, gains1, lw=3, label=label0)
+    ax.plot(percentages, gains2, lw=3, label=label1)
+
+    ax.plot([0, 1], [1, 1], "k--", lw=2, label="Baseline")
+
+    ax.set_xlabel("Percentage of sample", fontsize=text_fontsize)
+    ax.set_ylabel("Lift", fontsize=text_fontsize)
+    ax.tick_params(labelsize=text_fontsize)
+    ax.grid("on")
+    ax.legend(loc="best", fontsize=text_fontsize)
+
+    return ax

venv/lib/python3.9/site-packages/mlflow/models/evaluation/utils/metric.py

@@ -0,0 +1,123 @@
+import logging
+from dataclasses import dataclass
+from typing import Any, Callable, Optional
+
+import numpy as np
+
+from mlflow.metrics.base import MetricValue
+from mlflow.models.evaluation.base import EvaluationMetric
+
+_logger = logging.getLogger(__name__)
+
+
+@dataclass
+class MetricDefinition:
+    """
+    A namedtuple representing a metric function and its properties.
+
+    function : the metric function
+    name : the name of the metric function
+    index : the index of the function in the ``extra_metrics`` argument of mlflow.evaluate
+    """
+
+    function: Callable[..., Any]
+    name: str
+    index: int
+    version: Optional[str] = None
+    genai_metric_args: Optional[dict[str, Any]] = None
+
+    @classmethod
+    def from_index_and_metric(cls, index: int, metric: EvaluationMetric):
+        return cls(
+            function=metric.eval_fn,
+            index=index,
+            name=metric.name,
+            version=metric.version,
+            genai_metric_args=metric.genai_metric_args,
+        )
+
+    def evaluate(self, eval_fn_args) -> Optional[MetricValue]:
+        """
+        This function calls the metric function and performs validations on the returned
+        result to ensure that they are in the expected format. It will warn and will not log metrics
+        that are in the wrong format.
+
+        Args:
+            eval_fn_args: A dictionary of args needed to compute the eval metrics.
+
+        Returns:
+            MetricValue
+        """
+        if self.index < 0:
+            exception_header = f"Did not log builtin metric '{self.name}' because it"
+        else:
+            exception_header = (
+                f"Did not log metric '{self.name}' at index "
+                f"{self.index} in the `extra_metrics` parameter because it"
+            )
+
+        metric: MetricValue = self.function(*eval_fn_args)
+
+        def _is_numeric(value):
+            return isinstance(value, (int, float, np.number))
+
+        def _is_string(value):
+            return isinstance(value, str)
+
+        if metric is None:
+            _logger.warning(f"{exception_header} returned None.")
+            return
+
+        if _is_numeric(metric):
+            return MetricValue(aggregate_results={self.name: metric})
+
+        if not isinstance(metric, MetricValue):
+            _logger.warning(f"{exception_header} did not return a MetricValue.")
+            return
+
+        scores = metric.scores
+        justifications = metric.justifications
+        aggregates = metric.aggregate_results
+
+        if scores is not None:
+            if not isinstance(scores, list):
+                _logger.warning(
+                    f"{exception_header} must return MetricValue with scores as a list."
+                )
+                return
+            if any(not (_is_numeric(s) or _is_string(s) or s is None) for s in scores):
+                _logger.warning(
+                    f"{exception_header} must return MetricValue with numeric or string scores."
+                )
+                return
+
+        if justifications is not None:
+            if not isinstance(justifications, list):
+                _logger.warning(
+                    f"{exception_header} must return MetricValue with justifications as a list."
+                )
+                return
+            if any(not (_is_string(just) or just is None) for just in justifications):
+                _logger.warning(
+                    f"{exception_header} must return MetricValue with string justifications."
+                )
+                return
+
+        if aggregates is not None:
+            if not isinstance(aggregates, dict):
+                _logger.warning(
+                    f"{exception_header} must return MetricValue with aggregate_results as a dict."
+                )
+                return
+
+            if any(
+                not (isinstance(k, str) and (_is_numeric(v) or v is None))
+                for k, v in aggregates.items()
+            ):
+                _logger.warning(
+                    f"{exception_header} must return MetricValue with aggregate_results with "
+                    "str keys and numeric values."
+                )
+                return
+
+        return metric

venv/lib/python3.9/site-packages/mlflow/models/evaluation/utils/trace.py

@@ -0,0 +1,177 @@
+import contextlib
+import inspect
+import logging
+from typing import Any, Callable
+
+from mlflow.ml_package_versions import FLAVOR_TO_MODULE_NAME
+from mlflow.utils.autologging_utils import (
+    AUTOLOGGING_INTEGRATIONS,
+    autologging_conf_lock,
+    get_autolog_function,
+    is_autolog_supported,
+)
+from mlflow.utils.autologging_utils.safety import revert_patches
+from mlflow.utils.import_hooks import (
+    _post_import_hooks,
+    get_post_import_hooks,
+    register_post_import_hook,
+)
+
+_logger = logging.getLogger(__name__)
+
+
+# This flag is used to display the message only once when tracing is enabled during the evaluation.
+_SHOWN_TRACE_MESSAGE_BEFORE = False
+
+
+@contextlib.contextmanager
+@autologging_conf_lock
+def configure_autologging_for_evaluation(enable_tracing: bool = True):
+    """
+    Temporarily override the autologging configuration for all flavors during the model evaluation.
+    For example, model auto-logging must be disabled during the evaluation. After the evaluation
+    is done, the original autologging configurations are restored.
+
+    Args:
+        enable_tracing (bool): Whether to enable tracing for the supported flavors during eval.
+    """
+    original_import_hooks = {}
+    new_import_hooks = {}
+
+    # AUTOLOGGING_INTEGRATIONS can change during we iterate over flavors and enable/disable
+    # autologging, therefore, we snapshot the current configuration to restore it later.
+    global_config_snapshot = AUTOLOGGING_INTEGRATIONS.copy()
+
+    for flavor in FLAVOR_TO_MODULE_NAME:
+        if not is_autolog_supported(flavor):
+            continue
+
+        original_config = global_config_snapshot.get(flavor, {}).copy()
+
+        # If autologging is explicitly disabled, do nothing.
+        if original_config.get("disable", False):
+            continue
+
+        # NB: Using post-import hook to configure the autologging lazily when the target
+        # flavor's module is imported, rather than configuring it immediately. This is
+        # because the evaluation code usually only uses a subset of the supported flavors,
+        # hence we want to avoid unnecessary overhead of configuring all flavors.
+        @autologging_conf_lock
+        def _setup_autolog(module):
+            try:
+                autolog = get_autolog_function(flavor)
+
+                # If tracing is supported and not explicitly disabled, enable it.
+                if enable_tracing and _should_enable_tracing(flavor, global_config_snapshot):
+                    new_config = {
+                        k: False if k.startswith("log_") else v for k, v in original_config.items()
+                    }
+                    new_config |= {"log_traces": True, "silent": True}
+                    _kwargs_safe_invoke(autolog, new_config)
+
+                    global _SHOWN_TRACE_MESSAGE_BEFORE
+                    if not _SHOWN_TRACE_MESSAGE_BEFORE:
+                        _logger.info(
+                            "Auto tracing is temporarily enabled during the model evaluation "
+                            "for computing some metrics and debugging. To disable tracing, call "
+                            "`mlflow.autolog(disable=True)`."
+                        )
+                        _SHOWN_TRACE_MESSAGE_BEFORE = True
+                else:
+                    autolog(disable=True)
+
+            except Exception:
+                _logger.debug(f"Failed to update autologging config for {flavor}.", exc_info=True)
+
+        module = FLAVOR_TO_MODULE_NAME[flavor]
+        try:
+            original_import_hooks[module] = get_post_import_hooks(module)
+            new_import_hooks[module] = _setup_autolog
+            register_post_import_hook(_setup_autolog, module, overwrite=True)
+        except Exception:
+            _logger.debug(f"Failed to register post-import hook for {flavor}.", exc_info=True)
+
+    try:
+        yield
+    finally:
+        # Remove post-import hooks and patches the are registered during the evaluation.
+        for module, hooks in new_import_hooks.items():
+            # Restore original post-import hooks if any. Note that we don't use
+            # register_post_import_hook method to bypass some pre-checks and just
+            # restore the original state.
+            if hooks is None:
+                _post_import_hooks.pop(module, None)
+            else:
+                _post_import_hooks[module] = original_import_hooks[module]
+
+        # If any autologging configuration is updated, restore original autologging configurations.
+        for flavor, new_config in AUTOLOGGING_INTEGRATIONS.copy().items():
+            original_config = global_config_snapshot.get(flavor)
+            if original_config != new_config:
+                try:
+                    autolog = get_autolog_function(flavor)
+                    if original_config:
+                        _kwargs_safe_invoke(autolog, original_config)
+                        AUTOLOGGING_INTEGRATIONS[flavor] = original_config
+                    else:
+                        # If the original configuration is empty, autologging was not enabled before
+                        autolog(disable=True)
+                        # Remove all safe_patch applied by autologging
+                        revert_patches(flavor)
+                        # We also need to remove the config entry from AUTOLOGGING_INTEGRATIONS,
+                        # so as not to confuse with the case user explicitly disabled autologging.
+                        AUTOLOGGING_INTEGRATIONS.pop(flavor, None)
+                except ImportError:
+                    pass
+                except Exception as e:
+                    if original_config is None or (
+                        not original_config.get("disable", False)
+                        and not original_config.get("silent", False)
+                    ):
+                        _logger.warning(
+                            f"Exception raised while calling autologging for {flavor}: {e}"
+                        )
+
+
+def _should_enable_tracing(flavor: str, autologging_config: dict[str, Any]) -> bool:
+    """
+    Check if tracing should be enabled for the given flavor during the model evaluation.
+    """
+    # 1. Check if the autologging or tracing is globally disabled
+    # TODO: This check should not take precedence over the flavor-specific configuration
+    # set by the explicit mlflow.<flavor>.autolog() call by users.
+    # However, in Databricks, sometimes mlflow.<flavor>.autolog() is automatically
+    # called in the kernel startup, which is confused with the user's action. In
+    # such cases, even when user disables autologging globally, the flavor-specific
+    # autologging remains enabled. We are going to fix the Databricks side issue,
+    # and after that, we should move this check down after the flavor-specific check.
+    global_config = autologging_config.get("mlflow", {})
+    if global_config.get("disable", False) or (not global_config.get("log_traces", True)):
+        return False
+
+    if not _is_trace_autologging_supported(flavor):
+        return False
+
+    # 3. Check if tracing is explicitly disabled for the flavor
+    flavor_config = autologging_config.get(flavor, {})
+    return flavor_config.get("log_traces", True)
+
+
+def _kwargs_safe_invoke(func: Callable[..., Any], kwargs: dict[str, Any]):
+    """
+    Invoke the function with the given dictionary as keyword arguments, but only include the
+    arguments that are present in the function's signature.
+
+    This is particularly used for calling autolog() function with the configuration dictionary
+    stored in AUTOLOGGING_INTEGRATIONS. While the config keys mostly align with the autolog()'s
+    signature by design, some keys are not present in autolog(), such as "globally_configured".
+    """
+    sig = inspect.signature(func)
+    return func(**{k: v for k, v in kwargs.items() if k in sig.parameters})
+
+
+def _is_trace_autologging_supported(flavor_name: str) -> bool:
+    """Check if the given flavor supports trace autologging."""
+    if autolog_func := get_autolog_function(flavor_name):
+        return "log_traces" in inspect.signature(autolog_func).parameters
+    return False

venv/lib/python3.9/site-packages/mlflow/models/evaluation/validation.py

@@ -0,0 +1,456 @@
+import logging
+import operator
+import os
+from decimal import Decimal
+from typing import Optional
+
+from mlflow.exceptions import MlflowException
+from mlflow.models.evaluation import EvaluationResult
+from mlflow.protos.databricks_pb2 import BAD_REQUEST, INVALID_PARAMETER_VALUE
+from mlflow.utils.annotations import deprecated
+
+_logger = logging.getLogger(__name__)
+
+
+class MetricThreshold:
+    """
+    This class allows you to define metric thresholds for model validation.
+    Allowed thresholds are: threshold, min_absolute_change, min_relative_change.
+
+    Args:
+        threshold: (Optional) A number representing the value threshold for the metric.
+
+            - If higher is better for the metric, the metric value has to be
+              >= threshold to pass validation.
+            - Otherwise, the metric value has to be <= threshold to pass the validation.
+
+        min_absolute_change: (Optional) A positive number representing the minimum absolute
+            change required for candidate model to pass validation with
+            the baseline model.
+
+            - If higher is better for the metric, metric value has to be
+              >= baseline model metric value + min_absolute_change to pass the validation.
+            - Otherwise, metric value has to be <= baseline model metric value - min_absolute_change
+              to pass the validation.
+
+        min_relative_change: (Optional) A floating point number between 0 and 1 representing
+            the minimum relative change (in percentage of
+            baseline model metric value) for candidate model
+            to pass the comparison with the baseline model.
+
+            - If higher is better for the metric, metric value has to be
+              >= baseline model metric value * (1 + min_relative_change)
+            - Otherwise, metric value has to be
+              <= baseline model metric value * (1 - min_relative_change)
+            - Note that if the baseline model metric value is equal to 0, the
+              threshold falls back performing a simple verification that the
+              candidate metric value is better than the baseline metric value,
+              i.e. metric value >= baseline model metric value + 1e-10 if higher
+              is better; metric value <= baseline model metric value - 1e-10 if
+              lower is better.
+
+        greater_is_better: A required boolean representing whether higher value is
+            better for the metric.
+
+        higher_is_better:
+            .. deprecated:: 2.3.0
+                Use ``greater_is_better`` instead.
+
+            A required boolean representing whether higher value is better for the metric.
+    """
+
+    def __init__(
+        self,
+        threshold=None,
+        min_absolute_change=None,
+        min_relative_change=None,
+        greater_is_better=None,
+        higher_is_better=None,
+    ):
+        if threshold is not None and type(threshold) not in {int, float}:
+            raise MetricThresholdClassException("`threshold` parameter must be a number.")
+        if min_absolute_change is not None and (
+            type(min_absolute_change) not in {int, float} or min_absolute_change <= 0
+        ):
+            raise MetricThresholdClassException(
+                "`min_absolute_change` parameter must be a positive number."
+            )
+        if min_relative_change is not None:
+            if not isinstance(min_relative_change, float):
+                raise MetricThresholdClassException(
+                    "`min_relative_change` parameter must be a floating point number."
+                )
+            if min_relative_change < 0 or min_relative_change > 1:
+                raise MetricThresholdClassException(
+                    "`min_relative_change` parameter must be between 0 and 1."
+                )
+        if higher_is_better is None and greater_is_better is None:
+            raise MetricThresholdClassException("`greater_is_better` parameter must be defined.")
+        if higher_is_better is not None and greater_is_better is not None:
+            raise MetricThresholdClassException(
+                "`higher_is_better` parameter must be None when `greater_is_better` is defined."
+            )
+        if greater_is_better is None:
+            greater_is_better = higher_is_better
+        if not isinstance(greater_is_better, bool):
+            raise MetricThresholdClassException("`greater_is_better` parameter must be a boolean.")
+        if threshold is None and min_absolute_change is None and min_relative_change is None:
+            raise MetricThresholdClassException("no threshold was specified.")
+        self._threshold = threshold
+        self._min_absolute_change = min_absolute_change
+        self._min_relative_change = min_relative_change
+        self._greater_is_better = greater_is_better
+
+    @property
+    def threshold(self):
+        """
+        Value of the threshold.
+        """
+        return self._threshold
+
+    @property
+    def min_absolute_change(self):
+        """
+        Value of the minimum absolute change required to pass model comparison with baseline model.
+        """
+        return self._min_absolute_change
+
+    @property
+    def min_relative_change(self):
+        """
+        Float value of the minimum relative change required to pass model comparison with
+        baseline model.
+        """
+        return self._min_relative_change
+
+    @property
+    @deprecated("The attribute `higher_is_better` is deprecated. Use `greater_is_better` instead.")
+    def higher_is_better(self):
+        """
+        Boolean value representing whether higher value is better for the metric.
+        """
+        return self._greater_is_better
+
+    @property
+    def greater_is_better(self):
+        """
+        Boolean value representing whether higher value is better for the metric.
+        """
+        return self._greater_is_better
+
+    def __str__(self):
+        """
+        Returns a human-readable string consisting of all specified thresholds.
+        """
+        threshold_strs = []
+        if self._threshold is not None:
+            threshold_strs.append(f"Threshold: {self._threshold}.")
+        if self._min_absolute_change is not None:
+            threshold_strs.append(f"Minimum Absolute Change: {self._min_absolute_change}.")
+        if self._min_relative_change is not None:
+            threshold_strs.append(f"Minimum Relative Change: {self._min_relative_change}.")
+        if self._greater_is_better is not None:
+            if self._greater_is_better:
+                threshold_strs.append("Higher value is better.")
+            else:
+                threshold_strs.append("Lower value is better.")
+        return " ".join(threshold_strs)
+
+
+class MetricThresholdClassException(MlflowException):
+    def __init__(self, _message, **kwargs):
+        message = "Could not instantiate MetricThreshold class: " + _message
+        super().__init__(message, error_code=INVALID_PARAMETER_VALUE, **kwargs)
+
+
+class _MetricValidationResult:
+    """
+    Internal class for representing validation result per metric.
+    Not user facing, used for organizing metric failures and generating failure message
+    more conveniently.
+
+    Args:
+        metric_name: String representing the metric name
+        candidate_metric_value: value of metric for candidate model
+        metric_threshold: :py:class: `MetricThreshold<mlflow.models.validation.MetricThreshold>`
+            The MetricThreshold for the metric.
+        baseline_metric_value: value of metric for baseline model
+    """
+
+    missing_candidate = False
+    missing_baseline = False
+    threshold_failed = False
+    min_absolute_change_failed = False
+    min_relative_change_failed = False
+
+    def __init__(
+        self,
+        metric_name,
+        candidate_metric_value,
+        metric_threshold,
+        baseline_metric_value=None,
+    ):
+        self.metric_name = metric_name
+        self.candidate_metric_value = candidate_metric_value
+        self.baseline_metric_value = baseline_metric_value
+        self.metric_threshold = metric_threshold
+
+    def __str__(self):
+        """
+        Returns a human-readable string representing the validation result for the metric.
+        """
+        if self.is_success():
+            return f"Metric {self.metric_name} passed the validation."
+
+        if self.missing_candidate:
+            return (
+                f"Metric validation failed: metric {self.metric_name} was missing from the "
+                f"evaluation result of the candidate model."
+            )
+
+        result_strs = []
+        if self.threshold_failed:
+            result_strs.append(
+                f"Metric {self.metric_name} value threshold check failed: "
+                f"candidate model {self.metric_name} = {self.candidate_metric_value}, "
+                f"{self.metric_name} threshold = {self.metric_threshold.threshold}."
+            )
+        if self.missing_baseline:
+            result_strs.append(
+                f"Model comparison failed: metric {self.metric_name} was missing from "
+                f"the evaluation result of the baseline model."
+            )
+        else:
+            if self.min_absolute_change_failed:
+                result_strs.append(
+                    f"Metric {self.metric_name} minimum absolute change check failed: "
+                    f"candidate model {self.metric_name} = {self.candidate_metric_value}, "
+                    f"baseline model {self.metric_name} = {self.baseline_metric_value}, "
+                    f"{self.metric_name} minimum absolute change threshold = "
+                    f"{self.metric_threshold.min_absolute_change}."
+                )
+            if self.min_relative_change_failed:
+                result_strs.append(
+                    f"Metric {self.metric_name} minimum relative change check failed: "
+                    f"candidate model {self.metric_name} = {self.candidate_metric_value}, "
+                    f"baseline model {self.metric_name} = {self.baseline_metric_value}, "
+                    f"{self.metric_name} minimum relative change threshold = "
+                    f"{self.metric_threshold.min_relative_change}."
+                )
+        return " ".join(result_strs)
+
+    def is_success(self):
+        return (
+            not self.missing_candidate
+            and not self.missing_baseline
+            and not self.threshold_failed
+            and not self.min_absolute_change_failed
+            and not self.min_relative_change_failed
+        )
+
+
+class ModelValidationFailedException(MlflowException):
+    def __init__(self, message, **kwargs):
+        super().__init__(message, error_code=BAD_REQUEST, **kwargs)
+
+
+def validate_evaluation_results(
+    validation_thresholds: dict[str, MetricThreshold],
+    candidate_result: EvaluationResult,
+    baseline_result: Optional[EvaluationResult] = None,
+):
+    """
+    Validate the evaluation result from one model (candidate) against another
+    model (baseline). If the candidate results do not meet the validation
+    thresholds, an ModelValidationFailedException will be raised.
+
+    .. note::
+
+        This API is a replacement for the deprecated model validation
+        functionality in the :py:func:`mlflow.evaluate` API.
+
+    Args:
+        validation_thresholds: A dictionary of metric name to
+            :py:class:`mlflow.models.MetricThreshold` used for model validation.
+            Each metric name must either be the name of a builtin metric or the
+            name of a metric defined in the ``extra_metrics`` parameter.
+        candidate_result: The evaluation result of the candidate model.
+            Returned by the :py:func:`mlflow.evaluate` API.
+        baseline_result: The evaluation result of the baseline model.
+            Returned by the :py:func:`mlflow.evaluate` API.
+            If set to None, the candidate model result will be
+            compared against the threshold values directly.
+
+    Code Example:
+
+        .. code-block:: python
+            :caption: Example of Model Validation
+
+            import mlflow
+            from mlflow.models import MetricThreshold
+
+            thresholds = {
+                "accuracy_score": MetricThreshold(
+                    # accuracy should be >=0.8
+                    threshold=0.8,
+                    # accuracy should be at least 5 percent greater than baseline model accuracy
+                    min_absolute_change=0.05,
+                    # accuracy should be at least 0.05 greater than baseline model accuracy
+                    min_relative_change=0.05,
+                    greater_is_better=True,
+                ),
+            }
+
+            # Get evaluation results for the candidate model
+            candidate_result = mlflow.evaluate(
+                model="<YOUR_CANDIDATE_MODEL_URI>",
+                data=eval_dataset,
+                targets="ground_truth",
+                model_type="classifier",
+            )
+
+            # Get evaluation results for the baseline model
+            baseline_result = mlflow.evaluate(
+                model="<YOUR_BASELINE_MODEL_URI>",
+                data=eval_dataset,
+                targets="ground_truth",
+                model_type="classifier",
+            )
+
+            # Validate the results
+            mlflow.validate_evaluation_results(
+                thresholds,
+                candidate_result,
+                baseline_result,
+            )
+
+        See `the Model Validation documentation
+        <../../models/index.html#performing-model-validation>`_ for more details.
+    """
+    try:
+        assert type(validation_thresholds) is dict
+        for key in validation_thresholds.keys():
+            assert type(key) is str
+        for threshold in validation_thresholds.values():
+            assert isinstance(threshold, MetricThreshold)
+    except AssertionError:
+        raise MlflowException(
+            message="The validation thresholds argument must be a dictionary that maps strings "
+            "to MetricThreshold objects.",
+            error_code=INVALID_PARAMETER_VALUE,
+        )
+
+    _logger.info("Validating candidate model metrics against baseline")
+    _validate(
+        validation_thresholds,
+        candidate_result.metrics,
+        baseline_result.metrics if baseline_result else {},
+    )
+    _logger.info("Model validation passed!")
+
+
+def _validate(
+    validation_thresholds: dict[str, MetricThreshold],
+    candidate_metrics: dict[str, float],
+    baseline_metrics: dict[str, float],
+):
+    """
+    Validate the model based on validation_thresholds by metrics value and
+    metrics comparison between candidate model's metrics (candidate_metrics) and
+    baseline model's metrics (baseline_metrics).
+
+    Args:
+        validation_thresholds: A dictionary from metric_name to MetricThreshold.
+        candidate_metrics: The metric evaluation result of the candidate model.
+        baseline_metrics: The metric evaluation result of the baseline model.
+
+    Raises:
+        If the validation does not pass, raise an MlflowException with detail failure message.
+    """
+    validation_results = {
+        metric_name: _MetricValidationResult(
+            metric_name,
+            candidate_metrics.get(metric_name),
+            threshold,
+            baseline_metrics.get(metric_name),
+        )
+        for (metric_name, threshold) in validation_thresholds.items()
+    }
+
+    for metric_name, metric_threshold in validation_thresholds.items():
+        validation_result = validation_results[metric_name]
+
+        if metric_name not in candidate_metrics:
+            validation_result.missing_candidate = True
+            continue
+
+        candidate_metric_value = candidate_metrics[metric_name]
+        baseline_metric_value = baseline_metrics[metric_name] if baseline_metrics else None
+
+        # If metric is higher is better, >= is used, otherwise <= is used
+        # for thresholding metric value and model comparison
+        comparator_fn = operator.__ge__ if metric_threshold.greater_is_better else operator.__le__
+        operator_fn = operator.add if metric_threshold.greater_is_better else operator.sub
+
+        if metric_threshold.threshold is not None:
+            # metric threshold fails
+            # - if not (metric_value >= threshold) for higher is better
+            # - if not (metric_value <= threshold) for lower is better
+            validation_result.threshold_failed = not comparator_fn(
+                candidate_metric_value, metric_threshold.threshold
+            )
+
+        if (
+            metric_threshold.min_relative_change or metric_threshold.min_absolute_change
+        ) and metric_name not in baseline_metrics:
+            validation_result.missing_baseline = True
+            continue
+
+        if metric_threshold.min_absolute_change is not None:
+            # metric comparison absolute change fails
+            # - if not (metric_value >= baseline + min_absolute_change) for higher is better
+            # - if not (metric_value <= baseline - min_absolute_change) for lower is better
+            validation_result.min_absolute_change_failed = not comparator_fn(
+                Decimal(candidate_metric_value),
+                Decimal(operator_fn(baseline_metric_value, metric_threshold.min_absolute_change)),
+            )
+
+        if metric_threshold.min_relative_change is not None:
+            # If baseline metric value equals 0, fallback to simple comparison check
+            if baseline_metric_value == 0:
+                _logger.warning(
+                    f"Cannot perform relative model comparison for metric {metric_name} as "
+                    "baseline metric value is 0. Falling back to simple comparison: verifying "
+                    "that candidate metric value is better than the baseline metric value."
+                )
+                validation_result.min_relative_change_failed = not comparator_fn(
+                    Decimal(candidate_metric_value),
+                    Decimal(operator_fn(baseline_metric_value, 1e-10)),
+                )
+                continue
+            # metric comparison relative change fails
+            # - if (metric_value - baseline) / baseline < min_relative_change for higher is better
+            # - if (baseline - metric_value) / baseline < min_relative_change for lower is better
+            if metric_threshold.greater_is_better:
+                relative_change = (
+                    candidate_metric_value - baseline_metric_value
+                ) / baseline_metric_value
+            else:
+                relative_change = (
+                    baseline_metric_value - candidate_metric_value
+                ) / baseline_metric_value
+            validation_result.min_relative_change_failed = (
+                relative_change < metric_threshold.min_relative_change
+            )
+
+    failure_messages = []
+
+    for metric_validation_result in validation_results.values():
+        if metric_validation_result.is_success():
+            continue
+        failure_messages.append(str(metric_validation_result))
+
+    if not failure_messages:
+        return
+
+    raise ModelValidationFailedException(message=os.linesep.join(failure_messages))