star

venv/lib/python3.9/site-packages/mlflow/metrics/__init__.py

@@ -0,0 +1,489 @@
+from mlflow.metrics import genai
+from mlflow.metrics.base import (
+    MetricValue,
+)
+from mlflow.metrics.metric_definitions import (
+    _accuracy_eval_fn,
+    _ari_eval_fn,
+    _bleu_eval_fn,
+    _f1_score_eval_fn,
+    _flesch_kincaid_eval_fn,
+    _mae_eval_fn,
+    _mape_eval_fn,
+    _max_error_eval_fn,
+    _mse_eval_fn,
+    _ndcg_at_k_eval_fn,
+    _precision_at_k_eval_fn,
+    _precision_eval_fn,
+    _r2_score_eval_fn,
+    _recall_at_k_eval_fn,
+    _recall_eval_fn,
+    _rmse_eval_fn,
+    _rouge1_eval_fn,
+    _rouge2_eval_fn,
+    _rougeL_eval_fn,
+    _rougeLsum_eval_fn,
+    _token_count_eval_fn,
+    _toxicity_eval_fn,
+)
+from mlflow.models import (
+    EvaluationMetric,
+    make_metric,
+)
+from mlflow.utils.annotations import experimental
+
+
+@experimental
+def latency() -> EvaluationMetric:
+    """
+    This function will create a metric for calculating latency. Latency is determined by the time
+    it takes to generate a prediction for a given input. Note that computing latency requires
+    each row to be predicted sequentially, which will likely slow down the evaluation process.
+    """
+    return make_metric(
+        eval_fn=lambda x: MetricValue(),
+        greater_is_better=False,
+        name="latency",
+    )
+
+
+# general text metrics
+@experimental
+def token_count() -> EvaluationMetric:
+    """
+    This function will create a metric for calculating token_count. Token count is calculated
+    using tiktoken by using the `cl100k_base` tokenizer.
+    """
+    return make_metric(
+        eval_fn=_token_count_eval_fn,
+        greater_is_better=True,
+        name="token_count",
+    )
+
+
+@experimental
+def toxicity() -> EvaluationMetric:
+    """
+    This function will create a metric for evaluating `toxicity`_ using the model
+    `roberta-hate-speech-dynabench-r4`_, which defines hate as "abusive speech targeting
+    specific group characteristics, such as ethnic origin, religion, gender, or sexual
+    orientation."
+
+    The score ranges from 0 to 1, where scores closer to 1 are more toxic. The default threshold
+    for a text to be considered "toxic" is 0.5.
+
+    Aggregations calculated for this metric:
+        - ratio (of toxic input texts)
+
+    .. _toxicity: https://huggingface.co/spaces/evaluate-measurement/toxicity
+    .. _roberta-hate-speech-dynabench-r4: https://huggingface.co/facebook/roberta-hate-speech-dynabench-r4-target
+    """
+    return make_metric(
+        eval_fn=_toxicity_eval_fn,
+        greater_is_better=False,
+        name="toxicity",
+        long_name="toxicity/roberta-hate-speech-dynabench-r4",
+        version="v1",
+    )
+
+
+@experimental
+def flesch_kincaid_grade_level() -> EvaluationMetric:
+    """
+    This function will create a metric for calculating `flesch kincaid grade level`_ using
+    `textstat`_.
+
+    This metric outputs a number that approximates the grade level needed to comprehend the text,
+    which will likely range from around 0 to 15 (although it is not limited to this range).
+
+    Aggregations calculated for this metric:
+        - mean
+
+    .. _flesch kincaid grade level:
+        https://en.wikipedia.org/wiki/Flesch%E2%80%93Kincaid_readability_tests#Flesch%E2%80%93Kincaid_grade_level
+    .. _textstat: https://pypi.org/project/textstat/
+    """
+    return make_metric(
+        eval_fn=_flesch_kincaid_eval_fn,
+        greater_is_better=False,
+        name="flesch_kincaid_grade_level",
+        version="v1",
+    )
+
+
+@experimental
+def ari_grade_level() -> EvaluationMetric:
+    """
+    This function will create a metric for calculating `automated readability index`_ using
+    `textstat`_.
+
+    This metric outputs a number that approximates the grade level needed to comprehend the text,
+    which will likely range from around 0 to 15 (although it is not limited to this range).
+
+    Aggregations calculated for this metric:
+        - mean
+
+    .. _automated readability index: https://en.wikipedia.org/wiki/Automated_readability_index
+    .. _textstat: https://pypi.org/project/textstat/
+    """
+    return make_metric(
+        eval_fn=_ari_eval_fn,
+        greater_is_better=False,
+        name="ari_grade_level",
+        long_name="automated_readability_index_grade_level",
+        version="v1",
+    )
+
+
+# question answering metrics
+@experimental
+def exact_match() -> EvaluationMetric:
+    """
+    This function will create a metric for calculating `accuracy`_ using sklearn.
+
+    This metric only computes an aggregate score which ranges from 0 to 1.
+
+    .. _accuracy: https://scikit-learn.org/stable/modules/generated/sklearn.metrics.accuracy_score.html
+    """
+    return make_metric(
+        eval_fn=_accuracy_eval_fn, greater_is_better=True, name="exact_match", version="v1"
+    )
+
+
+# text summarization metrics
+@experimental
+def rouge1() -> EvaluationMetric:
+    """
+    This function will create a metric for evaluating `rouge1`_.
+
+    The score ranges from 0 to 1, where a higher score indicates higher similarity.
+    `rouge1`_ uses unigram based scoring to calculate similarity.
+
+    Aggregations calculated for this metric:
+        - mean
+
+    .. _rouge1: https://huggingface.co/spaces/evaluate-metric/rouge
+    """
+    return make_metric(
+        eval_fn=_rouge1_eval_fn,
+        greater_is_better=True,
+        name="rouge1",
+        version="v1",
+    )
+
+
+@experimental
+def rouge2() -> EvaluationMetric:
+    """
+    This function will create a metric for evaluating `rouge2`_.
+
+    The score ranges from 0 to 1, where a higher score indicates higher similarity.
+    `rouge2`_ uses bigram based scoring to calculate similarity.
+
+    Aggregations calculated for this metric:
+        - mean
+
+    .. _rouge2: https://huggingface.co/spaces/evaluate-metric/rouge
+    """
+    return make_metric(
+        eval_fn=_rouge2_eval_fn,
+        greater_is_better=True,
+        name="rouge2",
+        version="v1",
+    )
+
+
+@experimental
+def rougeL() -> EvaluationMetric:
+    """
+    This function will create a metric for evaluating `rougeL`_.
+
+    The score ranges from 0 to 1, where a higher score indicates higher similarity.
+    `rougeL`_ uses unigram based scoring to calculate similarity.
+
+    Aggregations calculated for this metric:
+        - mean
+
+    .. _rougeL: https://huggingface.co/spaces/evaluate-metric/rouge
+    """
+    return make_metric(
+        eval_fn=_rougeL_eval_fn,
+        greater_is_better=True,
+        name="rougeL",
+        version="v1",
+    )
+
+
+@experimental
+def rougeLsum() -> EvaluationMetric:
+    """
+    This function will create a metric for evaluating `rougeLsum`_.
+
+    The score ranges from 0 to 1, where a higher score indicates higher similarity.
+    `rougeLsum`_ uses longest common subsequence based scoring to calculate similarity.
+
+    Aggregations calculated for this metric:
+        - mean
+
+    .. _rougeLsum: https://huggingface.co/spaces/evaluate-metric/rouge
+    """
+    return make_metric(
+        eval_fn=_rougeLsum_eval_fn,
+        greater_is_better=True,
+        name="rougeLsum",
+        version="v1",
+    )
+
+
+@experimental
+def precision_at_k(k) -> EvaluationMetric:
+    """
+    This function will create a metric for calculating ``precision_at_k`` for retriever models.
+
+    This metric computes a score between 0 and 1 for each row representing the precision of the
+    retriever model at the given ``k`` value. If no relevant documents are retrieved, the score is
+    0, indicating that no relevant docs are retrieved. Let ``x = min(k, # of retrieved doc IDs)``.
+    Then, in all other cases, the precision at k is calculated as follows:
+
+        ``precision_at_k`` = (# of relevant retrieved doc IDs in top-``x`` ranked docs) / ``x``.
+    """
+    return make_metric(
+        eval_fn=_precision_at_k_eval_fn(k),
+        greater_is_better=True,
+        name=f"precision_at_{k}",
+    )
+
+
+@experimental
+def recall_at_k(k) -> EvaluationMetric:
+    """
+    This function will create a metric for calculating ``recall_at_k`` for retriever models.
+
+    This metric computes a score between 0 and 1 for each row representing the recall ability of
+    the retriever model at the given ``k`` value. If no ground truth doc IDs are provided and no
+    documents are retrieved, the score is 1. However, if no ground truth doc IDs are provided and
+    documents are retrieved, the score is 0. In all other cases, the recall at k is calculated as
+    follows:
+
+        ``recall_at_k`` = (# of unique relevant retrieved doc IDs in top-``k`` ranked docs) / (# of
+        ground truth doc IDs)
+    """
+    return make_metric(
+        eval_fn=_recall_at_k_eval_fn(k),
+        greater_is_better=True,
+        name=f"recall_at_{k}",
+    )
+
+
+@experimental
+def ndcg_at_k(k) -> EvaluationMetric:
+    """
+    This function will create a metric for evaluating `NDCG@k`_ for retriever models.
+
+    NDCG score is capable of handling non-binary notions of relevance. However, for simplicity,
+    we use binary relevance here. The relevance score for documents in the ground truth is 1,
+    and the relevance score for documents not in the ground truth is 0.
+
+    The NDCG score is calculated using sklearn.metrics.ndcg_score with the following edge cases
+    on top of the sklearn implementation:
+
+    1. If no ground truth doc IDs are provided and no documents are retrieved, the score is 1.
+    2. If no ground truth doc IDs are provided and documents are retrieved, the score is 0.
+    3. If ground truth doc IDs are provided and no documents are retrieved, the score is 0.
+    4. If duplicate doc IDs are retrieved and the duplicate doc IDs are in the ground truth,
+       they will be treated as different docs. For example, if the ground truth doc IDs are
+       [1, 2] and the retrieved doc IDs are [1, 1, 1, 3], the score will be equivalent to
+       ground truth doc IDs [10, 11, 12, 2] and retrieved doc IDs [10, 11, 12, 3].
+
+    .. _NDCG@k: https://scikit-learn.org/stable/modules/generated/sklearn.metrics.ndcg_score.html
+    """
+    return make_metric(
+        eval_fn=_ndcg_at_k_eval_fn(k),
+        greater_is_better=True,
+        name=f"ndcg_at_{k}",
+    )
+
+
+# General Regression Metrics
+def mae() -> EvaluationMetric:
+    """
+    This function will create a metric for evaluating `mae`_.
+
+    This metric computes an aggregate score for the mean absolute error for regression.
+
+    .. _mae: https://scikit-learn.org/stable/modules/generated/sklearn.metrics.mean_absolute_error.html
+    """
+    return make_metric(
+        eval_fn=_mae_eval_fn,
+        greater_is_better=False,
+        name="mean_absolute_error",
+    )
+
+
+def mse() -> EvaluationMetric:
+    """
+    This function will create a metric for evaluating `mse`_.
+
+    This metric computes an aggregate score for the mean squared error for regression.
+
+    .. _mse: https://scikit-learn.org/stable/modules/generated/sklearn.metrics.mean_squared_error.html
+    """
+    return make_metric(
+        eval_fn=_mse_eval_fn,
+        greater_is_better=False,
+        name="mean_squared_error",
+    )
+
+
+def rmse() -> EvaluationMetric:
+    """
+    This function will create a metric for evaluating the square root of `mse`_.
+
+    This metric computes an aggregate score for the root mean absolute error for regression.
+
+    .. _mse: https://scikit-learn.org/stable/modules/generated/sklearn.metrics.mean_squared_error.html
+    """
+
+    return make_metric(
+        eval_fn=_rmse_eval_fn,
+        greater_is_better=False,
+        name="root_mean_squared_error",
+    )
+
+
+def r2_score() -> EvaluationMetric:
+    """
+    This function will create a metric for evaluating `r2_score`_.
+
+    This metric computes an aggregate score for the coefficient of determination. R2 ranges from
+    negative infinity to 1, and measures the percentage of variance explained by the predictor
+    variables in a regression.
+
+    .. _r2_score: https://scikit-learn.org/stable/modules/generated/sklearn.metrics.r2_score.html
+    """
+    return make_metric(
+        eval_fn=_r2_score_eval_fn,
+        greater_is_better=True,
+        name="r2_score",
+    )
+
+
+def max_error() -> EvaluationMetric:
+    """
+    This function will create a metric for evaluating `max_error`_.
+
+    This metric computes an aggregate score for the maximum residual error for regression.
+
+    .. _max_error: https://scikit-learn.org/stable/modules/generated/sklearn.metrics.max_error.html
+    """
+    return make_metric(
+        eval_fn=_max_error_eval_fn,
+        greater_is_better=False,
+        name="max_error",
+    )
+
+
+def mape() -> EvaluationMetric:
+    """
+    This function will create a metric for evaluating `mape`_.
+
+    This metric computes an aggregate score for the mean absolute percentage error for regression.
+
+    .. _mape: https://scikit-learn.org/stable/modules/generated/sklearn.metrics.mean_absolute_percentage_error.html
+    """
+    return make_metric(
+        eval_fn=_mape_eval_fn,
+        greater_is_better=False,
+        name="mean_absolute_percentage_error",
+    )
+
+
+# Binary Classification Metrics
+
+
+def recall_score() -> EvaluationMetric:
+    """
+    This function will create a metric for evaluating `recall`_ for classification.
+
+    This metric computes an aggregate score between 0 and 1 for the recall of a classification task.
+
+    .. _recall: https://scikit-learn.org/stable/modules/generated/sklearn.metrics.recall_score.html
+    """
+    return make_metric(eval_fn=_recall_eval_fn, greater_is_better=True, name="recall_score")
+
+
+def precision_score() -> EvaluationMetric:
+    """
+    This function will create a metric for evaluating `precision`_ for classification.
+
+    This metric computes an aggregate score between 0 and 1 for the precision of
+    classification task.
+
+    .. _precision: https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_score.html
+    """
+    return make_metric(eval_fn=_precision_eval_fn, greater_is_better=True, name="precision_score")
+
+
+def f1_score() -> EvaluationMetric:
+    """
+    This function will create a metric for evaluating `f1_score`_ for binary classification.
+
+    This metric computes an aggregate score between 0 and 1 for the F1 score (F-measure) of a
+    classification task. F1 score is defined as 2 * (precision * recall) / (precision + recall).
+
+    .. _f1_score: https://scikit-learn.org/stable/modules/generated/sklearn.metrics.f1_score.html
+    """
+    return make_metric(eval_fn=_f1_score_eval_fn, greater_is_better=True, name="f1_score")
+
+
+@experimental
+def bleu() -> EvaluationMetric:
+    """
+    This function will create a metric for evaluating `bleu`_.
+
+    The BLEU scores range from 0 to 1, with higher scores indicating greater similarity to
+    reference texts. BLEU considers n-gram precision and brevity penalty. While adding more
+    references can boost the score, perfect scores are rare and not essential for effective
+    evaluation.
+
+    Aggregations calculated for this metric:
+        - mean
+        - variance
+        - p90
+
+    .. _bleu: https://huggingface.co/spaces/evaluate-metric/bleu
+    """
+    return make_metric(
+        eval_fn=_bleu_eval_fn,
+        greater_is_better=True,
+        name="bleu",
+        version="v1",
+    )
+
+
+__all__ = [
+    "EvaluationMetric",
+    "MetricValue",
+    "make_metric",
+    "flesch_kincaid_grade_level",
+    "ari_grade_level",
+    "exact_match",
+    "rouge1",
+    "rouge2",
+    "rougeL",
+    "rougeLsum",
+    "toxicity",
+    "mae",
+    "mse",
+    "rmse",
+    "r2_score",
+    "max_error",
+    "mape",
+    "recall_score",
+    "precision_score",
+    "f1_score",
+    "token_count",
+    "latency",
+    "genai",
+    "bleu",
+]

venv/lib/python3.9/site-packages/mlflow/metrics/base.py

@@ -0,0 +1,41 @@
+from dataclasses import dataclass
+from typing import Optional, Union
+
+import numpy as np
+
+from mlflow.utils.annotations import experimental
+from mlflow.utils.validation import _is_numeric
+
+
+def standard_aggregations(scores):
+    return {
+        "mean": np.mean(scores),
+        "variance": np.var(scores),
+        "p90": np.percentile(scores, 90),
+    }
+
+
+@experimental
+@dataclass
+class MetricValue:
+    """
+    The value of a metric.
+
+
+    Args:
+        scores: The value of the metric per row
+        justifications: The justification (if applicable) for the respective score
+        aggregate_results: A dictionary mapping the name of the aggregation to its value
+    """
+
+    scores: Optional[Union[list[str], list[float]]] = None
+    justifications: Optional[list[str]] = None
+    aggregate_results: Optional[dict[str, float]] = None
+
+    def __post_init__(self):
+        if (
+            self.aggregate_results is None
+            and isinstance(self.scores, (list, tuple))
+            and all(_is_numeric(score) for score in self.scores)
+        ):
+            self.aggregate_results = standard_aggregations(self.scores)

venv/lib/python3.9/site-packages/mlflow/metrics/genai/__init__.py

@@ -0,0 +1,25 @@
+from mlflow.metrics.genai.base import EvaluationExample
+from mlflow.metrics.genai.genai_metric import (
+    make_genai_metric,
+    make_genai_metric_from_prompt,
+    retrieve_custom_metrics,
+)
+from mlflow.metrics.genai.metric_definitions import (
+    answer_correctness,
+    answer_relevance,
+    answer_similarity,
+    faithfulness,
+    relevance,
+)
+
+__all__ = [
+    "EvaluationExample",
+    "make_genai_metric",
+    "make_genai_metric_from_prompt",
+    "answer_similarity",
+    "answer_correctness",
+    "faithfulness",
+    "answer_relevance",
+    "relevance",
+    "retrieve_custom_metrics",
+]

venv/lib/python3.9/site-packages/mlflow/metrics/genai/base.py

@@ -0,0 +1,103 @@
+from dataclasses import dataclass
+from typing import Optional, Union
+
+from mlflow.metrics.genai.prompt_template import PromptTemplate
+from mlflow.utils.annotations import experimental
+
+
+@experimental
+@dataclass
+class EvaluationExample:
+    """
+    Stores the sample example during few shot learning during LLM evaluation
+
+    Args:
+        input: The input provided to the model
+        output: The output generated by the model
+        score: The score given by the evaluator
+        justification: The justification given by the evaluator
+        grading_context: The grading_context provided to the evaluator for evaluation. Either
+            a dictionary of grading context column names and grading context strings
+            or a single grading context string.
+
+    .. code-block:: python
+        :caption: Example for creating an EvaluationExample
+
+        from mlflow.metrics.base import EvaluationExample
+
+        example = EvaluationExample(
+            input="What is MLflow?",
+            output="MLflow is an open-source platform for managing machine "
+            "learning workflows, including experiment tracking, model packaging, "
+            "versioning, and deployment, simplifying the ML lifecycle.",
+            score=4,
+            justification="The definition effectively explains what MLflow is "
+            "its purpose, and its developer. It could be more concise for a 5-score.",
+            grading_context={
+                "ground_truth": "MLflow is an open-source platform for managing "
+                "the end-to-end machine learning (ML) lifecycle. It was developed by Databricks, "
+                "a company that specializes in big data and machine learning solutions. MLflow is "
+                "designed to address the challenges that data scientists and machine learning "
+                "engineers face when developing, training, and deploying machine learning models."
+            },
+        )
+        print(str(example))
+
+    .. code-block:: text
+        :caption: Output
+
+        Input: What is MLflow?
+        Provided output: "MLflow is an open-source platform for managing machine "
+            "learning workflows, including experiment tracking, model packaging, "
+            "versioning, and deployment, simplifying the ML lifecycle."
+        Provided ground_truth: "MLflow is an open-source platform for managing "
+            "the end-to-end machine learning (ML) lifecycle. It was developed by Databricks, "
+            "a company that specializes in big data and machine learning solutions. MLflow is "
+            "designed to address the challenges that data scientists and machine learning "
+            "engineers face when developing, training, and deploying machine learning models."
+        Score: 4
+        Justification: "The definition effectively explains what MLflow is "
+            "its purpose, and its developer. It could be more concise for a 5-score."
+    """
+
+    output: str
+    score: float
+    justification: str
+    input: Optional[str] = None
+    grading_context: Optional[Union[dict[str, str], str]] = None
+
+    def _format_grading_context(self):
+        if isinstance(self.grading_context, dict):
+            return "\n".join(
+                [f"key: {key}\nvalue:\n{value}" for key, value in self.grading_context.items()]
+            )
+        else:
+            return self.grading_context
+
+    def __str__(self) -> str:
+        return PromptTemplate(
+            [
+                """
+Example Input:
+{input}
+""",
+                """
+Example Output:
+{output}
+""",
+                """
+Additional information used by the model:
+{grading_context}
+""",
+                """
+Example score: {score}
+Example justification: {justification}
+        """,
+            ]
+        ).format(
+            input=self.input,
+            output=self.output,
+            grading_context=self._format_grading_context(),
+            score=self.score,
+            justification=self.justification,
+        )

venv/lib/python3.9/site-packages/mlflow/metrics/genai/genai_metric.py

@@ -0,0 +1,768 @@
+import json
+import logging
+import re
+import warnings
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from inspect import Parameter, Signature
+from tempfile import TemporaryDirectory
+from typing import Any, Optional, Union
+
+import pandas as pd
+
+import mlflow
+from mlflow.exceptions import MlflowException
+from mlflow.metrics.base import MetricValue
+from mlflow.metrics.genai import model_utils
+from mlflow.metrics.genai.base import EvaluationExample
+from mlflow.metrics.genai.prompt_template import PromptTemplate
+from mlflow.metrics.genai.utils import _get_default_model, _get_latest_metric_version
+from mlflow.models import EvaluationMetric, make_metric
+from mlflow.models.evaluation.base import _make_metric
+from mlflow.protos.databricks_pb2 import (
+    BAD_REQUEST,
+    INTERNAL_ERROR,
+    INVALID_PARAMETER_VALUE,
+    UNAUTHENTICATED,
+    ErrorCode,
+)
+from mlflow.utils.annotations import experimental
+from mlflow.utils.class_utils import _get_class_from_string
+from mlflow.version import VERSION
+
+_logger = logging.getLogger(__name__)
+
+_GENAI_CUSTOM_METRICS_FILE_NAME = "genai_custom_metrics.json"
+_PROMPT_FORMATTING_WRAPPER = """
+
+You must return the following fields in your response in two lines, one below the other:
+score: Your numerical score based on the rubric
+justification: Your reasoning for giving this score
+
+Do not add additional new lines. Do not add any other fields."""
+
+
+def _format_args_string(grading_context_columns: Optional[list[str]], eval_values, indx) -> str:
+    import pandas as pd
+
+    args_dict = {}
+    for arg in grading_context_columns:
+        if arg in eval_values:
+            args_dict[arg] = (
+                eval_values[arg].iloc[indx]
+                if isinstance(eval_values[arg], pd.Series)
+                else eval_values[arg][indx]
+            )
+        else:
+            raise MlflowException(
+                f"{arg} does not exist in the eval function {list(eval_values.keys())}."
+            )
+
+    return (
+        ""
+        if args_dict is None or len(args_dict) == 0
+        else (
+            "Additional information used by the model:\n"
+            + "\n".join(
+                [f"key: {arg}\nvalue:\n{arg_value}" for arg, arg_value in args_dict.items()]
+            )
+        )
+    )
+
+
+# Function to extract Score and Justification
+def _extract_score_and_justification(text):
+    if text:
+        text = re.sub(r"score", "score", text, flags=re.IGNORECASE)
+        text = re.sub(r"justification", "justification", text, flags=re.IGNORECASE)
+        # Attempt to parse JSON
+        try:
+            data = json.loads(text)
+            score = int(data.get("score"))
+            justification = data.get("justification")
+        except json.JSONDecodeError:
+            # If parsing fails, use regex
+            if (match := re.search(r"score: (\d+),?\s*justification: (.+)", text)) or (
+                match := re.search(r"\s*score:\s*(\d+)\s*justification:\s*(.+)", text, re.DOTALL)
+            ):
+                score = int(match.group(1))
+                justification = match.group(2)
+            else:
+                score = None
+                justification = f"Failed to extract score and justification. Raw output: {text}"
+
+        if not isinstance(score, (int, float)) or not isinstance(justification, str):
+            return None, f"Failed to extract score and justification. Raw output: {text}"
+
+        return score, justification
+
+    return None, None
+
+
+def _score_model_on_one_payload(
+    payload: str,
+    eval_model: str,
+    parameters: Optional[dict[str, Any]],
+    extra_headers: Optional[dict[str, str]] = None,
+    proxy_url: Optional[str] = None,
+):
+    try:
+        # If the endpoint does not specify type, default to chat format
+        endpoint_type = model_utils.get_endpoint_type(eval_model) or "llm/v1/chat"
+        raw_result = model_utils.score_model_on_payload(
+            eval_model, payload, parameters, extra_headers, proxy_url, endpoint_type
+        )
+        return _extract_score_and_justification(raw_result)
+    except ImportError:
+        raise
+    except MlflowException as e:
+        if e.error_code in [
+            ErrorCode.Name(BAD_REQUEST),
+            ErrorCode.Name(UNAUTHENTICATED),
+            ErrorCode.Name(INVALID_PARAMETER_VALUE),
+        ]:
+            raise
+        else:
+            return None, f"Failed to score model on payload. Error: {e!s}"
+    except Exception as e:
+        return None, f"Failed to score model on payload. Error: {e!s}"
+
+
+def _score_model_on_payloads(
+    grading_payloads, model, parameters, headers, proxy_url, max_workers
+) -> tuple[list[int], list[str]]:
+    scores = [None] * len(grading_payloads)
+    justifications = [None] * len(grading_payloads)
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        futures = {
+            executor.submit(
+                _score_model_on_one_payload,
+                payload,
+                model,
+                parameters,
+                headers,
+                proxy_url,
+            ): indx
+            for indx, payload in enumerate(grading_payloads)
+        }
+
+        as_comp = as_completed(futures)
+        try:
+            from tqdm.auto import tqdm
+
+            as_comp = tqdm(as_comp, total=len(futures))
+        except ImportError:
+            pass
+
+        for future in as_comp:
+            indx = futures[future]
+            score, justification = future.result()
+            scores[indx] = score
+            justifications[indx] = justification
+
+    return scores, justifications
+
+
+def _get_aggregate_results(scores, aggregations):
+    # loop over the aggregations and compute the aggregate results on the scores
+    def aggregate_function(aggregate_option, scores):
+        import numpy as np
+
+        options = {
+            "min": np.min,
+            "max": np.max,
+            "mean": np.mean,
+            "median": np.median,
+            "variance": np.var,
+            "p90": lambda x: np.percentile(x, 90) if x else None,
+        }
+
+        if aggregate_option not in options:
+            raise MlflowException(
+                message=f"Invalid aggregate option {aggregate_option}.",
+                error_code=INVALID_PARAMETER_VALUE,
+            )
+
+        return options[aggregate_option](scores)
+
+    scores_for_aggregation = [score for score in scores if score is not None]
+
+    return (
+        {option: aggregate_function(option, scores_for_aggregation) for option in aggregations}
+        if aggregations is not None
+        else {}
+    )
+
+
+@experimental
+def make_genai_metric_from_prompt(
+    name: str,
+    judge_prompt: Optional[str] = None,
+    model: Optional[str] = _get_default_model(),
+    parameters: Optional[dict[str, Any]] = None,
+    aggregations: Optional[list[str]] = None,
+    greater_is_better: bool = True,
+    max_workers: int = 10,
+    metric_metadata: Optional[dict[str, Any]] = None,
+    extra_headers: Optional[dict[str, str]] = None,
+    proxy_url: Optional[str] = None,
+) -> EvaluationMetric:
+    """
+    Create a genai metric used to evaluate LLM using LLM as a judge in MLflow. This produces
+    a metric using only the supplied judge prompt without any pre-written system prompt.
+    This can be useful for use cases that are not covered by the full grading prompt in any
+    ``EvaluationModel`` version.
+
+    Args:
+        name: Name of the metric.
+        judge_prompt: The entire prompt to be used for the judge model.
+            The prompt will be minimally wrapped in formatting instructions to ensure
+            scores can be parsed. The prompt may use f-string formatting to include variables.
+            Corresponding variables must be passed as keyword arguments into the
+            resulting metric's eval function.
+        model: (Optional) Model uri of the judge model that will be used to compute the metric,
+            e.g., ``openai:/gpt-4``. Refer to the `LLM-as-a-Judge Metrics <https://mlflow.org/docs/latest/llms/llm-evaluate/index.html#selecting-the-llm-as-judge-model>`_
+            documentation for the supported model types and their URI format.
+        parameters: (Optional) Parameters for the LLM used to compute the metric. By default, we
+            set the temperature to 0.0, max_tokens to 200, and top_p to 1.0. We recommend
+            setting the temperature to 0.0 for the LLM used as a judge to ensure consistent results.
+        aggregations: (Optional) The list of options to aggregate the scores. Currently supported
+            options are: min, max, mean, median, variance, p90.
+        greater_is_better: (Optional) Whether the metric is better when it is greater.
+        max_workers: (Optional) The maximum number of workers to use for judge scoring.
+            Defaults to 10 workers.
+        metric_metadata: (Optional) Dictionary of metadata to be attached to the
+            EvaluationMetric object. Useful for model evaluators that require additional
+            information to determine how to evaluate this metric.
+        extra_headers: (Optional) Additional headers to be passed to the judge model.
+        proxy_url: (Optional) Proxy URL to be used for the judge model. This is useful when the
+            judge model is served via a proxy endpoint, not directly via LLM provider services.
+            If not specified, the default URL for the LLM provider will be used
+            (e.g., https://api.openai.com/v1/chat/completions for OpenAI chat models).
+
+    Returns:
+        A metric object.
+
+    .. code-block:: python
+        :test:
+        :caption: Example for creating a genai metric
+
+        from mlflow.metrics.genai import make_genai_metric_from_prompt
+
+        metric = make_genai_metric_from_prompt(
+            name="ease_of_understanding",
+            judge_prompt=(
+                "You must evaluate the output of a bot based on how easy it is to "
+                "understand its outputs."
+                "Evaluate the bot's output from the perspective of a layperson."
+                "The bot was provided with this input: {input} and this output: {output}."
+            ),
+            model="openai:/gpt-4",
+            parameters={"temperature": 0.0},
+            aggregations=["mean", "variance", "p90"],
+            greater_is_better=True,
+        )
+
+    """
+    import numpy as np
+
+    prompt_template = PromptTemplate([judge_prompt, _PROMPT_FORMATTING_WRAPPER])
+    allowed_variables = prompt_template.variables
+
+    # When users create a custom metric using this function,the metric configuration
+    # will be serialized and stored as an artifact. This enables us to later deserialize
+    # the configuration, allowing users to understand their LLM evaluation results more clearly.
+    genai_metric_args = {
+        "name": name,
+        "judge_prompt": judge_prompt,
+        "model": model,
+        "parameters": parameters,
+        "aggregations": aggregations,
+        "greater_is_better": greater_is_better,
+        "max_workers": max_workers,
+        "metric_metadata": metric_metadata,
+        # Record the mlflow version for serialization in case the function signature changes later
+        "mlflow_version": VERSION,
+        "fn_name": make_genai_metric_from_prompt.__name__,
+    }
+
+    aggregations = aggregations or ["mean", "variance", "p90"]
+
+    def eval_fn(
+        *args,
+        **kwargs,
+    ) -> MetricValue:
+        """
+        This is the function that is called when the metric is evaluated.
+        """
+        if missing_variables := allowed_variables - set(kwargs.keys()):
+            raise MlflowException(
+                message=f"Missing variable inputs to eval_fn: {missing_variables}",
+                error_code=INVALID_PARAMETER_VALUE,
+            )
+        kwargs = {k: [v] if np.isscalar(v) else v for k, v in kwargs.items()}
+        grading_payloads = pd.DataFrame(kwargs).to_dict(orient="records")
+        arg_strings = [prompt_template.format(**payload) for payload in grading_payloads]
+        scores, justifications = _score_model_on_payloads(
+            arg_strings, model, parameters, extra_headers, proxy_url, max_workers
+        )
+
+        aggregate_scores = _get_aggregate_results(scores, aggregations)
+
+        return MetricValue(scores, justifications, aggregate_scores)
+
+    if allowed_variables:
+        eval_fn.__signature__ = Signature(
+            parameters=[
+                Parameter(name=var, kind=Parameter.KEYWORD_ONLY) for var in allowed_variables
+            ]
+        )
+
+    return make_metric(
+        eval_fn=eval_fn,
+        greater_is_better=greater_is_better,
+        name=name,
+        metric_metadata=metric_metadata,
+        genai_metric_args=genai_metric_args,
+    )
+
+
+@experimental
+def make_genai_metric(
+    name: str,
+    definition: str,
+    grading_prompt: str,
+    examples: Optional[list[EvaluationExample]] = None,
+    version: Optional[str] = _get_latest_metric_version(),
+    model: Optional[str] = _get_default_model(),
+    grading_context_columns: Optional[Union[str, list[str]]] = None,
+    include_input: bool = True,
+    parameters: Optional[dict[str, Any]] = None,
+    aggregations: Optional[list[str]] = None,
+    greater_is_better: bool = True,
+    max_workers: int = 10,
+    metric_metadata: Optional[dict[str, Any]] = None,
+    extra_headers: Optional[dict[str, str]] = None,
+    proxy_url: Optional[str] = None,
+) -> EvaluationMetric:
+    """
+    Create a genai metric used to evaluate LLM using LLM as a judge in MLflow. The full grading
+    prompt is stored in the metric_details field of the ``EvaluationMetric`` object.
+
+    Args:
+        name: Name of the metric.
+        definition: Definition of the metric.
+        grading_prompt: Grading criteria of the metric.
+        examples: (Optional) Examples of the metric.
+        version: (Optional) Version of the metric. Currently supported versions are: v1.
+        model: (Optional) Model uri of the judge model that will be used to compute the metric,
+            e.g., ``openai:/gpt-4``. Refer to the `LLM-as-a-Judge Metrics <https://mlflow.org/docs/latest/llms/llm-evaluate/index.html#selecting-the-llm-as-judge-model>`_
+            documentation for the supported model types and their URI format.
+        grading_context_columns: (Optional) The name of the grading context column, or a list of
+            grading context column names, required to compute the metric. The
+            ``grading_context_columns`` are used by the LLM as a judge as additional information to
+            compute the metric. The columns are extracted from the input dataset or output
+            predictions based on ``col_mapping`` in the ``evaluator_config`` passed to
+            :py:func:`mlflow.evaluate()`. They can also be the name of other evaluated metrics.
+        include_input: (Optional) Whether to include the input
+            when computing the metric.
+        parameters: (Optional) Parameters for the LLM used to compute the metric. By default, we
+            set the temperature to 0.0, max_tokens to 200, and top_p to 1.0. We recommend
+            setting the temperature to 0.0 for the LLM used as a judge to ensure consistent results.
+        aggregations: (Optional) The list of options to aggregate the scores. Currently supported
+            options are: min, max, mean, median, variance, p90.
+        greater_is_better: (Optional) Whether the metric is better when it is greater.
+        max_workers: (Optional) The maximum number of workers to use for judge scoring.
+            Defaults to 10 workers.
+        metric_metadata: (Optional) Dictionary of metadata to be attached to the
+            EvaluationMetric object. Useful for model evaluators that require additional
+            information to determine how to evaluate this metric.
+        extra_headers: (Optional) Additional headers to be passed to the judge model.
+        proxy_url: (Optional) Proxy URL to be used for the judge model. This is useful when the
+            judge model is served via a proxy endpoint, not directly via LLM provider services.
+            If not specified, the default URL for the LLM provider will be used
+            (e.g., https://api.openai.com/v1/chat/completions for OpenAI chat models).
+
+
+    Returns:
+        A metric object.
+
+    .. code-block:: python
+        :test:
+        :caption: Example for creating a genai metric
+
+        from mlflow.metrics.genai import EvaluationExample, make_genai_metric
+
+        example = EvaluationExample(
+            input="What is MLflow?",
+            output=(
+                "MLflow is an open-source platform for managing machine "
+                "learning workflows, including experiment tracking, model packaging, "
+                "versioning, and deployment, simplifying the ML lifecycle."
+            ),
+            score=4,
+            justification=(
+                "The definition effectively explains what MLflow is "
+                "its purpose, and its developer. It could be more concise for a 5-score.",
+            ),
+            grading_context={
+                "targets": (
+                    "MLflow is an open-source platform for managing "
+                    "the end-to-end machine learning (ML) lifecycle. It was developed by "
+                    "Databricks, a company that specializes in big data and machine learning "
+                    "solutions. MLflow is designed to address the challenges that data "
+                    "scientists and machine learning engineers face when developing, training, "
+                    "and deploying machine learning models."
+                )
+            },
+        )
+        metric = make_genai_metric(
+            name="answer_correctness",
+            definition=(
+                "Answer correctness is evaluated on the accuracy of the provided output based on "
+                "the provided targets, which is the ground truth. Scores can be assigned based on "
+                "the degree of semantic similarity and factual correctness of the provided output "
+                "to the provided targets, where a higher score indicates higher degree of accuracy."
+            ),
+            grading_prompt=(
+                "Answer correctness: Below are the details for different scores:"
+                "- Score 1: The output is completely incorrect. It is completely different from "
+                "or contradicts the provided targets."
+                "- Score 2: The output demonstrates some degree of semantic similarity and "
+                "includes partially correct information. However, the output still has significant "
+                "discrepancies with the provided targets or inaccuracies."
+                "- Score 3: The output addresses a couple of aspects of the input accurately, "
+                "aligning with the provided targets. However, there are still omissions or minor "
+                "inaccuracies."
+                "- Score 4: The output is mostly correct. It provides mostly accurate information, "
+                "but there may be one or more minor omissions or inaccuracies."
+                "- Score 5: The output is correct. It demonstrates a high degree of accuracy and "
+                "semantic similarity to the targets."
+            ),
+            examples=[example],
+            version="v1",
+            model="openai:/gpt-4",
+            grading_context_columns=["targets"],
+            parameters={"temperature": 0.0},
+            aggregations=["mean", "variance", "p90"],
+            greater_is_better=True,
+        )
+    """
+    # When users create a custom metric using this function,the metric configuration
+    # will be serialized and stored as an artifact. This enables us to later deserialize
+    # the configuration, allowing users to understand their LLM evaluation results more clearly.
+    genai_metric_args = {
+        "name": name,
+        "definition": definition,
+        "grading_prompt": grading_prompt,
+        "examples": examples,
+        "version": version,
+        "model": model,
+        "grading_context_columns": grading_context_columns,
+        "include_input": include_input,
+        "parameters": parameters,
+        "aggregations": aggregations,
+        "greater_is_better": greater_is_better,
+        "max_workers": max_workers,
+        "metric_metadata": metric_metadata,
+        # Record the mlflow version for serialization in case the function signature changes later
+        "mlflow_version": VERSION,
+        "fn_name": make_genai_metric.__name__,
+    }
+
+    aggregations = aggregations or ["mean", "variance", "p90"]
+    grading_context_columns = grading_context_columns or []
+
+    if not isinstance(grading_context_columns, list):
+        grading_context_columns = [grading_context_columns]
+
+    def process_example(example):
+        if example.grading_context is None and len(grading_context_columns) == 0:
+            grading_context = {}
+        elif isinstance(example.grading_context, dict):
+            grading_context = example.grading_context
+        else:
+            # The grading context is string-like. Assume that it corresponds to the first
+            # grading context column and update the example accordingly
+            grading_context = {grading_context_columns[0]: example.grading_context}
+            example.grading_context = grading_context
+
+        if set(grading_context.keys()) != set(grading_context_columns):
+            raise MlflowException.invalid_parameter_value(
+                f"Example grading context does not contain required columns.\n"
+                f" Example grading context columns: {list(grading_context.keys())}\n"
+                f" Required grading context columns: {grading_context_columns}\n"
+            )
+
+        if not include_input:
+            return EvaluationExample(
+                output=example.output,
+                score=example.score,
+                justification=example.justification,
+                grading_context=example.grading_context,
+            )
+
+        return example
+
+    if examples is not None:
+        examples = [process_example(example) for example in examples]
+
+    class_name = f"mlflow.metrics.genai.prompts.{version}.EvaluationModel"
+    try:
+        evaluation_model_class_module = _get_class_from_string(class_name)
+    except ModuleNotFoundError:
+        raise MlflowException(
+            f"Failed to find evaluation model for version {version}."
+            f" Please check the correctness of the version",
+            error_code=INVALID_PARAMETER_VALUE,
+        ) from None
+    except Exception as e:
+        raise MlflowException(
+            f"Failed to construct evaluation model {version}. Error: {e!r}",
+            error_code=INTERNAL_ERROR,
+        ) from None
+
+    evaluation_context = evaluation_model_class_module(
+        name,
+        definition,
+        grading_prompt,
+        examples,
+        model,
+        *(parameters,) if parameters is not None else (),
+    ).to_dict()
+
+    def eval_fn(
+        predictions: "pd.Series",
+        metrics: dict[str, MetricValue],
+        inputs: "pd.Series",
+        *args,
+    ) -> MetricValue:
+        """
+        This is the function that is called when the metric is evaluated.
+        """
+        eval_values = dict(zip(grading_context_columns, args))
+
+        outputs = predictions.to_list()
+        inputs = inputs.to_list()
+        eval_model = evaluation_context["model"]
+        eval_parameters = evaluation_context["parameters"]
+
+        # TODO: Save the metric definition in a yaml file for model monitoring
+
+        if not isinstance(eval_model, str):
+            raise MlflowException(
+                message="The model argument must be a string URI referring to an openai model "
+                "(openai:/gpt-4o-mini) or an MLflow Deployments endpoint "
+                f"(endpoints:/my-endpoint), passed {eval_model} instead",
+                error_code=INVALID_PARAMETER_VALUE,
+            )
+
+        # generate grading payloads
+        grading_payloads = []
+        for indx, (input, output) in enumerate(zip(inputs, outputs)):
+            try:
+                arg_string = _format_args_string(grading_context_columns, eval_values, indx)
+            except Exception as e:
+                raise MlflowException(
+                    f"Values for grading_context_columns are malformed and cannot be "
+                    f"formatted into a prompt for metric '{name}'.\n"
+                    f"Required columns: {grading_context_columns}\n"
+                    f"Values: {eval_values}\n"
+                    f"Error: {e!r}\n"
+                    f"Please check the following: \n"
+                    "- predictions and targets (if required) are provided correctly\n"
+                    "- grading_context_columns are mapped correctly using the evaluator_config "
+                    "parameter\n"
+                    "- input and output data are formatted correctly."
+                )
+            grading_payloads.append(
+                evaluation_context["eval_prompt"].format(
+                    input=(input if include_input else None),
+                    output=output,
+                    grading_context_columns=arg_string,
+                )
+            )
+
+        scores = [None] * len(inputs)
+        justifications = [None] * len(inputs)
+
+        with ThreadPoolExecutor(max_workers=max_workers) as executor:
+            futures = {
+                executor.submit(
+                    _score_model_on_one_payload,
+                    payload,
+                    eval_model,
+                    eval_parameters,
+                    extra_headers,
+                    proxy_url,
+                ): indx
+                for indx, payload in enumerate(grading_payloads)
+            }
+
+            as_comp = as_completed(futures)
+            try:
+                from tqdm.auto import tqdm
+
+                as_comp = tqdm(as_comp, total=len(futures))
+            except ImportError:
+                pass
+
+            for future in as_comp:
+                indx = futures[future]
+                score, justification = future.result()
+                scores[indx] = score
+                justifications[indx] = justification
+
+        aggregate_results = _get_aggregate_results(scores, aggregations)
+        return MetricValue(scores, justifications, aggregate_results)
+
+    signature_parameters = [
+        Parameter("predictions", Parameter.POSITIONAL_OR_KEYWORD, annotation="pd.Series"),
+        Parameter("metrics", Parameter.POSITIONAL_OR_KEYWORD, annotation=dict[str, MetricValue]),
+        Parameter("inputs", Parameter.POSITIONAL_OR_KEYWORD, annotation="pd.Series"),
+    ]
+
+    # Add grading_context_columns to signature list
+    for var in grading_context_columns:
+        signature_parameters.append(Parameter(var, Parameter.POSITIONAL_OR_KEYWORD))
+
+    # Note: this doesn't change how python allows calling the function
+    # extra params in grading_context_columns can only be passed as positional args
+    eval_fn.__signature__ = Signature(signature_parameters)
+
+    return _make_metric(
+        eval_fn=eval_fn,
+        greater_is_better=greater_is_better,
+        name=name,
+        version=version,
+        metric_details=evaluation_context["eval_prompt"].__str__(),
+        metric_metadata=metric_metadata,
+        genai_metric_args=genai_metric_args,
+        require_strict_signature=True,
+    )
+
+
+def _filter_by_field(df, field_name, value):
+    return df[df[field_name] == value]
+
+
+def _deserialize_genai_metric_args(args_dict):
+    mlflow_version_at_ser = args_dict.pop("mlflow_version", None)
+    fn_name = args_dict.pop("fn_name", None)
+    if fn_name is None or mlflow_version_at_ser is None:
+        raise MlflowException(
+            message="The artifact JSON file appears to be corrupted and cannot be deserialized. "
+            "Please regenerate the custom metrics and rerun the evaluation. "
+            "Ensure that the file is correctly formatted and not tampered with.",
+            error_code=INTERNAL_ERROR,
+        )
+
+    if mlflow_version_at_ser != VERSION:
+        warnings.warn(
+            f"The custom metric definitions were serialized using MLflow {mlflow_version_at_ser}. "
+            f"Deserializing them with the current version {VERSION} might cause mismatches. "
+            "Please ensure compatibility or consider regenerating the metrics "
+            "using the current version.",
+            UserWarning,
+            stacklevel=2,
+        )
+
+    if fn_name == make_genai_metric_from_prompt.__name__:
+        return make_genai_metric_from_prompt(**args_dict)
+
+    examples = args_dict["examples"]
+    if examples is not None:
+        args_dict["examples"] = [EvaluationExample(**example) for example in examples]
+
+    return make_genai_metric(**args_dict)
+
+
+def retrieve_custom_metrics(
+    run_id: str,
+    name: Optional[str] = None,
+    version: Optional[str] = None,
+) -> list[EvaluationMetric]:
+    """
+    Retrieve the custom metrics created by users through `make_genai_metric()` or
+    `make_genai_metric_from_prompt()` that are associated with a particular evaluation run.
+
+    Args:
+        run_id: The unique identifier for the run.
+        name: (Optional) The name of the custom metric to retrieve.
+            If None, retrieve all metrics.
+        version: (Optional) The version of the custom metric to retrieve.
+            If None, retrieve all metrics.
+
+    Returns:
+        A list of EvaluationMetric objects that match the retrieval criteria.
+
+    .. code-block:: python
+        :caption: Example for retrieving a custom genai metric
+
+        import pandas as pd
+
+        import mlflow
+        from mlflow.metrics.genai.genai_metric import (
+            make_genai_metric_from_prompt,
+            retrieve_custom_metrics,
+        )
+
+        eval_df = pd.DataFrame(
+            {
+                "inputs": ["foo"],
+                "ground_truth": ["bar"],
+            }
+        )
+        with mlflow.start_run() as run:
+            system_prompt = "Answer the following question in two sentences"
+            basic_qa_model = mlflow.openai.log_model(
+                model="gpt-4o-mini",
+                task="chat.completions",
+                artifact_path="model",
+                messages=[
+                    {"role": "system", "content": system_prompt},
+                    {"role": "user", "content": "{question}"},
+                ],
+            )
+            custom_metric = make_genai_metric_from_prompt(
+                name="custom llm judge",
+                judge_prompt="This is a custom judge prompt.",
+                greater_is_better=False,
+                parameters={"temperature": 0.0},
+            )
+            results = mlflow.evaluate(
+                basic_qa_model.model_uri,
+                eval_df,
+                targets="ground_truth",
+                model_type="question-answering",
+                evaluators="default",
+                extra_metrics=[custom_metric],
+            )
+        metrics = retrieve_custom_metrics(
+            run_id=run.info.run_id,
+            name="custom llm judge",
+        )
+    """
+    client = mlflow.MlflowClient()
+    artifacts = [a.path for a in client.list_artifacts(run_id)]
+    if _GENAI_CUSTOM_METRICS_FILE_NAME not in artifacts:
+        _logger.warning("No custom metric definitions were found for this evaluation run.")
+        return []
+
+    with TemporaryDirectory() as tmpdir:
+        downloaded_artifact_path = mlflow.artifacts.download_artifacts(
+            run_id=run_id,
+            artifact_path=_GENAI_CUSTOM_METRICS_FILE_NAME,
+            dst_path=tmpdir,
+        )
+        custom_metrics = client._read_from_file(downloaded_artifact_path)
+
+    if name is not None:
+        custom_metrics = _filter_by_field(custom_metrics, "name", name)
+    if version is not None:
+        custom_metrics = _filter_by_field(custom_metrics, "version", version)
+    metric_args_list = custom_metrics["metric_args"].tolist()
+    if len(metric_args_list) == 0:
+        _logger.warning("No matching custom metric definitions were found.")
+        return []
+
+    return [_deserialize_genai_metric_args(a) for a in metric_args_list]

venv/lib/python3.9/site-packages/mlflow/metrics/genai/metric_definitions.py

@@ -0,0 +1,455 @@
+from typing import Any, Optional
+
+from mlflow.exceptions import MlflowException
+from mlflow.metrics.genai.base import EvaluationExample
+from mlflow.metrics.genai.genai_metric import make_genai_metric
+from mlflow.metrics.genai.utils import _get_latest_metric_version
+from mlflow.models import EvaluationMetric
+from mlflow.protos.databricks_pb2 import INTERNAL_ERROR, INVALID_PARAMETER_VALUE
+from mlflow.utils.annotations import experimental
+from mlflow.utils.class_utils import _get_class_from_string
+
+
+@experimental
+def answer_similarity(
+    model: Optional[str] = None,
+    metric_version: Optional[str] = None,
+    examples: Optional[list[EvaluationExample]] = None,
+    metric_metadata: Optional[dict[str, Any]] = None,
+    parameters: Optional[dict[str, Any]] = None,
+    extra_headers: Optional[dict[str, str]] = None,
+    proxy_url: Optional[str] = None,
+    max_workers: int = 10,
+) -> EvaluationMetric:
+    """
+    This function will create a genai metric used to evaluate the answer similarity of an LLM
+    using the model provided. Answer similarity will be assessed by the semantic similarity of the
+    output to the ``ground_truth``, which should be specified in the ``targets`` column. High
+    scores mean that your model outputs contain similar information as the ground_truth, while
+    low scores mean that outputs may disagree with the ground_truth.
+
+    The ``targets`` eval_arg must be provided as part of the input dataset or output
+    predictions. This can be mapped to a column of a different name using ``col_mapping``
+    in the ``evaluator_config`` parameter, or using the ``targets`` parameter in mlflow.evaluate().
+
+    An MlflowException will be raised if the specified version for this metric does not exist.
+
+    Args:
+        model: (Optional) Model uri of the judge model that will be used to compute the metric,
+            e.g., ``openai:/gpt-4``. Refer to the `LLM-as-a-Judge Metrics <https://mlflow.org/docs/latest/llms/llm-evaluate/index.html#selecting-the-llm-as-judge-model>`_
+            documentation for the supported model types and their URI format.
+        metric_version: (Optional) The version of the answer similarity metric to use.
+            Defaults to the latest version.
+        examples: (Optional) Provide a list of examples to help the judge model evaluate the
+            answer similarity. It is highly recommended to add examples to be used as a reference to
+            evaluate the new results.
+        metric_metadata: (Optional) Dictionary of metadata to be attached to the
+            EvaluationMetric object. Useful for model evaluators that require additional
+            information to determine how to evaluate this metric.
+        parameters: (Optional) Dictionary of parameters to be passed to the judge model,
+            e.g., {"temperature": 0.5}. When specified, these parameters will override
+            the default parameters defined in the metric implementation.
+        extra_headers: (Optional) Dictionary of extra headers to be passed to the judge model.
+        proxy_url: (Optional) Proxy URL to be used for the judge model. This is useful when the
+            judge model is served via a proxy endpoint, not directly via LLM provider services.
+            If not specified, the default URL for the LLM provider will be used
+            (e.g., https://api.openai.com/v1/chat/completions for OpenAI chat models).
+        max_workers: (Optional) The maximum number of workers to use for judge scoring.
+            Defaults to 10 workers.
+
+    Returns:
+        A metric object
+    """
+    if metric_version is None:
+        metric_version = _get_latest_metric_version()
+    class_name = f"mlflow.metrics.genai.prompts.{metric_version}.AnswerSimilarityMetric"
+    try:
+        answer_similarity_class_module = _get_class_from_string(class_name)
+    except ModuleNotFoundError:
+        raise MlflowException(
+            f"Failed to find answer similarity metric for version {metric_version}."
+            f" Please check the version",
+            error_code=INVALID_PARAMETER_VALUE,
+        ) from None
+    except Exception as e:
+        raise MlflowException(
+            f"Failed to construct answer similarity metric {metric_version}. Error: {e!r}",
+            error_code=INTERNAL_ERROR,
+        ) from None
+
+    if examples is None:
+        examples = answer_similarity_class_module.default_examples
+    if model is None:
+        model = answer_similarity_class_module.default_model
+
+    return make_genai_metric(
+        name="answer_similarity",
+        definition=answer_similarity_class_module.definition,
+        grading_prompt=answer_similarity_class_module.grading_prompt,
+        include_input=False,
+        examples=examples,
+        version=metric_version,
+        model=model,
+        grading_context_columns=answer_similarity_class_module.grading_context_columns,
+        parameters=parameters or answer_similarity_class_module.parameters,
+        aggregations=["mean", "variance", "p90"],
+        greater_is_better=True,
+        metric_metadata=metric_metadata,
+        extra_headers=extra_headers,
+        proxy_url=proxy_url,
+        max_workers=max_workers,
+    )
+
+
+@experimental
+def answer_correctness(
+    model: Optional[str] = None,
+    metric_version: Optional[str] = None,
+    examples: Optional[list[EvaluationExample]] = None,
+    metric_metadata: Optional[dict[str, Any]] = None,
+    parameters: Optional[dict[str, Any]] = None,
+    extra_headers: Optional[dict[str, str]] = None,
+    proxy_url: Optional[str] = None,
+    max_workers: int = 10,
+) -> EvaluationMetric:
+    """
+    This function will create a genai metric used to evaluate the answer correctness of an LLM
+    using the model provided. Answer correctness will be assessed by the accuracy of the provided
+    output based on the ``ground_truth``, which should be specified in the ``targets`` column.
+    High scores mean that your model outputs contain similar information as the ground_truth and
+    that this information is correct, while low scores mean that outputs may disagree with the
+    ground_truth or that the information in the output is incorrect. Note that this builds onto
+    answer_similarity.
+
+    The ``targets`` eval_arg must be provided as part of the input dataset or output
+    predictions. This can be mapped to a column of a different name using ``col_mapping``
+    in the ``evaluator_config`` parameter, or using the ``targets`` parameter in mlflow.evaluate().
+
+    An MlflowException will be raised if the specified version for this metric does not exist.
+
+    Args:
+        model: (Optional) Model uri of the judge model that will be used to compute the metric,
+            e.g., ``openai:/gpt-4``. Refer to the `LLM-as-a-Judge Metrics <https://mlflow.org/docs/latest/llms/llm-evaluate/index.html#selecting-the-llm-as-judge-model>`_
+            documentation for the supported model types and their URI format.
+        metric_version: The version of the answer correctness metric to use.
+            Defaults to the latest version.
+        examples: Provide a list of examples to help the judge model evaluate the
+            answer correctness. It is highly recommended to add examples to be used as a reference
+            to evaluate the new results.
+        metric_metadata: (Optional) Dictionary of metadata to be attached to the
+            EvaluationMetric object. Useful for model evaluators that require additional
+            information to determine how to evaluate this metric.
+        parameters: (Optional) Dictionary of parameters to be passed to the judge model,
+            e.g., {"temperature": 0.5}. When specified, these parameters will override
+            the default parameters defined in the metric implementation.
+        extra_headers: (Optional) Dictionary of extra headers to be passed to the judge model.
+        proxy_url: (Optional) Proxy URL to be used for the judge model. This is useful when the
+            judge model is served via a proxy endpoint, not directly via LLM provider services.
+            If not specified, the default URL for the LLM provider will be used
+            (e.g., https://api.openai.com/v1/chat/completions for OpenAI chat models).
+        max_workers: (Optional) The maximum number of workers to use for judge scoring.
+            Defaults to 10 workers.
+
+    Returns:
+        A metric object
+    """
+    if metric_version is None:
+        metric_version = _get_latest_metric_version()
+    class_name = f"mlflow.metrics.genai.prompts.{metric_version}.AnswerCorrectnessMetric"
+    try:
+        answer_correctness_class_module = _get_class_from_string(class_name)
+    except ModuleNotFoundError:
+        raise MlflowException(
+            f"Failed to find answer correctness metric for version {metric_version}."
+            f"Please check the version",
+            error_code=INVALID_PARAMETER_VALUE,
+        ) from None
+    except Exception as e:
+        raise MlflowException(
+            f"Failed to construct answer correctness metric {metric_version}. Error: {e!r}",
+            error_code=INTERNAL_ERROR,
+        ) from None
+
+    if examples is None:
+        examples = answer_correctness_class_module.default_examples
+    if model is None:
+        model = answer_correctness_class_module.default_model
+
+    return make_genai_metric(
+        name="answer_correctness",
+        definition=answer_correctness_class_module.definition,
+        grading_prompt=answer_correctness_class_module.grading_prompt,
+        examples=examples,
+        version=metric_version,
+        model=model,
+        grading_context_columns=answer_correctness_class_module.grading_context_columns,
+        parameters=parameters or answer_correctness_class_module.parameters,
+        aggregations=["mean", "variance", "p90"],
+        greater_is_better=True,
+        metric_metadata=metric_metadata,
+        extra_headers=extra_headers,
+        proxy_url=proxy_url,
+        max_workers=max_workers,
+    )
+
+
+@experimental
+def faithfulness(
+    model: Optional[str] = None,
+    metric_version: Optional[str] = _get_latest_metric_version(),
+    examples: Optional[list[EvaluationExample]] = None,
+    metric_metadata: Optional[dict[str, Any]] = None,
+    parameters: Optional[dict[str, Any]] = None,
+    extra_headers: Optional[dict[str, str]] = None,
+    proxy_url: Optional[str] = None,
+    max_workers: int = 10,
+) -> EvaluationMetric:
+    """
+    This function will create a genai metric used to evaluate the faithfullness of an LLM using the
+    model provided. Faithfulness will be assessed based on how factually consistent the output
+    is to the ``context``. High scores mean that the outputs contain information that is in
+    line with the context, while low scores mean that outputs may disagree with the context
+    (input is ignored).
+
+    The ``context`` eval_arg must be provided as part of the input dataset or output
+    predictions. This can be mapped to a column of a different name using ``col_mapping``
+    in the ``evaluator_config`` parameter.
+
+    An MlflowException will be raised if the specified version for this metric does not exist.
+
+    Args:
+        model: (Optional) Model uri of the judge model that will be used to compute the metric,
+            e.g., ``openai:/gpt-4``. Refer to the `LLM-as-a-Judge Metrics <https://mlflow.org/docs/latest/llms/llm-evaluate/index.html#selecting-the-llm-as-judge-model>`_
+            documentation for the supported model types and their URI format.
+        metric_version: The version of the faithfulness metric to use.
+            Defaults to the latest version.
+        examples: Provide a list of examples to help the judge model evaluate the
+            faithfulness. It is highly recommended to add examples to be used as a reference to
+            evaluate the new results.
+        metric_metadata: (Optional) Dictionary of metadata to be attached to the
+            EvaluationMetric object. Useful for model evaluators that require additional
+            information to determine how to evaluate this metric.
+        parameters: (Optional) Dictionary of parameters to be passed to the judge model,
+            e.g., {"temperature": 0.5}. When specified, these parameters will override
+            the default parameters defined in the metric implementation.
+        extra_headers: (Optional) Dictionary of extra headers to be passed to the judge model.
+        proxy_url: (Optional) Proxy URL to be used for the judge model. This is useful when the
+            judge model is served via a proxy endpoint, not directly via LLM provider services.
+            If not specified, the default URL for the LLM provider will be used
+            (e.g., https://api.openai.com/v1/chat/completions for OpenAI chat models).
+        max_workers: (Optional) The maximum number of workers to use for judge scoring.
+            Defaults to 10 workers.
+
+    Returns:
+        A metric object
+    """
+    class_name = f"mlflow.metrics.genai.prompts.{metric_version}.FaithfulnessMetric"
+    try:
+        faithfulness_class_module = _get_class_from_string(class_name)
+    except ModuleNotFoundError:
+        raise MlflowException(
+            f"Failed to find faithfulness metric for version {metric_version}."
+            f" Please check the version",
+            error_code=INVALID_PARAMETER_VALUE,
+        ) from None
+    except Exception as e:
+        raise MlflowException(
+            f"Failed to construct faithfulness metric {metric_version}. Error: {e!r}",
+            error_code=INTERNAL_ERROR,
+        ) from None
+
+    if examples is None:
+        examples = faithfulness_class_module.default_examples
+    if model is None:
+        model = faithfulness_class_module.default_model
+
+    return make_genai_metric(
+        name="faithfulness",
+        definition=faithfulness_class_module.definition,
+        grading_prompt=faithfulness_class_module.grading_prompt,
+        include_input=False,
+        examples=examples,
+        version=metric_version,
+        model=model,
+        grading_context_columns=faithfulness_class_module.grading_context_columns,
+        parameters=parameters or faithfulness_class_module.parameters,
+        aggregations=["mean", "variance", "p90"],
+        greater_is_better=True,
+        metric_metadata=metric_metadata,
+        extra_headers=extra_headers,
+        proxy_url=proxy_url,
+        max_workers=max_workers,
+    )
+
+
+@experimental
+def answer_relevance(
+    model: Optional[str] = None,
+    metric_version: Optional[str] = _get_latest_metric_version(),
+    examples: Optional[list[EvaluationExample]] = None,
+    metric_metadata: Optional[dict[str, Any]] = None,
+    parameters: Optional[dict[str, Any]] = None,
+    extra_headers: Optional[dict[str, str]] = None,
+    proxy_url: Optional[str] = None,
+    max_workers: int = 10,
+) -> EvaluationMetric:
+    """
+    This function will create a genai metric used to evaluate the answer relevance of an LLM
+    using the model provided. Answer relevance will be assessed based on the appropriateness and
+    applicability of the output with respect to the input. High scores mean that your model
+    outputs are about the same subject as the input, while low scores mean that outputs may
+    be non-topical.
+
+    An MlflowException will be raised if the specified version for this metric does not exist.
+
+    Args:
+        model: (Optional) Model uri of the judge model that will be used to compute the metric,
+            e.g., ``openai:/gpt-4``. Refer to the `LLM-as-a-Judge Metrics <https://mlflow.org/docs/latest/llms/llm-evaluate/index.html#selecting-the-llm-as-judge-model>`_
+            documentation for the supported model types and their URI format.
+        metric_version: The version of the answer relevance metric to use.
+            Defaults to the latest version.
+        examples: Provide a list of examples to help the judge model evaluate the
+            answer relevance. It is highly recommended to add examples to be used as a reference to
+            evaluate the new results.
+        metric_metadata: (Optional) Dictionary of metadata to be attached to the
+            EvaluationMetric object. Useful for model evaluators that require additional
+            information to determine how to evaluate this metric.
+        parameters: (Optional) Dictionary of parameters to be passed to the judge model,
+            e.g., {"temperature": 0.5}. When specified, these parameters will override
+            the default parameters defined in the metric implementation.
+        extra_headers: (Optional) Dictionary of extra headers to be passed to the judge model.
+        proxy_url: (Optional) Proxy URL to be used for the judge model. This is useful when the
+            judge model is served via a proxy endpoint, not directly via LLM provider services.
+            If not specified, the default URL for the LLM provider will be used
+            (e.g., https://api.openai.com/v1/chat/completions for OpenAI chat models).
+        max_workers: (Optional) The maximum number of workers to use for judge scoring.
+            Defaults to 10 workers.
+
+    Returns:
+        A metric object
+    """
+    class_name = f"mlflow.metrics.genai.prompts.{metric_version}.AnswerRelevanceMetric"
+    try:
+        answer_relevance_class_module = _get_class_from_string(class_name)
+    except ModuleNotFoundError:
+        raise MlflowException(
+            f"Failed to find answer relevance metric for version {metric_version}."
+            f" Please check the version",
+            error_code=INVALID_PARAMETER_VALUE,
+        ) from None
+    except Exception as e:
+        raise MlflowException(
+            f"Failed to construct answer relevance metric {metric_version}. Error: {e!r}",
+            error_code=INTERNAL_ERROR,
+        ) from None
+
+    if examples is None:
+        examples = answer_relevance_class_module.default_examples
+    if model is None:
+        model = answer_relevance_class_module.default_model
+
+    return make_genai_metric(
+        name="answer_relevance",
+        definition=answer_relevance_class_module.definition,
+        grading_prompt=answer_relevance_class_module.grading_prompt,
+        examples=examples,
+        version=metric_version,
+        model=model,
+        parameters=parameters or answer_relevance_class_module.parameters,
+        aggregations=["mean", "variance", "p90"],
+        greater_is_better=True,
+        metric_metadata=metric_metadata,
+        extra_headers=extra_headers,
+        proxy_url=proxy_url,
+        max_workers=max_workers,
+    )
+
+
+def relevance(
+    model: Optional[str] = None,
+    metric_version: Optional[str] = None,
+    examples: Optional[list[EvaluationExample]] = None,
+    metric_metadata: Optional[dict[str, Any]] = None,
+    parameters: Optional[dict[str, Any]] = None,
+    extra_headers: Optional[dict[str, str]] = None,
+    proxy_url: Optional[str] = None,
+    max_workers: int = 10,
+) -> EvaluationMetric:
+    """
+    This function will create a genai metric used to evaluate the evaluate the relevance of an
+    LLM using the model provided. Relevance will be assessed by the appropriateness, significance,
+    and applicability of the output with respect to the input and ``context``. High scores mean
+    that the model has understood the context and correct extracted relevant information from
+    the context, while low score mean that output has completely ignored the question and the
+    context and could be hallucinating.
+
+    The ``context`` eval_arg must be provided as part of the input dataset or output
+    predictions. This can be mapped to a column of a different name using ``col_mapping``
+    in the ``evaluator_config`` parameter.
+
+    An MlflowException will be raised if the specified version for this metric does not exist.
+
+    Args:
+        model: (Optional) Model uri of the judge model that will be used to compute the metric,
+            e.g., ``openai:/gpt-4``. Refer to the `LLM-as-a-Judge Metrics <https://mlflow.org/docs/latest/llms/llm-evaluate/index.html#selecting-the-llm-as-judge-model>`_
+            documentation for the supported model types and their URI format.
+        metric_version: (Optional) The version of the relevance metric to use.
+            Defaults to the latest version.
+        examples: (Optional) Provide a list of examples to help the judge model evaluate the
+            relevance. It is highly recommended to add examples to be used as a reference to
+            evaluate the new results.
+        metric_metadata: (Optional) Dictionary of metadata to be attached to the
+            EvaluationMetric object. Useful for model evaluators that require additional
+            information to determine how to evaluate this metric.
+        parameters: (Optional) Dictionary of parameters to be passed to the judge model,
+            e.g., {"temperature": 0.5}. When specified, these parameters will override
+            the default parameters defined in the metric implementation.
+        extra_headers: (Optional) Dictionary of extra headers to be passed to the judge model.
+        proxy_url: (Optional) Proxy URL to be used for the judge model. This is useful when the
+            judge model is served via a proxy endpoint, not directly via LLM provider services.
+            If not specified, the default URL for the LLM provider will be used
+            (e.g., https://api.openai.com/v1/chat/completions for OpenAI chat models).
+        max_workers: (Optional) The maximum number of workers to use for judge scoring.
+            Defaults to 10 workers.
+
+    Returns:
+        A metric object
+    """
+    if metric_version is None:
+        metric_version = _get_latest_metric_version()
+    class_name = f"mlflow.metrics.genai.prompts.{metric_version}.RelevanceMetric"
+    try:
+        relevance_class_module = _get_class_from_string(class_name)
+    except ModuleNotFoundError:
+        raise MlflowException(
+            f"Failed to find relevance metric for version {metric_version}."
+            f"Please check the version",
+            error_code=INVALID_PARAMETER_VALUE,
+        ) from None
+    except Exception as e:
+        raise MlflowException(
+            f"Failed to construct relevance metric {metric_version}. Error: {e!r}",
+            error_code=INTERNAL_ERROR,
+        ) from None
+
+    if examples is None:
+        examples = relevance_class_module.default_examples
+    if model is None:
+        model = relevance_class_module.default_model
+
+    return make_genai_metric(
+        name="relevance",
+        definition=relevance_class_module.definition,
+        grading_prompt=relevance_class_module.grading_prompt,
+        examples=examples,
+        version=metric_version,
+        model=model,
+        grading_context_columns=relevance_class_module.grading_context_columns,
+        parameters=parameters or relevance_class_module.parameters,
+        aggregations=["mean", "variance", "p90"],
+        greater_is_better=True,
+        metric_metadata=metric_metadata,
+        extra_headers=extra_headers,
+        proxy_url=proxy_url,
+        max_workers=max_workers,
+    )

venv/lib/python3.9/site-packages/mlflow/metrics/genai/model_utils.py

@@ -0,0 +1,395 @@
+import logging
+import os
+import urllib.parse
+from typing import TYPE_CHECKING, Any, Optional, Union
+
+import requests
+
+from mlflow.exceptions import MlflowException
+from mlflow.protos.databricks_pb2 import INVALID_PARAMETER_VALUE
+
+if TYPE_CHECKING:
+    from mlflow.gateway.providers import BaseProvider
+
+_logger = logging.getLogger(__name__)
+
+
+def get_endpoint_type(endpoint_uri: str) -> Optional[str]:
+    """
+    Get the type of the endpoint if it is MLflow deployment
+    endpoint. For other endpoints e.g. OpenAI, or if the
+    endpoint does not specify type, return None.
+    """
+    schema, path = _parse_model_uri(endpoint_uri)
+
+    if schema != "endpoints":
+        return None
+
+    from pydantic import BaseModel
+
+    from mlflow.deployments import get_deploy_client
+
+    client = get_deploy_client()
+
+    endpoint = client.get_endpoint(path)
+    # TODO: Standardize the return type of `get_endpoint` and remove this check
+    endpoint = endpoint.dict() if isinstance(endpoint, BaseModel) else endpoint
+    return endpoint.get("task", endpoint.get("endpoint_type"))
+
+
+# TODO: improve this name
+def score_model_on_payload(
+    model_uri,
+    payload,
+    eval_parameters=None,
+    extra_headers=None,
+    proxy_url=None,
+    endpoint_type=None,
+):
+    """Call the model identified by the given uri with the given string prompt."""
+
+    eval_parameters = eval_parameters or {}
+    extra_headers = extra_headers or {}
+
+    prefix, suffix = _parse_model_uri(model_uri)
+
+    if prefix == "gateway":
+        return _call_gateway_api(suffix, payload, eval_parameters)
+    elif prefix == "endpoints":
+        return call_deployments_api(suffix, payload, eval_parameters, endpoint_type)
+    elif prefix in ("model", "runs"):
+        # TODO: call _load_model_or_server
+        raise NotImplementedError
+
+    # Import here to avoid loading gateway module at the top level
+    from mlflow.gateway.provider_registry import is_supported_provider
+
+    if is_supported_provider(prefix):
+        return _call_llm_provider_api(
+            prefix, suffix, payload, eval_parameters, extra_headers, proxy_url
+        )
+
+    raise MlflowException(
+        f"Unknown model uri prefix '{prefix}'",
+        error_code=INVALID_PARAMETER_VALUE,
+    )
+
+
+def _parse_model_uri(model_uri):
+    parsed = urllib.parse.urlparse(model_uri, allow_fragments=False)
+    scheme = parsed.scheme
+    path = parsed.path
+    if not path.startswith("/") or len(path) <= 1:
+        raise MlflowException(
+            f"Malformed model uri '{model_uri}'", error_code=INVALID_PARAMETER_VALUE
+        )
+    path = path.lstrip("/")
+    return scheme, path
+
+
+_PREDICT_ERROR_MSG = """\
+Failed to call the deployment endpoint. Please check the deployment URL \
+is set correctly and the input payload is valid.\n
+- Error: {e}\n
+- Deployment URI: {uri}\n
+- Input payload: {payload}"""
+
+
+def _is_supported_llm_provider(schema: str) -> bool:
+    from mlflow.gateway.provider_registry import provider_registry
+
+    return schema in provider_registry.keys()
+
+
+def _call_llm_provider_api(
+    provider_name: str,
+    model: str,
+    input_data: str,
+    eval_parameters: dict[str, Any],
+    extra_headers: dict[str, str],
+    proxy_url: Optional[str] = None,
+) -> str:
+    """
+    Invoke chat endpoint of various LLM providers.
+
+    Under the hood, this function uses the MLflow Gateway to transform the input/output data
+    for different LLM providers.
+
+    Args:
+        provider_name: The provider name, e.g., "anthropic".
+        model: The model name, e.g., "claude-3-5-sonnet"
+        input_data: The input string prompt to send to the model as a chat message.
+        eval_parameters: The additional parameters to send to the model, e.g. temperature.
+        extra_headers: The additional headers to send to the provider.
+        proxy_url: Proxy URL to be used for the judge model. If not specified, the default
+            URL for the LLM provider will be used.
+    """
+    from mlflow.gateway.config import Provider
+    from mlflow.gateway.schemas import chat
+
+    provider = _get_provider_instance(provider_name, model)
+
+    chat_request = chat.RequestPayload(
+        model=model,
+        messages=[
+            chat.RequestMessage(role="user", content=input_data),
+        ],
+        **eval_parameters,
+    )
+
+    # Filter out keys in the payload to the specified ones + "messages".
+    # Does not include "model" key here because some providers do not accept it as a
+    # part of the payload. Whether or not to include "model" key must be determined
+    # by each provider implementation.
+    filtered_keys = {"messages", *eval_parameters.keys()}
+
+    payload = {
+        k: v
+        for k, v in chat_request.model_dump(exclude_none=True).items()
+        if (v is not None) and (k in filtered_keys)
+    }
+    chat_payload = provider.adapter_class.chat_to_model(payload, provider.config)
+    chat_payload.update(eval_parameters)
+
+    if provider_name in [Provider.AMAZON_BEDROCK, Provider.BEDROCK]:
+        if proxy_url or extra_headers:
+            _logger.warning(
+                "Proxy URL and extra headers are not supported for Bedrock LLMs. "
+                "Ignoring the provided proxy URL and extra headers.",
+            )
+        response = provider._request(chat_payload)
+    else:
+        response = _send_request(
+            endpoint=proxy_url or provider.get_endpoint_url("llm/v1/chat"),
+            headers={**provider.headers, **extra_headers},
+            payload=chat_payload,
+        )
+    chat_response = provider.adapter_class.model_to_chat(response, provider.config)
+    if len(chat_response.choices) == 0:
+        raise MlflowException(
+            "Failed to score the provided input as the judge LLM did not return "
+            "any chat completion results in the response."
+        )
+    content = chat_response.choices[0].message.content
+
+    # NB: Evaluation only handles text content for now.
+    return content[0].text if isinstance(content, list) else content
+
+
+def _get_provider_instance(provider: str, model: str) -> "BaseProvider":
+    """Get the provider instance for the given provider name and the model name."""
+    from mlflow.gateway.config import Provider, RouteConfig
+
+    def _get_route_config(config):
+        return RouteConfig(
+            name=provider,
+            route_type="llm/v1/chat",
+            model={
+                "provider": provider,
+                "name": model,
+                "config": config.model_dump(),
+            },
+        )
+
+    # NB: Not all LLM providers in MLflow Gateway are supported here. We can add
+    # new ones as requested, as long as the provider support chat endpoints.
+    if provider == Provider.OPENAI:
+        from mlflow.gateway.providers.openai import OpenAIConfig, OpenAIProvider
+        from mlflow.openai import _get_api_config, _OAITokenHolder
+
+        api_config = _get_api_config()
+        api_token = _OAITokenHolder(api_config.api_type)
+        api_token.refresh()
+
+        config = OpenAIConfig(
+            openai_api_key=api_token.token,
+            openai_api_type=api_config.api_type or "openai",
+            openai_api_base=api_config.api_base,
+            openai_api_version=api_config.api_version,
+            openai_deployment_name=api_config.deployment_id,
+            openai_organization=api_config.organization,
+        )
+        return OpenAIProvider(_get_route_config(config))
+
+    elif provider == Provider.ANTHROPIC:
+        from mlflow.gateway.providers.anthropic import AnthropicConfig, AnthropicProvider
+
+        config = AnthropicConfig(anthropic_api_key=os.environ.get("ANTHROPIC_API_KEY"))
+        return AnthropicProvider(_get_route_config(config))
+
+    elif provider in [Provider.AMAZON_BEDROCK, Provider.BEDROCK]:
+        from mlflow.gateway.config import AWSIdAndKey, AWSRole
+        from mlflow.gateway.providers.bedrock import AmazonBedrockConfig, AmazonBedrockProvider
+
+        if aws_role_arn := os.environ.get("AWS_ROLE_ARN"):
+            aws_config = AWSRole(
+                aws_region=os.environ.get("AWS_REGION"),
+                aws_role_arn=aws_role_arn,
+            )
+        else:
+            aws_config = AWSIdAndKey(
+                aws_region=os.environ.get("AWS_REGION"),
+                aws_access_key_id=os.environ.get("AWS_ACCESS_KEY_ID"),
+                aws_secret_access_key=os.environ.get("AWS_SECRET_ACCESS_KEY"),
+                aws_session_token=os.environ.get("AWS_SESSION_TOKEN"),
+            )
+        config = AmazonBedrockConfig(aws_config=aws_config)
+        return AmazonBedrockProvider(_get_route_config(config))
+
+    # # Cohere provider implementation seems to be broken and does not work with
+    # # their latest APIs. Uncomment once the provider implementation is fixed.
+    # elif provider == Provider.COHERE:
+    #     from mlflow.gateway.providers.cohere import CohereConfig, CohereProvider
+
+    #     config = CohereConfig(cohere_api_key=os.environ.get("COHERE_API_KEY"))
+    #     return CohereProvider(_get_route_config(config))
+
+    elif provider == Provider.MISTRAL:
+        from mlflow.gateway.providers.mistral import MistralConfig, MistralProvider
+
+        config = MistralConfig(mistral_api_key=os.environ.get("MISTRAL_API_KEY"))
+        return MistralProvider(_get_route_config(config))
+
+    elif provider == Provider.TOGETHERAI:
+        from mlflow.gateway.providers.togetherai import TogetherAIConfig, TogetherAIProvider
+
+        config = TogetherAIConfig(togetherai_api_key=os.environ.get("TOGETHERAI_API_KEY"))
+        return TogetherAIProvider(_get_route_config(config))
+
+    raise MlflowException(f"Provider '{provider}' is not supported for evaluation.")
+
+
+def _send_request(
+    endpoint: str, headers: dict[str, str], payload: dict[str, Any]
+) -> dict[str, Any]:
+    try:
+        response = requests.post(
+            url=endpoint,
+            headers=headers,
+            json=payload,
+            timeout=60,
+        )
+        response.raise_for_status()
+    except requests.exceptions.HTTPError as e:
+        raise MlflowException(
+            f"Failed to call LLM endpoint at {endpoint}.\n- Error: {e}\n- Input payload: {payload}."
+        )
+
+    return response.json()
+
+
+def call_deployments_api(
+    deployment_uri: str,
+    input_data: Union[str, dict[str, Any]],
+    eval_parameters: Optional[dict[str, Any]] = None,
+    endpoint_type: Optional[str] = None,
+):
+    """Call the deployment endpoint with the given payload and parameters.
+
+    Args:
+        deployment_uri: The URI of the deployment endpoint.
+        input_data: The input string or dictionary to send to the endpoint.
+            - If it is a string, MLflow tries to construct the payload based on the endpoint type.
+            - If it is a dictionary, MLflow directly sends it to the endpoint.
+        eval_parameters: The evaluation parameters to send to the endpoint.
+        endpoint_type: The type of the endpoint. If specified, must be 'llm/v1/completions'
+            or 'llm/v1/chat'. If not specified, MLflow tries to get the endpoint type
+            from the endpoint, and if not found, directly sends the payload to the endpoint.
+
+    Returns:
+        The unpacked response from the endpoint.
+    """
+    from mlflow.deployments import get_deploy_client
+
+    client = get_deploy_client()
+
+    if isinstance(input_data, str):
+        payload = _construct_payload_from_str(input_data, endpoint_type)
+    elif isinstance(input_data, dict):
+        # If the input is a dictionary, we assume it is already in the correct format
+        payload = input_data
+    else:
+        raise MlflowException(
+            f"Invalid input data type {type(input_data)}. Must be a string or a dictionary.",
+            error_code=INVALID_PARAMETER_VALUE,
+        )
+    payload = {**payload, **(eval_parameters or {})}
+
+    try:
+        response = client.predict(endpoint=deployment_uri, inputs=payload)
+    except Exception as e:
+        raise MlflowException(
+            _PREDICT_ERROR_MSG.format(e=e, uri=deployment_uri, payload=payload)
+        ) from e
+
+    return _parse_response(response, endpoint_type)
+
+
+def _call_gateway_api(gateway_uri, payload, eval_parameters):
+    from mlflow.gateway import get_route, query
+
+    route_info = get_route(gateway_uri).dict()
+    if route_info["endpoint_type"] == "llm/v1/completions":
+        completions_payload = {
+            "prompt": payload,
+            **eval_parameters,
+        }
+        response = query(gateway_uri, completions_payload)
+        return _parse_completions_response_format(response)
+    elif route_info["endpoint_type"] == "llm/v1/chat":
+        chat_payload = {
+            "messages": [{"role": "user", "content": payload}],
+            **eval_parameters,
+        }
+        response = query(gateway_uri, chat_payload)
+        return _parse_chat_response_format(response)
+    else:
+        raise MlflowException(
+            f"Unsupported gateway route type: {route_info['endpoint_type']}. Use a "
+            "route of type 'llm/v1/completions' or 'llm/v1/chat' instead.",
+            error_code=INVALID_PARAMETER_VALUE,
+        )
+
+
+def _construct_payload_from_str(prompt: str, endpoint_type: str) -> dict[str, Any]:
+    """
+    Construct the payload from the input string based on the endpoint type.
+    If the endpoint type is not specified or unsupported one, raise an exception.
+    """
+    if endpoint_type == "llm/v1/completions":
+        return {"prompt": prompt}
+    elif endpoint_type == "llm/v1/chat":
+        return {"messages": [{"role": "user", "content": prompt}]}
+    else:
+        raise MlflowException(
+            f"Unsupported endpoint type: {endpoint_type}. If string input is provided, "
+            "the endpoint type must be 'llm/v1/completions' or 'llm/v1/chat'.",
+            error_code=INVALID_PARAMETER_VALUE,
+        )
+
+
+def _parse_response(
+    response: dict[str, Any], endpoint_type: Optional[str]
+) -> Union[Optional[str], dict[str, Any]]:
+    if endpoint_type == "llm/v1/completions":
+        return _parse_completions_response_format(response)
+    elif endpoint_type == "llm/v1/chat":
+        return _parse_chat_response_format(response)
+    else:
+        return response
+
+
+def _parse_chat_response_format(response):
+    try:
+        text = response["choices"][0]["message"]["content"]
+    except (KeyError, IndexError, TypeError):
+        text = None
+    return text
+
+
+def _parse_completions_response_format(response):
+    try:
+        text = response["choices"][0]["text"]
+    except (KeyError, IndexError, TypeError):
+        text = None
+    return text

venv/lib/python3.9/site-packages/mlflow/metrics/genai/prompt_template.py

@@ -0,0 +1,68 @@
+import string
+from typing import Any, Union
+
+
+class PromptTemplate:
+    """A prompt template for a language model.
+
+    A prompt template consists of an array of strings that will be concatenated together. It accepts
+    a set of parameters from the user that can be used to generate a prompt for a language model.
+
+    The template can be formatted using f-strings.
+
+    Example:
+
+        .. code-block:: python
+
+            from mlflow.metrics.genai.prompt_template import PromptTemplate
+
+            # Instantiation using initializer
+            prompt = PromptTemplate(template_str="Say {foo} {baz}")
+
+            # Instantiation using partial_fill
+            prompt = PromptTemplate(template_str="Say {foo} {baz}").partial_fill(foo="bar")
+
+            # Format the prompt
+            prompt.format(baz="qux")
+    """
+
+    def __init__(self, template_str: Union[str, list[str]]):
+        self.template_strs = [template_str] if isinstance(template_str, str) else template_str
+
+    @property
+    def variables(self):
+        return {
+            fname
+            for template_str in self.template_strs
+            for _, fname, _, _ in string.Formatter().parse(template_str)
+            if fname
+        }
+
+    def format(self, **kwargs: Any) -> str:
+        safe_kwargs = {k: v for k, v in kwargs.items() if v is not None}
+        formatted_strs = []
+        for template_str in self.template_strs:
+            extracted_variables = [
+                fname for _, fname, _, _ in string.Formatter().parse(template_str) if fname
+            ]
+            if all(item in safe_kwargs.keys() for item in extracted_variables):
+                formatted_strs.append(template_str.format(**safe_kwargs))
+
+        return "".join(formatted_strs)
+
+    def partial_fill(self, **kwargs: Any) -> "PromptTemplate":
+        safe_kwargs = {k: v for k, v in kwargs.items() if v is not None}
+        new_template_strs = []
+        for template_str in self.template_strs:
+            extracted_variables = [
+                fname for _, fname, _, _ in string.Formatter().parse(template_str) if fname
+            ]
+            safe_available_kwargs = {
+                k: safe_kwargs.get(k, "{" + k + "}") for k in extracted_variables
+            }
+            new_template_strs.append(template_str.format_map(safe_available_kwargs))
+
+        return PromptTemplate(template_str=new_template_strs)
+
+    def __str__(self) -> str:
+        return "".join(self.template_strs)

venv/lib/python3.9/site-packages/mlflow/metrics/genai/prompts/v1.py

@@ -0,0 +1,422 @@
+from dataclasses import dataclass, field
+from typing import Any, Optional
+
+from mlflow.metrics.genai.base import EvaluationExample
+from mlflow.metrics.genai.prompt_template import PromptTemplate
+
+# TODO: Update the default_mode and default_parameters to the correct values post experimentation
+default_model = "openai:/gpt-4"
+# Default parameters expressed in the OpenAI format
+default_parameters = {
+    "temperature": 0.0,
+    "max_tokens": 200,
+    "top_p": 1.0,
+}
+
+grading_system_prompt_template = PromptTemplate(
+    [
+        """
+Task:
+You must return the following fields in your response in two lines, one below the other:
+score: Your numerical score for the model's {name} based on the rubric
+justification: Your reasoning about the model's {name} score
+
+You are an impartial judge. You will be given an input that was sent to a machine
+learning model, and you will be given an output that the model produced. You
+may also be given additional information that was used by the model to generate the output.
+
+Your task is to determine a numerical score called {name} based on the input and output.
+A definition of {name} and a grading rubric are provided below.
+You must use the grading rubric to determine your score. You must also justify your score.
+
+Examples could be included below for reference. Make sure to use them as references and to
+understand them before completing the task.""",
+        """
+
+Input:
+{input}""",
+        """
+
+Output:
+{output}
+
+{grading_context_columns}
+
+Metric definition:
+{definition}
+
+Grading rubric:
+{grading_prompt}
+
+{examples}
+
+You must return the following fields in your response in two lines, one below the other:
+score: Your numerical score for the model's {name} based on the rubric
+justification: Your reasoning about the model's {name} score
+
+Do not add additional new lines. Do not add any other fields.
+    """,
+    ]
+)
+
+
+@dataclass
+class EvaluationModel:
+    """
+    Useful to compute v1 prompt for make_genai_metric
+    """
+
+    name: str
+    definition: str
+    grading_prompt: str
+    examples: Optional[list[EvaluationExample]] = None
+    model: str = default_model
+    parameters: dict[str, Any] = field(default_factory=lambda: default_parameters)
+
+    def to_dict(self):
+        examples_str = (
+            ""
+            if self.examples is None or len(self.examples) == 0
+            else f"Examples:\n{self._format_examples()}"
+        )
+
+        return {
+            "model": self.model,
+            "eval_prompt": grading_system_prompt_template.partial_fill(
+                name=self.name,
+                definition=self.definition,
+                grading_prompt=self.grading_prompt,
+                examples=examples_str,
+            ),
+            "parameters": self.parameters,
+        }
+
+    def _format_examples(self):
+        return "\n".join(map(str, self.examples))
+
+
+@dataclass
+class AnswerSimilarityMetric:
+    definition = (
+        "Answer similarity is evaluated on the degree of semantic similarity of the provided "
+        "output to the provided targets, which is the ground truth. Scores can be assigned based "
+        "on the gradual similarity in meaning and description to the provided targets, where a "
+        "higher score indicates greater alignment between the provided output and provided targets."
+    )
+
+    grading_prompt = (
+        "Answer similarity: Below are the details for different scores:\n"
+        "- Score 1: The output has little to no semantic similarity to the provided targets.\n"
+        "- Score 2: The output displays partial semantic similarity to the provided targets on "
+        "some aspects.\n"
+        "- Score 3: The output has moderate semantic similarity to the provided targets.\n"
+        "- Score 4: The output aligns with the provided targets in most aspects and has "
+        "substantial semantic similarity.\n"
+        "- Score 5: The output closely aligns with the provided targets in all significant aspects."
+    )
+
+    grading_context_columns = ["targets"]
+    parameters = default_parameters
+    default_model = default_model
+
+    example_score_2 = EvaluationExample(
+        input="What is MLflow?",
+        output="MLflow is an open-source platform.",
+        score=2,
+        justification="The provided output is partially similar to the target, as it captures the "
+        "general idea that MLflow is an open-source platform. However, it lacks the comprehensive "
+        "details and context provided in the target about MLflow's purpose, development, and "
+        "challenges it addresses. Therefore, it demonstrates partial, but not complete, "
+        "semantic similarity.",
+        grading_context={
+            "targets": "MLflow is an open-source platform for managing the end-to-end "
+            "machine learning (ML) lifecycle. It was developed by Databricks, a company "
+            "that specializes in big data and machine learning solutions. MLflow is "
+            "designed to address the challenges that data scientists and machine learning "
+            "engineers face when developing, training, and deploying machine learning "
+            "models."
+        },
+    )
+
+    example_score_4 = EvaluationExample(
+        input="What is MLflow?",
+        output="MLflow is an open-source platform for managing machine learning workflows, "
+        "including experiment tracking, model packaging, versioning, and deployment, simplifying "
+        "the ML lifecycle.",
+        score=4,
+        justification="The provided output aligns closely with the target. It covers various key "
+        "aspects mentioned in the target, including managing machine learning workflows, "
+        "experiment tracking, model packaging, versioning, and deployment. While it may not include"
+        " every single detail from the target, it demonstrates substantial semantic similarity.",
+        grading_context={
+            "targets": "MLflow is an open-source platform for managing the end-to-end "
+            "machine learning (ML) lifecycle. It was developed by Databricks, a company "
+            "that specializes in big data and machine learning solutions. MLflow is "
+            "designed to address the challenges that data scientists and machine learning "
+            "engineers face when developing, training, and deploying machine learning "
+            "models."
+        },
+    )
+
+    default_examples = [example_score_2, example_score_4]
+
+
+@dataclass
+class FaithfulnessMetric:
+    definition = (
+        "Faithfulness is only evaluated with the provided output and provided context, please "
+        "ignore the provided input entirely when scoring faithfulness. Faithfulness assesses "
+        "how much of the provided output is factually consistent with the provided context. A "
+        "higher score indicates that a higher proportion of claims present in the output can be "
+        "derived from the provided context. Faithfulness does not consider how much extra "
+        "information from the context is not present in the output."
+    )
+
+    grading_prompt = (
+        "Faithfulness: Below are the details for different scores:\n"
+        "- Score 1: None of the claims in the output can be inferred from the provided context.\n"
+        "- Score 2: Some of the claims in the output can be inferred from the provided context, "
+        "but the majority of the output is missing from, inconsistent with, or contradictory to "
+        "the provided context.\n"
+        "- Score 3: Half or more of the claims in the output can be inferred from the provided "
+        "context.\n"
+        "- Score 4: Most of the claims in the output can be inferred from the provided context, "
+        "with very little information that is not directly supported by the provided context.\n"
+        "- Score 5: All of the claims in the output are directly supported by the provided "
+        "context, demonstrating high faithfulness to the provided context."
+    )
+
+    grading_context_columns = ["context"]
+    parameters = default_parameters
+    default_model = default_model
+
+    example_score_2 = EvaluationExample(
+        input="How is MLflow related to Databricks?",
+        output="Databricks is a company that specializes in big data and machine learning "
+        "solutions. MLflow has nothing to do with Databricks. MLflow is an open-source platform "
+        "for managing the end-to-end machine learning (ML) lifecycle.",
+        score=2,
+        justification='The output claims that "MLflow has nothing to do with Databricks" which is '
+        'contradictory to the provided context that states "It was developed by Databricks". This '
+        'is a major inconsistency. However, the output correctly identifies that "MLflow is an '
+        'open-source platform for managing the end-to-end machine learning (ML) lifecycle" and '
+        '"Databricks is a company that specializes in big data and machine learning solutions", '
+        "which are both supported by the context. Therefore, some of the claims in the output can "
+        "be inferred from the provided context, but the majority of the output is inconsistent "
+        "with the provided context, leading to a faithfulness score of 2.",
+        grading_context={
+            "context": "MLflow is an open-source platform for managing the end-to-end machine "
+            "learning (ML) lifecycle. It was developed by Databricks, a company that specializes "
+            "in big data and machine learning solutions. MLflow is designed to address the "
+            "challenges that data scientists and machine learning engineers face when developing, "
+            "training, and deploying machine learning models."
+        },
+    )
+
+    example_score_5 = EvaluationExample(
+        input="How is MLflow related to Databricks?",
+        output="Databricks is a company that specializes in big data and machine learning "
+        "solutions.",
+        score=5,
+        justification='The output states that "Databricks is a company that specializes in big data'
+        ' and machine learning solutions." This claim is directly supported by the context, which '
+        'states "It was developed by Databricks, a company that specializes in big data and '
+        'machine learning solutions." Therefore, the faithfulness score is 5 as all the claims in '
+        'the output are directly supported by the provided context."',
+        grading_context={
+            "context": "MLflow is an open-source platform for managing the end-to-end "
+            "machine learning (ML) lifecycle. It was developed by Databricks, a company "
+            "that specializes in big data and machine learning solutions. MLflow is "
+            "designed to address the challenges that data scientists and machine learning "
+            "engineers face when developing, training, and deploying machine learning "
+            "models."
+        },
+    )
+
+    default_examples = [example_score_2, example_score_5]
+
+
+@dataclass
+class AnswerCorrectnessMetric:
+    definition = (
+        "Answer correctness is evaluated on the accuracy of the provided output based on the "
+        "provided targets, which is the ground truth. Scores can be assigned based on the degree "
+        "of semantic similarity and factual correctness of the provided output to the provided "
+        "targets, where a higher score indicates higher degree of accuracy."
+    )
+
+    grading_prompt = (
+        "Answer Correctness: Below are the details for different scores:\n"
+        "- Score 1: The output is completely incorrect. It is completely different from or "
+        "contradicts the provided targets.\n"
+        "- Score 2: The output demonstrates some degree of semantic similarity and includes "
+        "partially correct information. However, the output still has significant discrepancies "
+        "with the provided targets or inaccuracies.\n"
+        "- Score 3: The output addresses a couple of aspects of the input accurately, aligning "
+        "with the provided targets. However, there are still omissions or minor inaccuracies.\n"
+        "- Score 4: The output is mostly correct. It provides mostly accurate information, but "
+        "there may be one or more minor omissions or inaccuracies.\n"
+        "- Score 5: The output is correct. It demonstrates a high degree of accuracy and "
+        "semantic similarity to the targets."
+    )
+
+    grading_context_columns = ["targets"]
+    parameters = default_parameters
+    default_model = default_model
+
+    example_score_2 = EvaluationExample(
+        input="How is MLflow related to Databricks?",
+        output="Databricks is a data engineering and analytics platform designed to help "
+        "organizations process and analyze large amounts of data. Databricks is a company "
+        "specializing in big data and machine learning solutions.",
+        score=2,
+        justification="The output provided by the model does demonstrate some degree of semantic "
+        "similarity to the targets, as it correctly identifies Databricks as a company "
+        "specializing in big data and machine learning solutions. However, it fails to address "
+        "the main point of the input question, which is the relationship between MLflow and "
+        "Databricks. The output does not mention MLflow at all, which is a significant discrepancy "
+        "with the provided targets. Therefore, the model's answer_correctness score is 2.",
+        grading_context={
+            "targets": "MLflow is an open-source platform for managing the end-to-end machine "
+            "learning (ML) lifecycle. It was developed by Databricks, a company that specializes "
+            "in big data and machine learning solutions. MLflow is designed to address the "
+            "challenges that data scientists and machine learning engineers face when developing, "
+            "training, and deploying machine learning models."
+        },
+    )
+
+    example_score_4 = EvaluationExample(
+        input="How is MLflow related to Databricks?",
+        output="MLflow is a product created by Databricks to enhance the efficiency of machine "
+        "learning processes.",
+        score=4,
+        justification="The output provided by the model is mostly correct. It correctly identifies "
+        "that MLflow is a product created by Databricks. However, it does not mention that MLflow "
+        "is an open-source platform for managing the end-to-end machine learning lifecycle, which "
+        "is a significant part of its function. Therefore, while the output is mostly accurate, "
+        "it has a minor omission, which is why it gets a score of 4 according to the grading "
+        "rubric.",
+        grading_context={
+            "targets": "MLflow is an open-source platform for managing the end-to-end machine "
+            "learning (ML) lifecycle. It was developed by Databricks, a company that specializes "
+            "in big data and machine learning solutions. MLflow is designed to address the "
+            "challenges that data scientists and machine learning engineers face when developing, "
+            "training, and deploying machine learning models."
+        },
+    )
+
+    default_examples = [example_score_2, example_score_4]
+
+
+@dataclass
+class AnswerRelevanceMetric:
+    definition = (
+        "Answer relevance measures the appropriateness and applicability of the output with "
+        "respect to the input. Scores should reflect the extent to which the output directly "
+        "addresses the question provided in the input, and give lower scores for incomplete or "
+        "redundant output."
+    )
+
+    grading_prompt = (
+        "Answer relevance: Please give a score from 1-5 based on the degree of relevance to the "
+        "input, where the lowest and highest scores are defined as follows:"
+        "- Score 1: The output doesn't mention anything about the question or is completely "
+        "irrelevant to the input.\n"
+        "- Score 5: The output addresses all aspects of the question and all parts of the output "
+        "are meaningful and relevant to the question."
+    )
+
+    parameters = default_parameters
+    default_model = default_model
+
+    example_score_2 = EvaluationExample(
+        input="How is MLflow related to Databricks?",
+        output="Databricks is a company that specializes in big data and machine learning "
+        "solutions.",
+        score=2,
+        justification="The output provided by the model does give some information about "
+        "Databricks, which is part of the input question. However, it does not address the main "
+        "point of the question, which is the relationship between MLflow and Databricks. "
+        "Therefore, while the output is not completely irrelevant, it does not fully answer the "
+        "question, leading to a lower score.",
+    )
+
+    example_score_5 = EvaluationExample(
+        input="How is MLflow related to Databricks?",
+        output="MLflow is a product created by Databricks to enhance the efficiency of machine "
+        "learning processes.",
+        score=5,
+        justification="The output directly addresses the input question by explaining the "
+        "relationship between MLflow and Databricks. It provides a clear and concise answer that "
+        "MLflow is a product created by Databricks, and also adds relevant information about the "
+        "purpose of MLflow, which is to enhance the efficiency of machine learning processes. "
+        "Therefore, the output is highly relevant to the input and deserves a full score.",
+    )
+
+    default_examples = [example_score_2, example_score_5]
+
+
+@dataclass
+class RelevanceMetric:
+    definition = (
+        "Relevance encompasses the appropriateness, significance, and applicability of the output "
+        "with respect to both the input and context. Scores should reflect the extent to which the "
+        "output directly addresses the question provided in the input, given the provided context."
+    )
+
+    grading_prompt = (
+        "Relevance: Below are the details for different scores:"
+        "- Score 1: The output doesn't mention anything about the question or is completely "
+        "irrelevant to the provided context.\n"
+        "- Score 2: The output provides some relevance to the question and is somehow related "
+        "to the provided context.\n"
+        "- Score 3: The output mostly answers the question and is largely consistent with the "
+        "provided context.\n"
+        "- Score 4: The output answers the question and is consistent with the provided context.\n"
+        "- Score 5: The output answers the question comprehensively using the provided context."
+    )
+
+    grading_context_columns = ["context"]
+    parameters = default_parameters
+    default_model = default_model
+
+    example_score_2 = EvaluationExample(
+        input="How is MLflow related to Databricks?",
+        output="Databricks is a data engineering and analytics platform designed to help "
+        "organizations process and analyze large amounts of data. Databricks is a company "
+        "specializing in big data and machine learning solutions.",
+        score=2,
+        justification="The output provides relevant information about Databricks, mentioning it "
+        "as a company specializing in big data and machine learning solutions. However, it doesn't "
+        "directly address how MLflow is related to Databricks, which is the specific question "
+        "asked in the input. Therefore, the output is only somewhat related to the provided "
+        "context.",
+        grading_context={
+            "context": "MLflow is an open-source platform for managing the end-to-end machine "
+            "learning (ML) lifecycle. It was developed by Databricks, a company that specializes "
+            "in big data and machine learning solutions. MLflow is designed to address the "
+            "challenges that data scientists and machine learning engineers face when developing, "
+            "training, and deploying machine learning models."
+        },
+    )
+
+    example_score_4 = EvaluationExample(
+        input="How is MLflow related to Databricks?",
+        output="MLflow is a product created by Databricks to enhance the efficiency of machine "
+        "learning processes.",
+        score=4,
+        justification="The output provides a relevant and accurate statement about the "
+        "relationship between MLflow and Databricks. While it doesn't provide extensive detail, "
+        "it still offers a substantial and meaningful response. To achieve a score of 5, the "
+        "response could be further improved by providing additional context or details about "
+        "how MLflow specifically functions within the Databricks ecosystem.",
+        grading_context={
+            "context": "MLflow is an open-source platform for managing the end-to-end machine "
+            "learning (ML) lifecycle. It was developed by Databricks, a company that specializes "
+            "in big data and machine learning solutions. MLflow is designed to address the "
+            "challenges that data scientists and machine learning engineers face when developing, "
+            "training, and deploying machine learning models."
+        },
+    )
+
+    default_examples = [example_score_2, example_score_4]

venv/lib/python3.9/site-packages/mlflow/metrics/genai/utils.py

@@ -0,0 +1,6 @@
+def _get_latest_metric_version():
+    return "v1"
+
+
+def _get_default_model():
+    return "openai:/gpt-4"

venv/lib/python3.9/site-packages/mlflow/metrics/metric_definitions.py

@@ -0,0 +1,615 @@
+import functools
+import logging
+import os
+import subprocess
+import tempfile
+from pathlib import Path
+
+import numpy as np
+
+from mlflow.environment_variables import _MLFLOW_TESTING
+from mlflow.metrics.base import MetricValue, standard_aggregations
+
+_logger = logging.getLogger(__name__)
+
+
+# used to silently fail with invalid metric params
+def noop(*args, **kwargs):
+    return None
+
+
+targets_col_specifier = "the column specified by the `targets` parameter"
+predictions_col_specifier = (
+    "the column specified by the `predictions` parameter or the model output column"
+)
+
+
+def _validate_text_data(data, metric_name, col_specifier):
+    """Validates that the data is a list of strs and is non-empty"""
+    if data is None or len(data) == 0:
+        _logger.warning(
+            f"Cannot calculate {metric_name} for empty inputs: "
+            f"{col_specifier} is empty or the parameter is not specified. Skipping metric logging."
+        )
+        return False
+
+    for row, line in enumerate(data):
+        if not isinstance(line, str):
+            _logger.warning(
+                f"Cannot calculate {metric_name} for non-string inputs. "
+                f"Non-string found for {col_specifier} on row {row}. Skipping metric logging."
+            )
+            return False
+
+    return True
+
+
+def _validate_array_like_id_data(data, metric_name, col_specifier):
+    """Validates that the data is a list of lists/np.ndarrays of strings/ints and is non-empty"""
+    if data is None or len(data) == 0:
+        return False
+
+    for index, value in data.items():
+        if not (
+            (isinstance(value, list) and all(isinstance(val, (str, int)) for val in value))
+            or (
+                isinstance(value, np.ndarray)
+                and (np.issubdtype(value.dtype, str) or np.issubdtype(value.dtype, int))
+            )
+        ):
+            _logger.warning(
+                f"Cannot calculate metric '{metric_name}' for non-arraylike of string or int "
+                f"inputs. Non-arraylike of strings/ints found for {col_specifier} on row "
+                f"{index}, value {value}. Skipping metric logging."
+            )
+            return False
+
+    return True
+
+
+def _token_count_eval_fn(predictions, targets=None, metrics=None):
+    import tiktoken
+
+    # ref: https://github.com/openai/tiktoken/issues/75
+    os.environ["TIKTOKEN_CACHE_DIR"] = ""
+    encoding = tiktoken.get_encoding("cl100k_base")
+
+    num_tokens = []
+    for prediction in predictions:
+        if isinstance(prediction, str):
+            num_tokens.append(len(encoding.encode(prediction)))
+        else:
+            num_tokens.append(None)
+
+    return MetricValue(
+        scores=num_tokens,
+        aggregate_results={},
+    )
+
+
+def _load_from_github(path: str, module_type: str = "metric"):
+    import evaluate
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        tmpdir = Path(tmpdir)
+        subprocess.check_call(
+            [
+                "git",
+                "clone",
+                "--filter=blob:none",
+                "--no-checkout",
+                "https://github.com/huggingface/evaluate.git",
+                tmpdir,
+            ]
+        )
+        path = f"{module_type}s/{path}"
+        subprocess.check_call(["git", "sparse-checkout", "set", path], cwd=tmpdir)
+        subprocess.check_call(["git", "checkout"], cwd=tmpdir)
+        return evaluate.load(str(tmpdir / path))
+
+
+@functools.lru_cache(maxsize=8)
+def _cached_evaluate_load(path: str, module_type: str = "metric"):
+    import evaluate
+
+    try:
+        return evaluate.load(path, module_type=module_type)
+    except FileNotFoundError:
+        if _MLFLOW_TESTING.get():
+            # `evaluate.load` is highly unstable and often fails due to a network error or
+            # huggingface hub being down. In testing, we want to avoid this instability, so we
+            # load the metric from the evaluate repository on GitHub.
+            return _load_from_github(path, module_type=module_type)
+        raise
+
+
+def _toxicity_eval_fn(predictions, targets=None, metrics=None):
+    if not _validate_text_data(predictions, "toxicity", predictions_col_specifier):
+        return
+    try:
+        toxicity = _cached_evaluate_load("toxicity", module_type="measurement")
+    except Exception as e:
+        _logger.warning(
+            f"Failed to load 'toxicity' metric (error: {e!r}), skipping metric logging."
+        )
+        return
+
+    scores = toxicity.compute(predictions=predictions)["toxicity"]
+    toxicity_ratio = toxicity.compute(predictions=predictions, aggregation="ratio")[
+        "toxicity_ratio"
+    ]
+    return MetricValue(
+        scores=scores,
+        aggregate_results={
+            **standard_aggregations(scores),
+            "ratio": toxicity_ratio,
+        },
+    )
+
+
+def _flesch_kincaid_eval_fn(predictions, targets=None, metrics=None):
+    if not _validate_text_data(predictions, "flesch_kincaid", predictions_col_specifier):
+        return
+
+    try:
+        import textstat
+    except ImportError:
+        _logger.warning(
+            "Failed to import textstat for flesch kincaid metric, skipping metric logging. "
+            "Please install textstat using 'pip install textstat'."
+        )
+        return
+
+    scores = [textstat.flesch_kincaid_grade(prediction) for prediction in predictions]
+    return MetricValue(
+        scores=scores,
+        aggregate_results=standard_aggregations(scores),
+    )
+
+
+def _ari_eval_fn(predictions, targets=None, metrics=None):
+    if not _validate_text_data(predictions, "ari", predictions_col_specifier):
+        return
+
+    try:
+        import textstat
+    except ImportError:
+        _logger.warning(
+            "Failed to import textstat for automated readability index metric, "
+            "skipping metric logging. "
+            "Please install textstat using 'pip install textstat'."
+        )
+        return
+
+    scores = [textstat.automated_readability_index(prediction) for prediction in predictions]
+    return MetricValue(
+        scores=scores,
+        aggregate_results=standard_aggregations(scores),
+    )
+
+
+def _accuracy_eval_fn(predictions, targets=None, metrics=None, sample_weight=None):
+    if targets is not None and len(targets) != 0:
+        from sklearn.metrics import accuracy_score
+
+        acc = accuracy_score(y_true=targets, y_pred=predictions, sample_weight=sample_weight)
+        return MetricValue(aggregate_results={"exact_match": acc})
+
+
+def _rouge1_eval_fn(predictions, targets=None, metrics=None):
+    if not _validate_text_data(targets, "rouge1", targets_col_specifier) or not _validate_text_data(
+        predictions, "rouge1", predictions_col_specifier
+    ):
+        return
+
+    try:
+        rouge = _cached_evaluate_load("rouge")
+    except Exception as e:
+        _logger.warning(f"Failed to load 'rouge' metric (error: {e!r}), skipping metric logging.")
+        return
+
+    scores = rouge.compute(
+        predictions=predictions,
+        references=targets,
+        rouge_types=["rouge1"],
+        use_aggregator=False,
+    )["rouge1"]
+    return MetricValue(
+        scores=scores,
+        aggregate_results=standard_aggregations(scores),
+    )
+
+
+def _rouge2_eval_fn(predictions, targets=None, metrics=None):
+    if not _validate_text_data(targets, "rouge2", targets_col_specifier) or not _validate_text_data(
+        predictions, "rouge2", predictions_col_specifier
+    ):
+        return
+
+    try:
+        rouge = _cached_evaluate_load("rouge")
+    except Exception as e:
+        _logger.warning(f"Failed to load 'rouge' metric (error: {e!r}), skipping metric logging.")
+        return
+
+    scores = rouge.compute(
+        predictions=predictions,
+        references=targets,
+        rouge_types=["rouge2"],
+        use_aggregator=False,
+    )["rouge2"]
+    return MetricValue(
+        scores=scores,
+        aggregate_results=standard_aggregations(scores),
+    )
+
+
+def _rougeL_eval_fn(predictions, targets=None, metrics=None):
+    if not _validate_text_data(targets, "rougeL", targets_col_specifier) or not _validate_text_data(
+        predictions, "rougeL", predictions_col_specifier
+    ):
+        return
+
+    try:
+        rouge = _cached_evaluate_load("rouge")
+    except Exception as e:
+        _logger.warning(f"Failed to load 'rouge' metric (error: {e!r}), skipping metric logging.")
+        return
+
+    scores = rouge.compute(
+        predictions=predictions,
+        references=targets,
+        rouge_types=["rougeL"],
+        use_aggregator=False,
+    )["rougeL"]
+    return MetricValue(
+        scores=scores,
+        aggregate_results=standard_aggregations(scores),
+    )
+
+
+def _rougeLsum_eval_fn(predictions, targets=None, metrics=None):
+    if not _validate_text_data(
+        targets, "rougeLsum", targets_col_specifier
+    ) or not _validate_text_data(predictions, "rougeLsum", predictions_col_specifier):
+        return
+
+    try:
+        rouge = _cached_evaluate_load("rouge")
+    except Exception as e:
+        _logger.warning(f"Failed to load 'rouge' metric (error: {e!r}), skipping metric logging.")
+        return
+
+    scores = rouge.compute(
+        predictions=predictions,
+        references=targets,
+        rouge_types=["rougeLsum"],
+        use_aggregator=False,
+    )["rougeLsum"]
+    return MetricValue(
+        scores=scores,
+        aggregate_results=standard_aggregations(scores),
+    )
+
+
+def _mae_eval_fn(predictions, targets=None, metrics=None, sample_weight=None):
+    if targets is not None and len(targets) != 0:
+        from sklearn.metrics import mean_absolute_error
+
+        mae = mean_absolute_error(targets, predictions, sample_weight=sample_weight)
+        return MetricValue(aggregate_results={"mean_absolute_error": mae})
+
+
+def _mse_eval_fn(predictions, targets=None, metrics=None, sample_weight=None):
+    if targets is not None and len(targets) != 0:
+        from sklearn.metrics import mean_squared_error
+
+        mse = mean_squared_error(targets, predictions, sample_weight=sample_weight)
+        return MetricValue(aggregate_results={"mean_squared_error": mse})
+
+
+def _root_mean_squared_error(*, y_true, y_pred, sample_weight):
+    try:
+        from sklearn.metrics import root_mean_squared_error
+    except ImportError:
+        # If root_mean_squared_error is unavailable, fall back to
+        # `mean_squared_error(..., squared=False)`, which is deprecated in scikit-learn >= 1.4.
+        from sklearn.metrics import mean_squared_error
+
+        return mean_squared_error(
+            y_true=y_true, y_pred=y_pred, sample_weight=sample_weight, squared=False
+        )
+    else:
+        return root_mean_squared_error(y_true=y_true, y_pred=y_pred, sample_weight=sample_weight)
+
+
+def _rmse_eval_fn(predictions, targets=None, metrics=None, sample_weight=None):
+    if targets is not None and len(targets) != 0:
+        rmse = _root_mean_squared_error(
+            y_true=targets, y_pred=predictions, sample_weight=sample_weight
+        )
+        return MetricValue(aggregate_results={"root_mean_squared_error": rmse})
+
+
+def _r2_score_eval_fn(predictions, targets=None, metrics=None, sample_weight=None):
+    if targets is not None and len(targets) != 0:
+        from sklearn.metrics import r2_score
+
+        r2 = r2_score(targets, predictions, sample_weight=sample_weight)
+        return MetricValue(aggregate_results={"r2_score": r2})
+
+
+def _max_error_eval_fn(predictions, targets=None, metrics=None):
+    if targets is not None and len(targets) != 0:
+        from sklearn.metrics import max_error
+
+        error = max_error(targets, predictions)
+        return MetricValue(aggregate_results={"max_error": error})
+
+
+def _mape_eval_fn(predictions, targets=None, metrics=None, sample_weight=None):
+    if targets is not None and len(targets) != 0:
+        from sklearn.metrics import mean_absolute_percentage_error
+
+        mape = mean_absolute_percentage_error(targets, predictions, sample_weight=sample_weight)
+        return MetricValue(aggregate_results={"mean_absolute_percentage_error": mape})
+
+
+def _recall_eval_fn(
+    predictions, targets=None, metrics=None, pos_label=1, average="binary", sample_weight=None
+):
+    if targets is not None and len(targets) != 0:
+        from sklearn.metrics import recall_score
+
+        recall = recall_score(
+            targets, predictions, pos_label=pos_label, average=average, sample_weight=sample_weight
+        )
+        return MetricValue(aggregate_results={"recall_score": recall})
+
+
+def _precision_eval_fn(
+    predictions, targets=None, metrics=None, pos_label=1, average="binary", sample_weight=None
+):
+    if targets is not None and len(targets) != 0:
+        from sklearn.metrics import precision_score
+
+        precision = precision_score(
+            targets,
+            predictions,
+            pos_label=pos_label,
+            average=average,
+            sample_weight=sample_weight,
+        )
+        return MetricValue(aggregate_results={"precision_score": precision})
+
+
+def _f1_score_eval_fn(
+    predictions, targets=None, metrics=None, pos_label=1, average="binary", sample_weight=None
+):
+    if targets is not None and len(targets) != 0:
+        from sklearn.metrics import f1_score
+
+        f1 = f1_score(
+            targets,
+            predictions,
+            pos_label=pos_label,
+            average=average,
+            sample_weight=sample_weight,
+        )
+        return MetricValue(aggregate_results={"f1_score": f1})
+
+
+def _precision_at_k_eval_fn(k):
+    if not (isinstance(k, int) and k > 0):
+        _logger.warning(
+            f"Cannot calculate 'precision_at_k' for invalid parameter 'k'. "
+            f"'k' should be a positive integer; found: {k}. Skipping metric logging."
+        )
+        return noop
+
+    def _fn(predictions, targets):
+        if not _validate_array_like_id_data(
+            predictions, "precision_at_k", predictions_col_specifier
+        ) or not _validate_array_like_id_data(targets, "precision_at_k", targets_col_specifier):
+            return
+
+        scores = []
+        for target, prediction in zip(targets, predictions):
+            # only include the top k retrieved chunks
+            ground_truth, retrieved = set(target), prediction[:k]
+            relevant_doc_count = sum(1 for doc in retrieved if doc in ground_truth)
+            if len(retrieved) > 0:
+                scores.append(relevant_doc_count / len(retrieved))
+            else:
+                # when no documents are retrieved, precision is 0
+                scores.append(0)
+
+        return MetricValue(scores=scores, aggregate_results=standard_aggregations(scores))
+
+    return _fn
+
+
+def _expand_duplicate_retrieved_docs(predictions, targets):
+    counter = {}
+    expanded_predictions = []
+    expanded_targets = targets
+    for doc_id in predictions:
+        if doc_id not in counter:
+            counter[doc_id] = 1
+            expanded_predictions.append(doc_id)
+        else:
+            counter[doc_id] += 1
+            new_doc_id = (
+                f"{doc_id}_bc574ae_{counter[doc_id]}"  # adding a random string to avoid collisions
+            )
+            expanded_predictions.append(new_doc_id)
+            if doc_id in expanded_targets:
+                expanded_targets.add(new_doc_id)
+    return expanded_predictions, expanded_targets
+
+
+def _prepare_row_for_ndcg(predictions, targets):
+    """Prepare data one row from predictions and targets to y_score, y_true for ndcg calculation.
+
+    Args:
+        predictions: A list of strings of at most k doc IDs retrieved.
+        targets: A list of strings of ground-truth doc IDs.
+
+    Returns:
+        y_true : ndarray of shape (1, n_docs) Representing the ground-truth relevant docs.
+            n_docs is the number of unique docs in union of predictions and targets.
+        y_score : ndarray of shape (1, n_docs) Representing the retrieved docs.
+            n_docs is the number of unique docs in union of predictions and targets.
+    """
+    # sklearn does an internal sort of y_score, so to preserve the order of our retrieved
+    # docs, we need to modify the relevance value slightly
+    eps = 1e-6
+
+    # support predictions containing duplicate doc ID
+    targets = set(targets)
+    predictions, targets = _expand_duplicate_retrieved_docs(predictions, targets)
+
+    all_docs = targets.union(predictions)
+    doc_id_to_index = {doc_id: i for i, doc_id in enumerate(all_docs)}
+    n_labels = max(len(doc_id_to_index), 2)  # sklearn.metrics.ndcg_score requires at least 2 labels
+    y_true = np.zeros((1, n_labels), dtype=np.float32)
+    y_score = np.zeros((1, n_labels), dtype=np.float32)
+    for i, doc_id in enumerate(predictions):
+        # "1 - i * eps" means we assign higher score to docs that are ranked higher,
+        # but all scores are still approximately 1.
+        y_score[0, doc_id_to_index[doc_id]] = 1 - i * eps
+    for doc_id in targets:
+        y_true[0, doc_id_to_index[doc_id]] = 1
+    return y_score, y_true
+
+
+def _ndcg_at_k_eval_fn(k):
+    if not (isinstance(k, int) and k > 0):
+        _logger.warning(
+            f"Cannot calculate 'ndcg_at_k' for invalid parameter 'k'. "
+            f"'k' should be a positive integer; found: {k}. Skipping metric logging."
+        )
+        return noop
+
+    def _fn(predictions, targets):
+        from sklearn.metrics import ndcg_score
+
+        if not _validate_array_like_id_data(
+            predictions, "ndcg_at_k", predictions_col_specifier
+        ) or not _validate_array_like_id_data(targets, "ndcg_at_k", targets_col_specifier):
+            return
+
+        scores = []
+        for ground_truth, retrieved in zip(targets, predictions):
+            # 1. If no ground truth doc IDs are provided and no documents are retrieved,
+            # the score is 1.
+            if len(retrieved) == 0 and len(ground_truth) == 0:
+                scores.append(1)  # no error is made
+                continue
+            # 2. If no ground truth doc IDs are provided and documents are retrieved,
+            # the score is 0.
+            # 3. If ground truth doc IDs are provided and no documents are retrieved,
+            # the score is 0.
+            if len(retrieved) == 0 or len(ground_truth) == 0:
+                scores.append(0)
+                continue
+
+            # only include the top k retrieved chunks
+            y_score, y_true = _prepare_row_for_ndcg(retrieved[:k], ground_truth)
+            score = ndcg_score(y_true, y_score, k=len(retrieved[:k]), ignore_ties=True)
+            scores.append(score)
+
+        return MetricValue(scores=scores, aggregate_results=standard_aggregations(scores))
+
+    return _fn
+
+
+def _recall_at_k_eval_fn(k):
+    if not (isinstance(k, int) and k > 0):
+        _logger.warning(
+            f"Cannot calculate 'recall_at_k' for invalid parameter 'k'. "
+            f"'k' should be a positive integer; found: {k}. Skipping metric logging."
+        )
+        return noop
+
+    def _fn(predictions, targets):
+        if not _validate_array_like_id_data(
+            predictions, "recall_at_k", predictions_col_specifier
+        ) or not _validate_array_like_id_data(targets, "recall_at_k", targets_col_specifier):
+            return
+
+        scores = []
+        for target, prediction in zip(targets, predictions):
+            # only include the top k retrieved chunks
+            ground_truth, retrieved = set(target), set(prediction[:k])
+            relevant_doc_count = len(ground_truth.intersection(retrieved))
+            if len(ground_truth) > 0:
+                scores.append(relevant_doc_count / len(ground_truth))
+            elif len(retrieved) == 0:
+                # there are 0 retrieved and ground truth docs, so reward for the match
+                scores.append(1)
+            else:
+                # there are > 0 retrieved, but 0 ground truth, so penalize
+                scores.append(0)
+
+        return MetricValue(scores=scores, aggregate_results=standard_aggregations(scores))
+
+    return _fn
+
+
+def _bleu_eval_fn(predictions, targets=None, metrics=None):
+    # Validate input data
+    if not _validate_text_data(targets, "bleu", targets_col_specifier):
+        _logger.error(
+            """Target validation failed.
+            Ensure targets are valid for BLEU computation."""
+        )
+        return
+    if not _validate_text_data(predictions, "bleu", predictions_col_specifier):
+        _logger.error(
+            """Prediction validation failed.
+            Ensure predictions are valid for BLEU computation."""
+        )
+        return
+
+    # Load BLEU metric
+    try:
+        bleu = _cached_evaluate_load("bleu")
+    except Exception as e:
+        _logger.warning(f"Failed to load 'bleu' metric (error: {e!r}), skipping metric logging.")
+        return
+
+    # Calculate BLEU scores for each prediction-target pair
+    result = []
+    invalid_indices = []
+
+    for i, (prediction, target) in enumerate(zip(predictions, targets)):
+        if len(target) == 0 or len(prediction) == 0:
+            invalid_indices.append(i)
+            result.append(0)  # Append 0 as a placeholder for invalid entries
+            continue
+
+        try:
+            score = bleu.compute(predictions=[prediction], references=[[target]])
+            result.append(score["bleu"])
+        except Exception as e:
+            _logger.warning(f"Failed to calculate BLEU for row {i} (error: {e!r}). Skipping.")
+            result.append(0)  # Append 0 for consistency if an unexpected error occurs
+
+    # Log warning for any invalid indices
+    if invalid_indices:
+        _logger.warning(
+            f"BLEU score calculation skipped for the following indices "
+            f"due to empty target or prediction: {invalid_indices}. "
+            f"A score of 0 was appended for these entries."
+        )
+
+    # Return results
+    if not result:
+        _logger.warning("No BLEU scores were calculated due to input errors.")
+        return
+
+    return MetricValue(
+        scores=result,
+        aggregate_results=standard_aggregations(result),
+    )