star

venv/lib/python3.9/site-packages/mlflow/metrics/genai/__init__.py

@@ -0,0 +1,25 @@
+from mlflow.metrics.genai.base import EvaluationExample
+from mlflow.metrics.genai.genai_metric import (
+    make_genai_metric,
+    make_genai_metric_from_prompt,
+    retrieve_custom_metrics,
+)
+from mlflow.metrics.genai.metric_definitions import (
+    answer_correctness,
+    answer_relevance,
+    answer_similarity,
+    faithfulness,
+    relevance,
+)
+
+__all__ = [
+    "EvaluationExample",
+    "make_genai_metric",
+    "make_genai_metric_from_prompt",
+    "answer_similarity",
+    "answer_correctness",
+    "faithfulness",
+    "answer_relevance",
+    "relevance",
+    "retrieve_custom_metrics",
+]

venv/lib/python3.9/site-packages/mlflow/metrics/genai/base.py

@@ -0,0 +1,103 @@
+from dataclasses import dataclass
+from typing import Optional, Union
+
+from mlflow.metrics.genai.prompt_template import PromptTemplate
+from mlflow.utils.annotations import experimental
+
+
+@experimental
+@dataclass
+class EvaluationExample:
+    """
+    Stores the sample example during few shot learning during LLM evaluation
+
+    Args:
+        input: The input provided to the model
+        output: The output generated by the model
+        score: The score given by the evaluator
+        justification: The justification given by the evaluator
+        grading_context: The grading_context provided to the evaluator for evaluation. Either
+            a dictionary of grading context column names and grading context strings
+            or a single grading context string.
+
+    .. code-block:: python
+        :caption: Example for creating an EvaluationExample
+
+        from mlflow.metrics.base import EvaluationExample
+
+        example = EvaluationExample(
+            input="What is MLflow?",
+            output="MLflow is an open-source platform for managing machine "
+            "learning workflows, including experiment tracking, model packaging, "
+            "versioning, and deployment, simplifying the ML lifecycle.",
+            score=4,
+            justification="The definition effectively explains what MLflow is "
+            "its purpose, and its developer. It could be more concise for a 5-score.",
+            grading_context={
+                "ground_truth": "MLflow is an open-source platform for managing "
+                "the end-to-end machine learning (ML) lifecycle. It was developed by Databricks, "
+                "a company that specializes in big data and machine learning solutions. MLflow is "
+                "designed to address the challenges that data scientists and machine learning "
+                "engineers face when developing, training, and deploying machine learning models."
+            },
+        )
+        print(str(example))
+
+    .. code-block:: text
+        :caption: Output
+
+        Input: What is MLflow?
+        Provided output: "MLflow is an open-source platform for managing machine "
+            "learning workflows, including experiment tracking, model packaging, "
+            "versioning, and deployment, simplifying the ML lifecycle."
+        Provided ground_truth: "MLflow is an open-source platform for managing "
+            "the end-to-end machine learning (ML) lifecycle. It was developed by Databricks, "
+            "a company that specializes in big data and machine learning solutions. MLflow is "
+            "designed to address the challenges that data scientists and machine learning "
+            "engineers face when developing, training, and deploying machine learning models."
+        Score: 4
+        Justification: "The definition effectively explains what MLflow is "
+            "its purpose, and its developer. It could be more concise for a 5-score."
+    """
+
+    output: str
+    score: float
+    justification: str
+    input: Optional[str] = None
+    grading_context: Optional[Union[dict[str, str], str]] = None
+
+    def _format_grading_context(self):
+        if isinstance(self.grading_context, dict):
+            return "\n".join(
+                [f"key: {key}\nvalue:\n{value}" for key, value in self.grading_context.items()]
+            )
+        else:
+            return self.grading_context
+
+    def __str__(self) -> str:
+        return PromptTemplate(
+            [
+                """
+Example Input:
+{input}
+""",
+                """
+Example Output:
+{output}
+""",
+                """
+Additional information used by the model:
+{grading_context}
+""",
+                """
+Example score: {score}
+Example justification: {justification}
+        """,
+            ]
+        ).format(
+            input=self.input,
+            output=self.output,
+            grading_context=self._format_grading_context(),
+            score=self.score,
+            justification=self.justification,
+        )

venv/lib/python3.9/site-packages/mlflow/metrics/genai/genai_metric.py

@@ -0,0 +1,768 @@
+import json
+import logging
+import re
+import warnings
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from inspect import Parameter, Signature
+from tempfile import TemporaryDirectory
+from typing import Any, Optional, Union
+
+import pandas as pd
+
+import mlflow
+from mlflow.exceptions import MlflowException
+from mlflow.metrics.base import MetricValue
+from mlflow.metrics.genai import model_utils
+from mlflow.metrics.genai.base import EvaluationExample
+from mlflow.metrics.genai.prompt_template import PromptTemplate
+from mlflow.metrics.genai.utils import _get_default_model, _get_latest_metric_version
+from mlflow.models import EvaluationMetric, make_metric
+from mlflow.models.evaluation.base import _make_metric
+from mlflow.protos.databricks_pb2 import (
+    BAD_REQUEST,
+    INTERNAL_ERROR,
+    INVALID_PARAMETER_VALUE,
+    UNAUTHENTICATED,
+    ErrorCode,
+)
+from mlflow.utils.annotations import experimental
+from mlflow.utils.class_utils import _get_class_from_string
+from mlflow.version import VERSION
+
+_logger = logging.getLogger(__name__)
+
+_GENAI_CUSTOM_METRICS_FILE_NAME = "genai_custom_metrics.json"
+_PROMPT_FORMATTING_WRAPPER = """
+
+You must return the following fields in your response in two lines, one below the other:
+score: Your numerical score based on the rubric
+justification: Your reasoning for giving this score
+
+Do not add additional new lines. Do not add any other fields."""
+
+
+def _format_args_string(grading_context_columns: Optional[list[str]], eval_values, indx) -> str:
+    import pandas as pd
+
+    args_dict = {}
+    for arg in grading_context_columns:
+        if arg in eval_values:
+            args_dict[arg] = (
+                eval_values[arg].iloc[indx]
+                if isinstance(eval_values[arg], pd.Series)
+                else eval_values[arg][indx]
+            )
+        else:
+            raise MlflowException(
+                f"{arg} does not exist in the eval function {list(eval_values.keys())}."
+            )
+
+    return (
+        ""
+        if args_dict is None or len(args_dict) == 0
+        else (
+            "Additional information used by the model:\n"
+            + "\n".join(
+                [f"key: {arg}\nvalue:\n{arg_value}" for arg, arg_value in args_dict.items()]
+            )
+        )
+    )
+
+
+# Function to extract Score and Justification
+def _extract_score_and_justification(text):
+    if text:
+        text = re.sub(r"score", "score", text, flags=re.IGNORECASE)
+        text = re.sub(r"justification", "justification", text, flags=re.IGNORECASE)
+        # Attempt to parse JSON
+        try:
+            data = json.loads(text)
+            score = int(data.get("score"))
+            justification = data.get("justification")
+        except json.JSONDecodeError:
+            # If parsing fails, use regex
+            if (match := re.search(r"score: (\d+),?\s*justification: (.+)", text)) or (
+                match := re.search(r"\s*score:\s*(\d+)\s*justification:\s*(.+)", text, re.DOTALL)
+            ):
+                score = int(match.group(1))
+                justification = match.group(2)
+            else:
+                score = None
+                justification = f"Failed to extract score and justification. Raw output: {text}"
+
+        if not isinstance(score, (int, float)) or not isinstance(justification, str):
+            return None, f"Failed to extract score and justification. Raw output: {text}"
+
+        return score, justification
+
+    return None, None
+
+
+def _score_model_on_one_payload(
+    payload: str,
+    eval_model: str,
+    parameters: Optional[dict[str, Any]],
+    extra_headers: Optional[dict[str, str]] = None,
+    proxy_url: Optional[str] = None,
+):
+    try:
+        # If the endpoint does not specify type, default to chat format
+        endpoint_type = model_utils.get_endpoint_type(eval_model) or "llm/v1/chat"
+        raw_result = model_utils.score_model_on_payload(
+            eval_model, payload, parameters, extra_headers, proxy_url, endpoint_type
+        )
+        return _extract_score_and_justification(raw_result)
+    except ImportError:
+        raise
+    except MlflowException as e:
+        if e.error_code in [
+            ErrorCode.Name(BAD_REQUEST),
+            ErrorCode.Name(UNAUTHENTICATED),
+            ErrorCode.Name(INVALID_PARAMETER_VALUE),
+        ]:
+            raise
+        else:
+            return None, f"Failed to score model on payload. Error: {e!s}"
+    except Exception as e:
+        return None, f"Failed to score model on payload. Error: {e!s}"
+
+
+def _score_model_on_payloads(
+    grading_payloads, model, parameters, headers, proxy_url, max_workers
+) -> tuple[list[int], list[str]]:
+    scores = [None] * len(grading_payloads)
+    justifications = [None] * len(grading_payloads)
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        futures = {
+            executor.submit(
+                _score_model_on_one_payload,
+                payload,
+                model,
+                parameters,
+                headers,
+                proxy_url,
+            ): indx
+            for indx, payload in enumerate(grading_payloads)
+        }
+
+        as_comp = as_completed(futures)
+        try:
+            from tqdm.auto import tqdm
+
+            as_comp = tqdm(as_comp, total=len(futures))
+        except ImportError:
+            pass
+
+        for future in as_comp:
+            indx = futures[future]
+            score, justification = future.result()
+            scores[indx] = score
+            justifications[indx] = justification
+
+    return scores, justifications
+
+
+def _get_aggregate_results(scores, aggregations):
+    # loop over the aggregations and compute the aggregate results on the scores
+    def aggregate_function(aggregate_option, scores):
+        import numpy as np
+
+        options = {
+            "min": np.min,
+            "max": np.max,
+            "mean": np.mean,
+            "median": np.median,
+            "variance": np.var,
+            "p90": lambda x: np.percentile(x, 90) if x else None,
+        }
+
+        if aggregate_option not in options:
+            raise MlflowException(
+                message=f"Invalid aggregate option {aggregate_option}.",
+                error_code=INVALID_PARAMETER_VALUE,
+            )
+
+        return options[aggregate_option](scores)
+
+    scores_for_aggregation = [score for score in scores if score is not None]
+
+    return (
+        {option: aggregate_function(option, scores_for_aggregation) for option in aggregations}
+        if aggregations is not None
+        else {}
+    )
+
+
+@experimental
+def make_genai_metric_from_prompt(
+    name: str,
+    judge_prompt: Optional[str] = None,
+    model: Optional[str] = _get_default_model(),
+    parameters: Optional[dict[str, Any]] = None,
+    aggregations: Optional[list[str]] = None,
+    greater_is_better: bool = True,
+    max_workers: int = 10,
+    metric_metadata: Optional[dict[str, Any]] = None,
+    extra_headers: Optional[dict[str, str]] = None,
+    proxy_url: Optional[str] = None,
+) -> EvaluationMetric:
+    """
+    Create a genai metric used to evaluate LLM using LLM as a judge in MLflow. This produces
+    a metric using only the supplied judge prompt without any pre-written system prompt.
+    This can be useful for use cases that are not covered by the full grading prompt in any
+    ``EvaluationModel`` version.
+
+    Args:
+        name: Name of the metric.
+        judge_prompt: The entire prompt to be used for the judge model.
+            The prompt will be minimally wrapped in formatting instructions to ensure
+            scores can be parsed. The prompt may use f-string formatting to include variables.
+            Corresponding variables must be passed as keyword arguments into the
+            resulting metric's eval function.
+        model: (Optional) Model uri of the judge model that will be used to compute the metric,
+            e.g., ``openai:/gpt-4``. Refer to the `LLM-as-a-Judge Metrics <https://mlflow.org/docs/latest/llms/llm-evaluate/index.html#selecting-the-llm-as-judge-model>`_
+            documentation for the supported model types and their URI format.
+        parameters: (Optional) Parameters for the LLM used to compute the metric. By default, we
+            set the temperature to 0.0, max_tokens to 200, and top_p to 1.0. We recommend
+            setting the temperature to 0.0 for the LLM used as a judge to ensure consistent results.
+        aggregations: (Optional) The list of options to aggregate the scores. Currently supported
+            options are: min, max, mean, median, variance, p90.
+        greater_is_better: (Optional) Whether the metric is better when it is greater.
+        max_workers: (Optional) The maximum number of workers to use for judge scoring.
+            Defaults to 10 workers.
+        metric_metadata: (Optional) Dictionary of metadata to be attached to the
+            EvaluationMetric object. Useful for model evaluators that require additional
+            information to determine how to evaluate this metric.
+        extra_headers: (Optional) Additional headers to be passed to the judge model.
+        proxy_url: (Optional) Proxy URL to be used for the judge model. This is useful when the
+            judge model is served via a proxy endpoint, not directly via LLM provider services.
+            If not specified, the default URL for the LLM provider will be used
+            (e.g., https://api.openai.com/v1/chat/completions for OpenAI chat models).
+
+    Returns:
+        A metric object.
+
+    .. code-block:: python
+        :test:
+        :caption: Example for creating a genai metric
+
+        from mlflow.metrics.genai import make_genai_metric_from_prompt
+
+        metric = make_genai_metric_from_prompt(
+            name="ease_of_understanding",
+            judge_prompt=(
+                "You must evaluate the output of a bot based on how easy it is to "
+                "understand its outputs."
+                "Evaluate the bot's output from the perspective of a layperson."
+                "The bot was provided with this input: {input} and this output: {output}."
+            ),
+            model="openai:/gpt-4",
+            parameters={"temperature": 0.0},
+            aggregations=["mean", "variance", "p90"],
+            greater_is_better=True,
+        )
+
+    """
+    import numpy as np
+
+    prompt_template = PromptTemplate([judge_prompt, _PROMPT_FORMATTING_WRAPPER])
+    allowed_variables = prompt_template.variables
+
+    # When users create a custom metric using this function,the metric configuration
+    # will be serialized and stored as an artifact. This enables us to later deserialize
+    # the configuration, allowing users to understand their LLM evaluation results more clearly.
+    genai_metric_args = {
+        "name": name,
+        "judge_prompt": judge_prompt,
+        "model": model,
+        "parameters": parameters,
+        "aggregations": aggregations,
+        "greater_is_better": greater_is_better,
+        "max_workers": max_workers,
+        "metric_metadata": metric_metadata,
+        # Record the mlflow version for serialization in case the function signature changes later
+        "mlflow_version": VERSION,
+        "fn_name": make_genai_metric_from_prompt.__name__,
+    }
+
+    aggregations = aggregations or ["mean", "variance", "p90"]
+
+    def eval_fn(
+        *args,
+        **kwargs,
+    ) -> MetricValue:
+        """
+        This is the function that is called when the metric is evaluated.
+        """
+        if missing_variables := allowed_variables - set(kwargs.keys()):
+            raise MlflowException(
+                message=f"Missing variable inputs to eval_fn: {missing_variables}",
+                error_code=INVALID_PARAMETER_VALUE,
+            )
+        kwargs = {k: [v] if np.isscalar(v) else v for k, v in kwargs.items()}
+        grading_payloads = pd.DataFrame(kwargs).to_dict(orient="records")
+        arg_strings = [prompt_template.format(**payload) for payload in grading_payloads]
+        scores, justifications = _score_model_on_payloads(
+            arg_strings, model, parameters, extra_headers, proxy_url, max_workers
+        )
+
+        aggregate_scores = _get_aggregate_results(scores, aggregations)
+
+        return MetricValue(scores, justifications, aggregate_scores)
+
+    if allowed_variables:
+        eval_fn.__signature__ = Signature(
+            parameters=[
+                Parameter(name=var, kind=Parameter.KEYWORD_ONLY) for var in allowed_variables
+            ]
+        )
+
+    return make_metric(
+        eval_fn=eval_fn,
+        greater_is_better=greater_is_better,
+        name=name,
+        metric_metadata=metric_metadata,
+        genai_metric_args=genai_metric_args,
+    )
+
+
+@experimental
+def make_genai_metric(
+    name: str,
+    definition: str,
+    grading_prompt: str,
+    examples: Optional[list[EvaluationExample]] = None,
+    version: Optional[str] = _get_latest_metric_version(),
+    model: Optional[str] = _get_default_model(),
+    grading_context_columns: Optional[Union[str, list[str]]] = None,
+    include_input: bool = True,
+    parameters: Optional[dict[str, Any]] = None,
+    aggregations: Optional[list[str]] = None,
+    greater_is_better: bool = True,
+    max_workers: int = 10,
+    metric_metadata: Optional[dict[str, Any]] = None,
+    extra_headers: Optional[dict[str, str]] = None,
+    proxy_url: Optional[str] = None,
+) -> EvaluationMetric:
+    """
+    Create a genai metric used to evaluate LLM using LLM as a judge in MLflow. The full grading
+    prompt is stored in the metric_details field of the ``EvaluationMetric`` object.
+
+    Args:
+        name: Name of the metric.
+        definition: Definition of the metric.
+        grading_prompt: Grading criteria of the metric.
+        examples: (Optional) Examples of the metric.
+        version: (Optional) Version of the metric. Currently supported versions are: v1.
+        model: (Optional) Model uri of the judge model that will be used to compute the metric,
+            e.g., ``openai:/gpt-4``. Refer to the `LLM-as-a-Judge Metrics <https://mlflow.org/docs/latest/llms/llm-evaluate/index.html#selecting-the-llm-as-judge-model>`_
+            documentation for the supported model types and their URI format.
+        grading_context_columns: (Optional) The name of the grading context column, or a list of
+            grading context column names, required to compute the metric. The
+            ``grading_context_columns`` are used by the LLM as a judge as additional information to
+            compute the metric. The columns are extracted from the input dataset or output
+            predictions based on ``col_mapping`` in the ``evaluator_config`` passed to
+            :py:func:`mlflow.evaluate()`. They can also be the name of other evaluated metrics.
+        include_input: (Optional) Whether to include the input
+            when computing the metric.
+        parameters: (Optional) Parameters for the LLM used to compute the metric. By default, we
+            set the temperature to 0.0, max_tokens to 200, and top_p to 1.0. We recommend
+            setting the temperature to 0.0 for the LLM used as a judge to ensure consistent results.
+        aggregations: (Optional) The list of options to aggregate the scores. Currently supported
+            options are: min, max, mean, median, variance, p90.
+        greater_is_better: (Optional) Whether the metric is better when it is greater.
+        max_workers: (Optional) The maximum number of workers to use for judge scoring.
+            Defaults to 10 workers.
+        metric_metadata: (Optional) Dictionary of metadata to be attached to the
+            EvaluationMetric object. Useful for model evaluators that require additional
+            information to determine how to evaluate this metric.
+        extra_headers: (Optional) Additional headers to be passed to the judge model.
+        proxy_url: (Optional) Proxy URL to be used for the judge model. This is useful when the
+            judge model is served via a proxy endpoint, not directly via LLM provider services.
+            If not specified, the default URL for the LLM provider will be used
+            (e.g., https://api.openai.com/v1/chat/completions for OpenAI chat models).
+
+
+    Returns:
+        A metric object.
+
+    .. code-block:: python
+        :test:
+        :caption: Example for creating a genai metric
+
+        from mlflow.metrics.genai import EvaluationExample, make_genai_metric
+
+        example = EvaluationExample(
+            input="What is MLflow?",
+            output=(
+                "MLflow is an open-source platform for managing machine "
+                "learning workflows, including experiment tracking, model packaging, "
+                "versioning, and deployment, simplifying the ML lifecycle."
+            ),
+            score=4,
+            justification=(
+                "The definition effectively explains what MLflow is "
+                "its purpose, and its developer. It could be more concise for a 5-score.",
+            ),
+            grading_context={
+                "targets": (
+                    "MLflow is an open-source platform for managing "
+                    "the end-to-end machine learning (ML) lifecycle. It was developed by "
+                    "Databricks, a company that specializes in big data and machine learning "
+                    "solutions. MLflow is designed to address the challenges that data "
+                    "scientists and machine learning engineers face when developing, training, "
+                    "and deploying machine learning models."
+                )
+            },
+        )
+        metric = make_genai_metric(
+            name="answer_correctness",
+            definition=(
+                "Answer correctness is evaluated on the accuracy of the provided output based on "
+                "the provided targets, which is the ground truth. Scores can be assigned based on "
+                "the degree of semantic similarity and factual correctness of the provided output "
+                "to the provided targets, where a higher score indicates higher degree of accuracy."
+            ),
+            grading_prompt=(
+                "Answer correctness: Below are the details for different scores:"
+                "- Score 1: The output is completely incorrect. It is completely different from "
+                "or contradicts the provided targets."
+                "- Score 2: The output demonstrates some degree of semantic similarity and "
+                "includes partially correct information. However, the output still has significant "
+                "discrepancies with the provided targets or inaccuracies."
+                "- Score 3: The output addresses a couple of aspects of the input accurately, "
+                "aligning with the provided targets. However, there are still omissions or minor "
+                "inaccuracies."
+                "- Score 4: The output is mostly correct. It provides mostly accurate information, "
+                "but there may be one or more minor omissions or inaccuracies."
+                "- Score 5: The output is correct. It demonstrates a high degree of accuracy and "
+                "semantic similarity to the targets."
+            ),
+            examples=[example],
+            version="v1",
+            model="openai:/gpt-4",
+            grading_context_columns=["targets"],
+            parameters={"temperature": 0.0},
+            aggregations=["mean", "variance", "p90"],
+            greater_is_better=True,
+        )
+    """
+    # When users create a custom metric using this function,the metric configuration
+    # will be serialized and stored as an artifact. This enables us to later deserialize
+    # the configuration, allowing users to understand their LLM evaluation results more clearly.
+    genai_metric_args = {
+        "name": name,
+        "definition": definition,
+        "grading_prompt": grading_prompt,
+        "examples": examples,
+        "version": version,
+        "model": model,
+        "grading_context_columns": grading_context_columns,
+        "include_input": include_input,
+        "parameters": parameters,
+        "aggregations": aggregations,
+        "greater_is_better": greater_is_better,
+        "max_workers": max_workers,
+        "metric_metadata": metric_metadata,
+        # Record the mlflow version for serialization in case the function signature changes later
+        "mlflow_version": VERSION,
+        "fn_name": make_genai_metric.__name__,
+    }
+
+    aggregations = aggregations or ["mean", "variance", "p90"]
+    grading_context_columns = grading_context_columns or []
+
+    if not isinstance(grading_context_columns, list):
+        grading_context_columns = [grading_context_columns]
+
+    def process_example(example):
+        if example.grading_context is None and len(grading_context_columns) == 0:
+            grading_context = {}
+        elif isinstance(example.grading_context, dict):
+            grading_context = example.grading_context
+        else:
+            # The grading context is string-like. Assume that it corresponds to the first
+            # grading context column and update the example accordingly
+            grading_context = {grading_context_columns[0]: example.grading_context}
+            example.grading_context = grading_context
+
+        if set(grading_context.keys()) != set(grading_context_columns):
+            raise MlflowException.invalid_parameter_value(
+                f"Example grading context does not contain required columns.\n"
+                f" Example grading context columns: {list(grading_context.keys())}\n"
+                f" Required grading context columns: {grading_context_columns}\n"
+            )
+
+        if not include_input:
+            return EvaluationExample(
+                output=example.output,
+                score=example.score,
+                justification=example.justification,
+                grading_context=example.grading_context,
+            )
+
+        return example
+
+    if examples is not None:
+        examples = [process_example(example) for example in examples]
+
+    class_name = f"mlflow.metrics.genai.prompts.{version}.EvaluationModel"
+    try:
+        evaluation_model_class_module = _get_class_from_string(class_name)
+    except ModuleNotFoundError:
+        raise MlflowException(
+            f"Failed to find evaluation model for version {version}."
+            f" Please check the correctness of the version",
+            error_code=INVALID_PARAMETER_VALUE,
+        ) from None
+    except Exception as e:
+        raise MlflowException(
+            f"Failed to construct evaluation model {version}. Error: {e!r}",
+            error_code=INTERNAL_ERROR,
+        ) from None
+
+    evaluation_context = evaluation_model_class_module(
+        name,
+        definition,
+        grading_prompt,
+        examples,
+        model,
+        *(parameters,) if parameters is not None else (),
+    ).to_dict()
+
+    def eval_fn(
+        predictions: "pd.Series",
+        metrics: dict[str, MetricValue],
+        inputs: "pd.Series",
+        *args,
+    ) -> MetricValue:
+        """
+        This is the function that is called when the metric is evaluated.
+        """
+        eval_values = dict(zip(grading_context_columns, args))
+
+        outputs = predictions.to_list()
+        inputs = inputs.to_list()
+        eval_model = evaluation_context["model"]
+        eval_parameters = evaluation_context["parameters"]
+
+        # TODO: Save the metric definition in a yaml file for model monitoring
+
+        if not isinstance(eval_model, str):
+            raise MlflowException(
+                message="The model argument must be a string URI referring to an openai model "
+                "(openai:/gpt-4o-mini) or an MLflow Deployments endpoint "
+                f"(endpoints:/my-endpoint), passed {eval_model} instead",
+                error_code=INVALID_PARAMETER_VALUE,
+            )
+
+        # generate grading payloads
+        grading_payloads = []
+        for indx, (input, output) in enumerate(zip(inputs, outputs)):
+            try:
+                arg_string = _format_args_string(grading_context_columns, eval_values, indx)
+            except Exception as e:
+                raise MlflowException(
+                    f"Values for grading_context_columns are malformed and cannot be "
+                    f"formatted into a prompt for metric '{name}'.\n"
+                    f"Required columns: {grading_context_columns}\n"
+                    f"Values: {eval_values}\n"
+                    f"Error: {e!r}\n"
+                    f"Please check the following: \n"
+                    "- predictions and targets (if required) are provided correctly\n"
+                    "- grading_context_columns are mapped correctly using the evaluator_config "
+                    "parameter\n"
+                    "- input and output data are formatted correctly."
+                )
+            grading_payloads.append(
+                evaluation_context["eval_prompt"].format(
+                    input=(input if include_input else None),
+                    output=output,
+                    grading_context_columns=arg_string,
+                )
+            )
+
+        scores = [None] * len(inputs)
+        justifications = [None] * len(inputs)
+
+        with ThreadPoolExecutor(max_workers=max_workers) as executor:
+            futures = {
+                executor.submit(
+                    _score_model_on_one_payload,
+                    payload,
+                    eval_model,
+                    eval_parameters,
+                    extra_headers,
+                    proxy_url,
+                ): indx
+                for indx, payload in enumerate(grading_payloads)
+            }
+
+            as_comp = as_completed(futures)
+            try:
+                from tqdm.auto import tqdm
+
+                as_comp = tqdm(as_comp, total=len(futures))
+            except ImportError:
+                pass
+
+            for future in as_comp:
+                indx = futures[future]
+                score, justification = future.result()
+                scores[indx] = score
+                justifications[indx] = justification
+
+        aggregate_results = _get_aggregate_results(scores, aggregations)
+        return MetricValue(scores, justifications, aggregate_results)
+
+    signature_parameters = [
+        Parameter("predictions", Parameter.POSITIONAL_OR_KEYWORD, annotation="pd.Series"),
+        Parameter("metrics", Parameter.POSITIONAL_OR_KEYWORD, annotation=dict[str, MetricValue]),
+        Parameter("inputs", Parameter.POSITIONAL_OR_KEYWORD, annotation="pd.Series"),
+    ]
+
+    # Add grading_context_columns to signature list
+    for var in grading_context_columns:
+        signature_parameters.append(Parameter(var, Parameter.POSITIONAL_OR_KEYWORD))
+
+    # Note: this doesn't change how python allows calling the function
+    # extra params in grading_context_columns can only be passed as positional args
+    eval_fn.__signature__ = Signature(signature_parameters)
+
+    return _make_metric(
+        eval_fn=eval_fn,
+        greater_is_better=greater_is_better,
+        name=name,
+        version=version,
+        metric_details=evaluation_context["eval_prompt"].__str__(),
+        metric_metadata=metric_metadata,
+        genai_metric_args=genai_metric_args,
+        require_strict_signature=True,
+    )
+
+
+def _filter_by_field(df, field_name, value):
+    return df[df[field_name] == value]
+
+
+def _deserialize_genai_metric_args(args_dict):
+    mlflow_version_at_ser = args_dict.pop("mlflow_version", None)
+    fn_name = args_dict.pop("fn_name", None)
+    if fn_name is None or mlflow_version_at_ser is None:
+        raise MlflowException(
+            message="The artifact JSON file appears to be corrupted and cannot be deserialized. "
+            "Please regenerate the custom metrics and rerun the evaluation. "
+            "Ensure that the file is correctly formatted and not tampered with.",
+            error_code=INTERNAL_ERROR,
+        )
+
+    if mlflow_version_at_ser != VERSION:
+        warnings.warn(
+            f"The custom metric definitions were serialized using MLflow {mlflow_version_at_ser}. "
+            f"Deserializing them with the current version {VERSION} might cause mismatches. "
+            "Please ensure compatibility or consider regenerating the metrics "
+            "using the current version.",
+            UserWarning,
+            stacklevel=2,
+        )
+
+    if fn_name == make_genai_metric_from_prompt.__name__:
+        return make_genai_metric_from_prompt(**args_dict)
+
+    examples = args_dict["examples"]
+    if examples is not None:
+        args_dict["examples"] = [EvaluationExample(**example) for example in examples]
+
+    return make_genai_metric(**args_dict)
+
+
+def retrieve_custom_metrics(
+    run_id: str,
+    name: Optional[str] = None,
+    version: Optional[str] = None,
+) -> list[EvaluationMetric]:
+    """
+    Retrieve the custom metrics created by users through `make_genai_metric()` or
+    `make_genai_metric_from_prompt()` that are associated with a particular evaluation run.
+
+    Args:
+        run_id: The unique identifier for the run.
+        name: (Optional) The name of the custom metric to retrieve.
+            If None, retrieve all metrics.
+        version: (Optional) The version of the custom metric to retrieve.
+            If None, retrieve all metrics.
+
+    Returns:
+        A list of EvaluationMetric objects that match the retrieval criteria.
+
+    .. code-block:: python
+        :caption: Example for retrieving a custom genai metric
+
+        import pandas as pd
+
+        import mlflow
+        from mlflow.metrics.genai.genai_metric import (
+            make_genai_metric_from_prompt,
+            retrieve_custom_metrics,
+        )
+
+        eval_df = pd.DataFrame(
+            {
+                "inputs": ["foo"],
+                "ground_truth": ["bar"],
+            }
+        )
+        with mlflow.start_run() as run:
+            system_prompt = "Answer the following question in two sentences"
+            basic_qa_model = mlflow.openai.log_model(
+                model="gpt-4o-mini",
+                task="chat.completions",
+                artifact_path="model",
+                messages=[
+                    {"role": "system", "content": system_prompt},
+                    {"role": "user", "content": "{question}"},
+                ],
+            )
+            custom_metric = make_genai_metric_from_prompt(
+                name="custom llm judge",
+                judge_prompt="This is a custom judge prompt.",
+                greater_is_better=False,
+                parameters={"temperature": 0.0},
+            )
+            results = mlflow.evaluate(
+                basic_qa_model.model_uri,
+                eval_df,
+                targets="ground_truth",
+                model_type="question-answering",
+                evaluators="default",
+                extra_metrics=[custom_metric],
+            )
+        metrics = retrieve_custom_metrics(
+            run_id=run.info.run_id,
+            name="custom llm judge",
+        )
+    """
+    client = mlflow.MlflowClient()
+    artifacts = [a.path for a in client.list_artifacts(run_id)]
+    if _GENAI_CUSTOM_METRICS_FILE_NAME not in artifacts:
+        _logger.warning("No custom metric definitions were found for this evaluation run.")
+        return []
+
+    with TemporaryDirectory() as tmpdir:
+        downloaded_artifact_path = mlflow.artifacts.download_artifacts(
+            run_id=run_id,
+            artifact_path=_GENAI_CUSTOM_METRICS_FILE_NAME,
+            dst_path=tmpdir,
+        )
+        custom_metrics = client._read_from_file(downloaded_artifact_path)
+
+    if name is not None:
+        custom_metrics = _filter_by_field(custom_metrics, "name", name)
+    if version is not None:
+        custom_metrics = _filter_by_field(custom_metrics, "version", version)
+    metric_args_list = custom_metrics["metric_args"].tolist()
+    if len(metric_args_list) == 0:
+        _logger.warning("No matching custom metric definitions were found.")
+        return []
+
+    return [_deserialize_genai_metric_args(a) for a in metric_args_list]

venv/lib/python3.9/site-packages/mlflow/metrics/genai/metric_definitions.py

@@ -0,0 +1,455 @@
+from typing import Any, Optional
+
+from mlflow.exceptions import MlflowException
+from mlflow.metrics.genai.base import EvaluationExample
+from mlflow.metrics.genai.genai_metric import make_genai_metric
+from mlflow.metrics.genai.utils import _get_latest_metric_version
+from mlflow.models import EvaluationMetric
+from mlflow.protos.databricks_pb2 import INTERNAL_ERROR, INVALID_PARAMETER_VALUE
+from mlflow.utils.annotations import experimental
+from mlflow.utils.class_utils import _get_class_from_string
+
+
+@experimental
+def answer_similarity(
+    model: Optional[str] = None,
+    metric_version: Optional[str] = None,
+    examples: Optional[list[EvaluationExample]] = None,
+    metric_metadata: Optional[dict[str, Any]] = None,
+    parameters: Optional[dict[str, Any]] = None,
+    extra_headers: Optional[dict[str, str]] = None,
+    proxy_url: Optional[str] = None,
+    max_workers: int = 10,
+) -> EvaluationMetric:
+    """
+    This function will create a genai metric used to evaluate the answer similarity of an LLM
+    using the model provided. Answer similarity will be assessed by the semantic similarity of the
+    output to the ``ground_truth``, which should be specified in the ``targets`` column. High
+    scores mean that your model outputs contain similar information as the ground_truth, while
+    low scores mean that outputs may disagree with the ground_truth.
+
+    The ``targets`` eval_arg must be provided as part of the input dataset or output
+    predictions. This can be mapped to a column of a different name using ``col_mapping``
+    in the ``evaluator_config`` parameter, or using the ``targets`` parameter in mlflow.evaluate().
+
+    An MlflowException will be raised if the specified version for this metric does not exist.
+
+    Args:
+        model: (Optional) Model uri of the judge model that will be used to compute the metric,
+            e.g., ``openai:/gpt-4``. Refer to the `LLM-as-a-Judge Metrics <https://mlflow.org/docs/latest/llms/llm-evaluate/index.html#selecting-the-llm-as-judge-model>`_
+            documentation for the supported model types and their URI format.
+        metric_version: (Optional) The version of the answer similarity metric to use.
+            Defaults to the latest version.
+        examples: (Optional) Provide a list of examples to help the judge model evaluate the
+            answer similarity. It is highly recommended to add examples to be used as a reference to
+            evaluate the new results.
+        metric_metadata: (Optional) Dictionary of metadata to be attached to the
+            EvaluationMetric object. Useful for model evaluators that require additional
+            information to determine how to evaluate this metric.
+        parameters: (Optional) Dictionary of parameters to be passed to the judge model,
+            e.g., {"temperature": 0.5}. When specified, these parameters will override
+            the default parameters defined in the metric implementation.
+        extra_headers: (Optional) Dictionary of extra headers to be passed to the judge model.
+        proxy_url: (Optional) Proxy URL to be used for the judge model. This is useful when the
+            judge model is served via a proxy endpoint, not directly via LLM provider services.
+            If not specified, the default URL for the LLM provider will be used
+            (e.g., https://api.openai.com/v1/chat/completions for OpenAI chat models).
+        max_workers: (Optional) The maximum number of workers to use for judge scoring.
+            Defaults to 10 workers.
+
+    Returns:
+        A metric object
+    """
+    if metric_version is None:
+        metric_version = _get_latest_metric_version()
+    class_name = f"mlflow.metrics.genai.prompts.{metric_version}.AnswerSimilarityMetric"
+    try:
+        answer_similarity_class_module = _get_class_from_string(class_name)
+    except ModuleNotFoundError:
+        raise MlflowException(
+            f"Failed to find answer similarity metric for version {metric_version}."
+            f" Please check the version",
+            error_code=INVALID_PARAMETER_VALUE,
+        ) from None
+    except Exception as e:
+        raise MlflowException(
+            f"Failed to construct answer similarity metric {metric_version}. Error: {e!r}",
+            error_code=INTERNAL_ERROR,
+        ) from None
+
+    if examples is None:
+        examples = answer_similarity_class_module.default_examples
+    if model is None:
+        model = answer_similarity_class_module.default_model
+
+    return make_genai_metric(
+        name="answer_similarity",
+        definition=answer_similarity_class_module.definition,
+        grading_prompt=answer_similarity_class_module.grading_prompt,
+        include_input=False,
+        examples=examples,
+        version=metric_version,
+        model=model,
+        grading_context_columns=answer_similarity_class_module.grading_context_columns,
+        parameters=parameters or answer_similarity_class_module.parameters,
+        aggregations=["mean", "variance", "p90"],
+        greater_is_better=True,
+        metric_metadata=metric_metadata,
+        extra_headers=extra_headers,
+        proxy_url=proxy_url,
+        max_workers=max_workers,
+    )
+
+
+@experimental
+def answer_correctness(
+    model: Optional[str] = None,
+    metric_version: Optional[str] = None,
+    examples: Optional[list[EvaluationExample]] = None,
+    metric_metadata: Optional[dict[str, Any]] = None,
+    parameters: Optional[dict[str, Any]] = None,
+    extra_headers: Optional[dict[str, str]] = None,
+    proxy_url: Optional[str] = None,
+    max_workers: int = 10,
+) -> EvaluationMetric:
+    """
+    This function will create a genai metric used to evaluate the answer correctness of an LLM
+    using the model provided. Answer correctness will be assessed by the accuracy of the provided
+    output based on the ``ground_truth``, which should be specified in the ``targets`` column.
+    High scores mean that your model outputs contain similar information as the ground_truth and
+    that this information is correct, while low scores mean that outputs may disagree with the
+    ground_truth or that the information in the output is incorrect. Note that this builds onto
+    answer_similarity.
+
+    The ``targets`` eval_arg must be provided as part of the input dataset or output
+    predictions. This can be mapped to a column of a different name using ``col_mapping``
+    in the ``evaluator_config`` parameter, or using the ``targets`` parameter in mlflow.evaluate().
+
+    An MlflowException will be raised if the specified version for this metric does not exist.
+
+    Args:
+        model: (Optional) Model uri of the judge model that will be used to compute the metric,
+            e.g., ``openai:/gpt-4``. Refer to the `LLM-as-a-Judge Metrics <https://mlflow.org/docs/latest/llms/llm-evaluate/index.html#selecting-the-llm-as-judge-model>`_
+            documentation for the supported model types and their URI format.
+        metric_version: The version of the answer correctness metric to use.
+            Defaults to the latest version.
+        examples: Provide a list of examples to help the judge model evaluate the
+            answer correctness. It is highly recommended to add examples to be used as a reference
+            to evaluate the new results.
+        metric_metadata: (Optional) Dictionary of metadata to be attached to the
+            EvaluationMetric object. Useful for model evaluators that require additional
+            information to determine how to evaluate this metric.
+        parameters: (Optional) Dictionary of parameters to be passed to the judge model,
+            e.g., {"temperature": 0.5}. When specified, these parameters will override
+            the default parameters defined in the metric implementation.
+        extra_headers: (Optional) Dictionary of extra headers to be passed to the judge model.
+        proxy_url: (Optional) Proxy URL to be used for the judge model. This is useful when the
+            judge model is served via a proxy endpoint, not directly via LLM provider services.
+            If not specified, the default URL for the LLM provider will be used
+            (e.g., https://api.openai.com/v1/chat/completions for OpenAI chat models).
+        max_workers: (Optional) The maximum number of workers to use for judge scoring.
+            Defaults to 10 workers.
+
+    Returns:
+        A metric object
+    """
+    if metric_version is None:
+        metric_version = _get_latest_metric_version()
+    class_name = f"mlflow.metrics.genai.prompts.{metric_version}.AnswerCorrectnessMetric"
+    try:
+        answer_correctness_class_module = _get_class_from_string(class_name)
+    except ModuleNotFoundError:
+        raise MlflowException(
+            f"Failed to find answer correctness metric for version {metric_version}."
+            f"Please check the version",
+            error_code=INVALID_PARAMETER_VALUE,
+        ) from None
+    except Exception as e:
+        raise MlflowException(
+            f"Failed to construct answer correctness metric {metric_version}. Error: {e!r}",
+            error_code=INTERNAL_ERROR,
+        ) from None
+
+    if examples is None:
+        examples = answer_correctness_class_module.default_examples
+    if model is None:
+        model = answer_correctness_class_module.default_model
+
+    return make_genai_metric(
+        name="answer_correctness",
+        definition=answer_correctness_class_module.definition,
+        grading_prompt=answer_correctness_class_module.grading_prompt,
+        examples=examples,
+        version=metric_version,
+        model=model,
+        grading_context_columns=answer_correctness_class_module.grading_context_columns,
+        parameters=parameters or answer_correctness_class_module.parameters,
+        aggregations=["mean", "variance", "p90"],
+        greater_is_better=True,
+        metric_metadata=metric_metadata,
+        extra_headers=extra_headers,
+        proxy_url=proxy_url,
+        max_workers=max_workers,
+    )
+
+
+@experimental
+def faithfulness(
+    model: Optional[str] = None,
+    metric_version: Optional[str] = _get_latest_metric_version(),
+    examples: Optional[list[EvaluationExample]] = None,
+    metric_metadata: Optional[dict[str, Any]] = None,
+    parameters: Optional[dict[str, Any]] = None,
+    extra_headers: Optional[dict[str, str]] = None,
+    proxy_url: Optional[str] = None,
+    max_workers: int = 10,
+) -> EvaluationMetric:
+    """
+    This function will create a genai metric used to evaluate the faithfullness of an LLM using the
+    model provided. Faithfulness will be assessed based on how factually consistent the output
+    is to the ``context``. High scores mean that the outputs contain information that is in
+    line with the context, while low scores mean that outputs may disagree with the context
+    (input is ignored).
+
+    The ``context`` eval_arg must be provided as part of the input dataset or output
+    predictions. This can be mapped to a column of a different name using ``col_mapping``
+    in the ``evaluator_config`` parameter.
+
+    An MlflowException will be raised if the specified version for this metric does not exist.
+
+    Args:
+        model: (Optional) Model uri of the judge model that will be used to compute the metric,
+            e.g., ``openai:/gpt-4``. Refer to the `LLM-as-a-Judge Metrics <https://mlflow.org/docs/latest/llms/llm-evaluate/index.html#selecting-the-llm-as-judge-model>`_
+            documentation for the supported model types and their URI format.
+        metric_version: The version of the faithfulness metric to use.
+            Defaults to the latest version.
+        examples: Provide a list of examples to help the judge model evaluate the
+            faithfulness. It is highly recommended to add examples to be used as a reference to
+            evaluate the new results.
+        metric_metadata: (Optional) Dictionary of metadata to be attached to the
+            EvaluationMetric object. Useful for model evaluators that require additional
+            information to determine how to evaluate this metric.
+        parameters: (Optional) Dictionary of parameters to be passed to the judge model,
+            e.g., {"temperature": 0.5}. When specified, these parameters will override
+            the default parameters defined in the metric implementation.
+        extra_headers: (Optional) Dictionary of extra headers to be passed to the judge model.
+        proxy_url: (Optional) Proxy URL to be used for the judge model. This is useful when the
+            judge model is served via a proxy endpoint, not directly via LLM provider services.
+            If not specified, the default URL for the LLM provider will be used
+            (e.g., https://api.openai.com/v1/chat/completions for OpenAI chat models).
+        max_workers: (Optional) The maximum number of workers to use for judge scoring.
+            Defaults to 10 workers.
+
+    Returns:
+        A metric object
+    """
+    class_name = f"mlflow.metrics.genai.prompts.{metric_version}.FaithfulnessMetric"
+    try:
+        faithfulness_class_module = _get_class_from_string(class_name)
+    except ModuleNotFoundError:
+        raise MlflowException(
+            f"Failed to find faithfulness metric for version {metric_version}."
+            f" Please check the version",
+            error_code=INVALID_PARAMETER_VALUE,
+        ) from None
+    except Exception as e:
+        raise MlflowException(
+            f"Failed to construct faithfulness metric {metric_version}. Error: {e!r}",
+            error_code=INTERNAL_ERROR,
+        ) from None
+
+    if examples is None:
+        examples = faithfulness_class_module.default_examples
+    if model is None:
+        model = faithfulness_class_module.default_model
+
+    return make_genai_metric(
+        name="faithfulness",
+        definition=faithfulness_class_module.definition,
+        grading_prompt=faithfulness_class_module.grading_prompt,
+        include_input=False,
+        examples=examples,
+        version=metric_version,
+        model=model,
+        grading_context_columns=faithfulness_class_module.grading_context_columns,
+        parameters=parameters or faithfulness_class_module.parameters,
+        aggregations=["mean", "variance", "p90"],
+        greater_is_better=True,
+        metric_metadata=metric_metadata,
+        extra_headers=extra_headers,
+        proxy_url=proxy_url,
+        max_workers=max_workers,
+    )
+
+
+@experimental
+def answer_relevance(
+    model: Optional[str] = None,
+    metric_version: Optional[str] = _get_latest_metric_version(),
+    examples: Optional[list[EvaluationExample]] = None,
+    metric_metadata: Optional[dict[str, Any]] = None,
+    parameters: Optional[dict[str, Any]] = None,
+    extra_headers: Optional[dict[str, str]] = None,
+    proxy_url: Optional[str] = None,
+    max_workers: int = 10,
+) -> EvaluationMetric:
+    """
+    This function will create a genai metric used to evaluate the answer relevance of an LLM
+    using the model provided. Answer relevance will be assessed based on the appropriateness and
+    applicability of the output with respect to the input. High scores mean that your model
+    outputs are about the same subject as the input, while low scores mean that outputs may
+    be non-topical.
+
+    An MlflowException will be raised if the specified version for this metric does not exist.
+
+    Args:
+        model: (Optional) Model uri of the judge model that will be used to compute the metric,
+            e.g., ``openai:/gpt-4``. Refer to the `LLM-as-a-Judge Metrics <https://mlflow.org/docs/latest/llms/llm-evaluate/index.html#selecting-the-llm-as-judge-model>`_
+            documentation for the supported model types and their URI format.
+        metric_version: The version of the answer relevance metric to use.
+            Defaults to the latest version.
+        examples: Provide a list of examples to help the judge model evaluate the
+            answer relevance. It is highly recommended to add examples to be used as a reference to
+            evaluate the new results.
+        metric_metadata: (Optional) Dictionary of metadata to be attached to the
+            EvaluationMetric object. Useful for model evaluators that require additional
+            information to determine how to evaluate this metric.
+        parameters: (Optional) Dictionary of parameters to be passed to the judge model,
+            e.g., {"temperature": 0.5}. When specified, these parameters will override
+            the default parameters defined in the metric implementation.
+        extra_headers: (Optional) Dictionary of extra headers to be passed to the judge model.
+        proxy_url: (Optional) Proxy URL to be used for the judge model. This is useful when the
+            judge model is served via a proxy endpoint, not directly via LLM provider services.
+            If not specified, the default URL for the LLM provider will be used
+            (e.g., https://api.openai.com/v1/chat/completions for OpenAI chat models).
+        max_workers: (Optional) The maximum number of workers to use for judge scoring.
+            Defaults to 10 workers.
+
+    Returns:
+        A metric object
+    """
+    class_name = f"mlflow.metrics.genai.prompts.{metric_version}.AnswerRelevanceMetric"
+    try:
+        answer_relevance_class_module = _get_class_from_string(class_name)
+    except ModuleNotFoundError:
+        raise MlflowException(
+            f"Failed to find answer relevance metric for version {metric_version}."
+            f" Please check the version",
+            error_code=INVALID_PARAMETER_VALUE,
+        ) from None
+    except Exception as e:
+        raise MlflowException(
+            f"Failed to construct answer relevance metric {metric_version}. Error: {e!r}",
+            error_code=INTERNAL_ERROR,
+        ) from None
+
+    if examples is None:
+        examples = answer_relevance_class_module.default_examples
+    if model is None:
+        model = answer_relevance_class_module.default_model
+
+    return make_genai_metric(
+        name="answer_relevance",
+        definition=answer_relevance_class_module.definition,
+        grading_prompt=answer_relevance_class_module.grading_prompt,
+        examples=examples,
+        version=metric_version,
+        model=model,
+        parameters=parameters or answer_relevance_class_module.parameters,
+        aggregations=["mean", "variance", "p90"],
+        greater_is_better=True,
+        metric_metadata=metric_metadata,
+        extra_headers=extra_headers,
+        proxy_url=proxy_url,
+        max_workers=max_workers,
+    )
+
+
+def relevance(
+    model: Optional[str] = None,
+    metric_version: Optional[str] = None,
+    examples: Optional[list[EvaluationExample]] = None,
+    metric_metadata: Optional[dict[str, Any]] = None,
+    parameters: Optional[dict[str, Any]] = None,
+    extra_headers: Optional[dict[str, str]] = None,
+    proxy_url: Optional[str] = None,
+    max_workers: int = 10,
+) -> EvaluationMetric:
+    """
+    This function will create a genai metric used to evaluate the evaluate the relevance of an
+    LLM using the model provided. Relevance will be assessed by the appropriateness, significance,
+    and applicability of the output with respect to the input and ``context``. High scores mean
+    that the model has understood the context and correct extracted relevant information from
+    the context, while low score mean that output has completely ignored the question and the
+    context and could be hallucinating.
+
+    The ``context`` eval_arg must be provided as part of the input dataset or output
+    predictions. This can be mapped to a column of a different name using ``col_mapping``
+    in the ``evaluator_config`` parameter.
+
+    An MlflowException will be raised if the specified version for this metric does not exist.
+
+    Args:
+        model: (Optional) Model uri of the judge model that will be used to compute the metric,
+            e.g., ``openai:/gpt-4``. Refer to the `LLM-as-a-Judge Metrics <https://mlflow.org/docs/latest/llms/llm-evaluate/index.html#selecting-the-llm-as-judge-model>`_
+            documentation for the supported model types and their URI format.
+        metric_version: (Optional) The version of the relevance metric to use.
+            Defaults to the latest version.
+        examples: (Optional) Provide a list of examples to help the judge model evaluate the
+            relevance. It is highly recommended to add examples to be used as a reference to
+            evaluate the new results.
+        metric_metadata: (Optional) Dictionary of metadata to be attached to the
+            EvaluationMetric object. Useful for model evaluators that require additional
+            information to determine how to evaluate this metric.
+        parameters: (Optional) Dictionary of parameters to be passed to the judge model,
+            e.g., {"temperature": 0.5}. When specified, these parameters will override
+            the default parameters defined in the metric implementation.
+        extra_headers: (Optional) Dictionary of extra headers to be passed to the judge model.
+        proxy_url: (Optional) Proxy URL to be used for the judge model. This is useful when the
+            judge model is served via a proxy endpoint, not directly via LLM provider services.
+            If not specified, the default URL for the LLM provider will be used
+            (e.g., https://api.openai.com/v1/chat/completions for OpenAI chat models).
+        max_workers: (Optional) The maximum number of workers to use for judge scoring.
+            Defaults to 10 workers.
+
+    Returns:
+        A metric object
+    """
+    if metric_version is None:
+        metric_version = _get_latest_metric_version()
+    class_name = f"mlflow.metrics.genai.prompts.{metric_version}.RelevanceMetric"
+    try:
+        relevance_class_module = _get_class_from_string(class_name)
+    except ModuleNotFoundError:
+        raise MlflowException(
+            f"Failed to find relevance metric for version {metric_version}."
+            f"Please check the version",
+            error_code=INVALID_PARAMETER_VALUE,
+        ) from None
+    except Exception as e:
+        raise MlflowException(
+            f"Failed to construct relevance metric {metric_version}. Error: {e!r}",
+            error_code=INTERNAL_ERROR,
+        ) from None
+
+    if examples is None:
+        examples = relevance_class_module.default_examples
+    if model is None:
+        model = relevance_class_module.default_model
+
+    return make_genai_metric(
+        name="relevance",
+        definition=relevance_class_module.definition,
+        grading_prompt=relevance_class_module.grading_prompt,
+        examples=examples,
+        version=metric_version,
+        model=model,
+        grading_context_columns=relevance_class_module.grading_context_columns,
+        parameters=parameters or relevance_class_module.parameters,
+        aggregations=["mean", "variance", "p90"],
+        greater_is_better=True,
+        metric_metadata=metric_metadata,
+        extra_headers=extra_headers,
+        proxy_url=proxy_url,
+        max_workers=max_workers,
+    )

venv/lib/python3.9/site-packages/mlflow/metrics/genai/model_utils.py

@@ -0,0 +1,395 @@
+import logging
+import os
+import urllib.parse
+from typing import TYPE_CHECKING, Any, Optional, Union
+
+import requests
+
+from mlflow.exceptions import MlflowException
+from mlflow.protos.databricks_pb2 import INVALID_PARAMETER_VALUE
+
+if TYPE_CHECKING:
+    from mlflow.gateway.providers import BaseProvider
+
+_logger = logging.getLogger(__name__)
+
+
+def get_endpoint_type(endpoint_uri: str) -> Optional[str]:
+    """
+    Get the type of the endpoint if it is MLflow deployment
+    endpoint. For other endpoints e.g. OpenAI, or if the
+    endpoint does not specify type, return None.
+    """
+    schema, path = _parse_model_uri(endpoint_uri)
+
+    if schema != "endpoints":
+        return None
+
+    from pydantic import BaseModel
+
+    from mlflow.deployments import get_deploy_client
+
+    client = get_deploy_client()
+
+    endpoint = client.get_endpoint(path)
+    # TODO: Standardize the return type of `get_endpoint` and remove this check
+    endpoint = endpoint.dict() if isinstance(endpoint, BaseModel) else endpoint
+    return endpoint.get("task", endpoint.get("endpoint_type"))
+
+
+# TODO: improve this name
+def score_model_on_payload(
+    model_uri,
+    payload,
+    eval_parameters=None,
+    extra_headers=None,
+    proxy_url=None,
+    endpoint_type=None,
+):
+    """Call the model identified by the given uri with the given string prompt."""
+
+    eval_parameters = eval_parameters or {}
+    extra_headers = extra_headers or {}
+
+    prefix, suffix = _parse_model_uri(model_uri)
+
+    if prefix == "gateway":
+        return _call_gateway_api(suffix, payload, eval_parameters)
+    elif prefix == "endpoints":
+        return call_deployments_api(suffix, payload, eval_parameters, endpoint_type)
+    elif prefix in ("model", "runs"):
+        # TODO: call _load_model_or_server
+        raise NotImplementedError
+
+    # Import here to avoid loading gateway module at the top level
+    from mlflow.gateway.provider_registry import is_supported_provider
+
+    if is_supported_provider(prefix):
+        return _call_llm_provider_api(
+            prefix, suffix, payload, eval_parameters, extra_headers, proxy_url
+        )
+
+    raise MlflowException(
+        f"Unknown model uri prefix '{prefix}'",
+        error_code=INVALID_PARAMETER_VALUE,
+    )
+
+
+def _parse_model_uri(model_uri):
+    parsed = urllib.parse.urlparse(model_uri, allow_fragments=False)
+    scheme = parsed.scheme
+    path = parsed.path
+    if not path.startswith("/") or len(path) <= 1:
+        raise MlflowException(
+            f"Malformed model uri '{model_uri}'", error_code=INVALID_PARAMETER_VALUE
+        )
+    path = path.lstrip("/")
+    return scheme, path
+
+
+_PREDICT_ERROR_MSG = """\
+Failed to call the deployment endpoint. Please check the deployment URL \
+is set correctly and the input payload is valid.\n
+- Error: {e}\n
+- Deployment URI: {uri}\n
+- Input payload: {payload}"""
+
+
+def _is_supported_llm_provider(schema: str) -> bool:
+    from mlflow.gateway.provider_registry import provider_registry
+
+    return schema in provider_registry.keys()
+
+
+def _call_llm_provider_api(
+    provider_name: str,
+    model: str,
+    input_data: str,
+    eval_parameters: dict[str, Any],
+    extra_headers: dict[str, str],
+    proxy_url: Optional[str] = None,
+) -> str:
+    """
+    Invoke chat endpoint of various LLM providers.
+
+    Under the hood, this function uses the MLflow Gateway to transform the input/output data
+    for different LLM providers.
+
+    Args:
+        provider_name: The provider name, e.g., "anthropic".
+        model: The model name, e.g., "claude-3-5-sonnet"
+        input_data: The input string prompt to send to the model as a chat message.
+        eval_parameters: The additional parameters to send to the model, e.g. temperature.
+        extra_headers: The additional headers to send to the provider.
+        proxy_url: Proxy URL to be used for the judge model. If not specified, the default
+            URL for the LLM provider will be used.
+    """
+    from mlflow.gateway.config import Provider
+    from mlflow.gateway.schemas import chat
+
+    provider = _get_provider_instance(provider_name, model)
+
+    chat_request = chat.RequestPayload(
+        model=model,
+        messages=[
+            chat.RequestMessage(role="user", content=input_data),
+        ],
+        **eval_parameters,
+    )
+
+    # Filter out keys in the payload to the specified ones + "messages".
+    # Does not include "model" key here because some providers do not accept it as a
+    # part of the payload. Whether or not to include "model" key must be determined
+    # by each provider implementation.
+    filtered_keys = {"messages", *eval_parameters.keys()}
+
+    payload = {
+        k: v
+        for k, v in chat_request.model_dump(exclude_none=True).items()
+        if (v is not None) and (k in filtered_keys)
+    }
+    chat_payload = provider.adapter_class.chat_to_model(payload, provider.config)
+    chat_payload.update(eval_parameters)
+
+    if provider_name in [Provider.AMAZON_BEDROCK, Provider.BEDROCK]:
+        if proxy_url or extra_headers:
+            _logger.warning(
+                "Proxy URL and extra headers are not supported for Bedrock LLMs. "
+                "Ignoring the provided proxy URL and extra headers.",
+            )
+        response = provider._request(chat_payload)
+    else:
+        response = _send_request(
+            endpoint=proxy_url or provider.get_endpoint_url("llm/v1/chat"),
+            headers={**provider.headers, **extra_headers},
+            payload=chat_payload,
+        )
+    chat_response = provider.adapter_class.model_to_chat(response, provider.config)
+    if len(chat_response.choices) == 0:
+        raise MlflowException(
+            "Failed to score the provided input as the judge LLM did not return "
+            "any chat completion results in the response."
+        )
+    content = chat_response.choices[0].message.content
+
+    # NB: Evaluation only handles text content for now.
+    return content[0].text if isinstance(content, list) else content
+
+
+def _get_provider_instance(provider: str, model: str) -> "BaseProvider":
+    """Get the provider instance for the given provider name and the model name."""
+    from mlflow.gateway.config import Provider, RouteConfig
+
+    def _get_route_config(config):
+        return RouteConfig(
+            name=provider,
+            route_type="llm/v1/chat",
+            model={
+                "provider": provider,
+                "name": model,
+                "config": config.model_dump(),
+            },
+        )
+
+    # NB: Not all LLM providers in MLflow Gateway are supported here. We can add
+    # new ones as requested, as long as the provider support chat endpoints.
+    if provider == Provider.OPENAI:
+        from mlflow.gateway.providers.openai import OpenAIConfig, OpenAIProvider
+        from mlflow.openai import _get_api_config, _OAITokenHolder
+
+        api_config = _get_api_config()
+        api_token = _OAITokenHolder(api_config.api_type)
+        api_token.refresh()
+
+        config = OpenAIConfig(
+            openai_api_key=api_token.token,
+            openai_api_type=api_config.api_type or "openai",
+            openai_api_base=api_config.api_base,
+            openai_api_version=api_config.api_version,
+            openai_deployment_name=api_config.deployment_id,
+            openai_organization=api_config.organization,
+        )
+        return OpenAIProvider(_get_route_config(config))
+
+    elif provider == Provider.ANTHROPIC:
+        from mlflow.gateway.providers.anthropic import AnthropicConfig, AnthropicProvider
+
+        config = AnthropicConfig(anthropic_api_key=os.environ.get("ANTHROPIC_API_KEY"))
+        return AnthropicProvider(_get_route_config(config))
+
+    elif provider in [Provider.AMAZON_BEDROCK, Provider.BEDROCK]:
+        from mlflow.gateway.config import AWSIdAndKey, AWSRole
+        from mlflow.gateway.providers.bedrock import AmazonBedrockConfig, AmazonBedrockProvider
+
+        if aws_role_arn := os.environ.get("AWS_ROLE_ARN"):
+            aws_config = AWSRole(
+                aws_region=os.environ.get("AWS_REGION"),
+                aws_role_arn=aws_role_arn,
+            )
+        else:
+            aws_config = AWSIdAndKey(
+                aws_region=os.environ.get("AWS_REGION"),
+                aws_access_key_id=os.environ.get("AWS_ACCESS_KEY_ID"),
+                aws_secret_access_key=os.environ.get("AWS_SECRET_ACCESS_KEY"),
+                aws_session_token=os.environ.get("AWS_SESSION_TOKEN"),
+            )
+        config = AmazonBedrockConfig(aws_config=aws_config)
+        return AmazonBedrockProvider(_get_route_config(config))
+
+    # # Cohere provider implementation seems to be broken and does not work with
+    # # their latest APIs. Uncomment once the provider implementation is fixed.
+    # elif provider == Provider.COHERE:
+    #     from mlflow.gateway.providers.cohere import CohereConfig, CohereProvider
+
+    #     config = CohereConfig(cohere_api_key=os.environ.get("COHERE_API_KEY"))
+    #     return CohereProvider(_get_route_config(config))
+
+    elif provider == Provider.MISTRAL:
+        from mlflow.gateway.providers.mistral import MistralConfig, MistralProvider
+
+        config = MistralConfig(mistral_api_key=os.environ.get("MISTRAL_API_KEY"))
+        return MistralProvider(_get_route_config(config))
+
+    elif provider == Provider.TOGETHERAI:
+        from mlflow.gateway.providers.togetherai import TogetherAIConfig, TogetherAIProvider
+
+        config = TogetherAIConfig(togetherai_api_key=os.environ.get("TOGETHERAI_API_KEY"))
+        return TogetherAIProvider(_get_route_config(config))
+
+    raise MlflowException(f"Provider '{provider}' is not supported for evaluation.")
+
+
+def _send_request(
+    endpoint: str, headers: dict[str, str], payload: dict[str, Any]
+) -> dict[str, Any]:
+    try:
+        response = requests.post(
+            url=endpoint,
+            headers=headers,
+            json=payload,
+            timeout=60,
+        )
+        response.raise_for_status()
+    except requests.exceptions.HTTPError as e:
+        raise MlflowException(
+            f"Failed to call LLM endpoint at {endpoint}.\n- Error: {e}\n- Input payload: {payload}."
+        )
+
+    return response.json()
+
+
+def call_deployments_api(
+    deployment_uri: str,
+    input_data: Union[str, dict[str, Any]],
+    eval_parameters: Optional[dict[str, Any]] = None,
+    endpoint_type: Optional[str] = None,
+):
+    """Call the deployment endpoint with the given payload and parameters.
+
+    Args:
+        deployment_uri: The URI of the deployment endpoint.
+        input_data: The input string or dictionary to send to the endpoint.
+            - If it is a string, MLflow tries to construct the payload based on the endpoint type.
+            - If it is a dictionary, MLflow directly sends it to the endpoint.
+        eval_parameters: The evaluation parameters to send to the endpoint.
+        endpoint_type: The type of the endpoint. If specified, must be 'llm/v1/completions'
+            or 'llm/v1/chat'. If not specified, MLflow tries to get the endpoint type
+            from the endpoint, and if not found, directly sends the payload to the endpoint.
+
+    Returns:
+        The unpacked response from the endpoint.
+    """
+    from mlflow.deployments import get_deploy_client
+
+    client = get_deploy_client()
+
+    if isinstance(input_data, str):
+        payload = _construct_payload_from_str(input_data, endpoint_type)
+    elif isinstance(input_data, dict):
+        # If the input is a dictionary, we assume it is already in the correct format
+        payload = input_data
+    else:
+        raise MlflowException(
+            f"Invalid input data type {type(input_data)}. Must be a string or a dictionary.",
+            error_code=INVALID_PARAMETER_VALUE,
+        )
+    payload = {**payload, **(eval_parameters or {})}
+
+    try:
+        response = client.predict(endpoint=deployment_uri, inputs=payload)
+    except Exception as e:
+        raise MlflowException(
+            _PREDICT_ERROR_MSG.format(e=e, uri=deployment_uri, payload=payload)
+        ) from e
+
+    return _parse_response(response, endpoint_type)
+
+
+def _call_gateway_api(gateway_uri, payload, eval_parameters):
+    from mlflow.gateway import get_route, query
+
+    route_info = get_route(gateway_uri).dict()
+    if route_info["endpoint_type"] == "llm/v1/completions":
+        completions_payload = {
+            "prompt": payload,
+            **eval_parameters,
+        }
+        response = query(gateway_uri, completions_payload)
+        return _parse_completions_response_format(response)
+    elif route_info["endpoint_type"] == "llm/v1/chat":
+        chat_payload = {
+            "messages": [{"role": "user", "content": payload}],
+            **eval_parameters,
+        }
+        response = query(gateway_uri, chat_payload)
+        return _parse_chat_response_format(response)
+    else:
+        raise MlflowException(
+            f"Unsupported gateway route type: {route_info['endpoint_type']}. Use a "
+            "route of type 'llm/v1/completions' or 'llm/v1/chat' instead.",
+            error_code=INVALID_PARAMETER_VALUE,
+        )
+
+
+def _construct_payload_from_str(prompt: str, endpoint_type: str) -> dict[str, Any]:
+    """
+    Construct the payload from the input string based on the endpoint type.
+    If the endpoint type is not specified or unsupported one, raise an exception.
+    """
+    if endpoint_type == "llm/v1/completions":
+        return {"prompt": prompt}
+    elif endpoint_type == "llm/v1/chat":
+        return {"messages": [{"role": "user", "content": prompt}]}
+    else:
+        raise MlflowException(
+            f"Unsupported endpoint type: {endpoint_type}. If string input is provided, "
+            "the endpoint type must be 'llm/v1/completions' or 'llm/v1/chat'.",
+            error_code=INVALID_PARAMETER_VALUE,
+        )
+
+
+def _parse_response(
+    response: dict[str, Any], endpoint_type: Optional[str]
+) -> Union[Optional[str], dict[str, Any]]:
+    if endpoint_type == "llm/v1/completions":
+        return _parse_completions_response_format(response)
+    elif endpoint_type == "llm/v1/chat":
+        return _parse_chat_response_format(response)
+    else:
+        return response
+
+
+def _parse_chat_response_format(response):
+    try:
+        text = response["choices"][0]["message"]["content"]
+    except (KeyError, IndexError, TypeError):
+        text = None
+    return text
+
+
+def _parse_completions_response_format(response):
+    try:
+        text = response["choices"][0]["text"]
+    except (KeyError, IndexError, TypeError):
+        text = None
+    return text

venv/lib/python3.9/site-packages/mlflow/metrics/genai/prompt_template.py

@@ -0,0 +1,68 @@
+import string
+from typing import Any, Union
+
+
+class PromptTemplate:
+    """A prompt template for a language model.
+
+    A prompt template consists of an array of strings that will be concatenated together. It accepts
+    a set of parameters from the user that can be used to generate a prompt for a language model.
+
+    The template can be formatted using f-strings.
+
+    Example:
+
+        .. code-block:: python
+
+            from mlflow.metrics.genai.prompt_template import PromptTemplate
+
+            # Instantiation using initializer
+            prompt = PromptTemplate(template_str="Say {foo} {baz}")
+
+            # Instantiation using partial_fill
+            prompt = PromptTemplate(template_str="Say {foo} {baz}").partial_fill(foo="bar")
+
+            # Format the prompt
+            prompt.format(baz="qux")
+    """
+
+    def __init__(self, template_str: Union[str, list[str]]):
+        self.template_strs = [template_str] if isinstance(template_str, str) else template_str
+
+    @property
+    def variables(self):
+        return {
+            fname
+            for template_str in self.template_strs
+            for _, fname, _, _ in string.Formatter().parse(template_str)
+            if fname
+        }
+
+    def format(self, **kwargs: Any) -> str:
+        safe_kwargs = {k: v for k, v in kwargs.items() if v is not None}
+        formatted_strs = []
+        for template_str in self.template_strs:
+            extracted_variables = [
+                fname for _, fname, _, _ in string.Formatter().parse(template_str) if fname
+            ]
+            if all(item in safe_kwargs.keys() for item in extracted_variables):
+                formatted_strs.append(template_str.format(**safe_kwargs))
+
+        return "".join(formatted_strs)
+
+    def partial_fill(self, **kwargs: Any) -> "PromptTemplate":
+        safe_kwargs = {k: v for k, v in kwargs.items() if v is not None}
+        new_template_strs = []
+        for template_str in self.template_strs:
+            extracted_variables = [
+                fname for _, fname, _, _ in string.Formatter().parse(template_str) if fname
+            ]
+            safe_available_kwargs = {
+                k: safe_kwargs.get(k, "{" + k + "}") for k in extracted_variables
+            }
+            new_template_strs.append(template_str.format_map(safe_available_kwargs))
+
+        return PromptTemplate(template_str=new_template_strs)
+
+    def __str__(self) -> str:
+        return "".join(self.template_strs)

venv/lib/python3.9/site-packages/mlflow/metrics/genai/prompts/v1.py

@@ -0,0 +1,422 @@
+from dataclasses import dataclass, field
+from typing import Any, Optional
+
+from mlflow.metrics.genai.base import EvaluationExample
+from mlflow.metrics.genai.prompt_template import PromptTemplate
+
+# TODO: Update the default_mode and default_parameters to the correct values post experimentation
+default_model = "openai:/gpt-4"
+# Default parameters expressed in the OpenAI format
+default_parameters = {
+    "temperature": 0.0,
+    "max_tokens": 200,
+    "top_p": 1.0,
+}
+
+grading_system_prompt_template = PromptTemplate(
+    [
+        """
+Task:
+You must return the following fields in your response in two lines, one below the other:
+score: Your numerical score for the model's {name} based on the rubric
+justification: Your reasoning about the model's {name} score
+
+You are an impartial judge. You will be given an input that was sent to a machine
+learning model, and you will be given an output that the model produced. You
+may also be given additional information that was used by the model to generate the output.
+
+Your task is to determine a numerical score called {name} based on the input and output.
+A definition of {name} and a grading rubric are provided below.
+You must use the grading rubric to determine your score. You must also justify your score.
+
+Examples could be included below for reference. Make sure to use them as references and to
+understand them before completing the task.""",
+        """
+
+Input:
+{input}""",
+        """
+
+Output:
+{output}
+
+{grading_context_columns}
+
+Metric definition:
+{definition}
+
+Grading rubric:
+{grading_prompt}
+
+{examples}
+
+You must return the following fields in your response in two lines, one below the other:
+score: Your numerical score for the model's {name} based on the rubric
+justification: Your reasoning about the model's {name} score
+
+Do not add additional new lines. Do not add any other fields.
+    """,
+    ]
+)
+
+
+@dataclass
+class EvaluationModel:
+    """
+    Useful to compute v1 prompt for make_genai_metric
+    """
+
+    name: str
+    definition: str
+    grading_prompt: str
+    examples: Optional[list[EvaluationExample]] = None
+    model: str = default_model
+    parameters: dict[str, Any] = field(default_factory=lambda: default_parameters)
+
+    def to_dict(self):
+        examples_str = (
+            ""
+            if self.examples is None or len(self.examples) == 0
+            else f"Examples:\n{self._format_examples()}"
+        )
+
+        return {
+            "model": self.model,
+            "eval_prompt": grading_system_prompt_template.partial_fill(
+                name=self.name,
+                definition=self.definition,
+                grading_prompt=self.grading_prompt,
+                examples=examples_str,
+            ),
+            "parameters": self.parameters,
+        }
+
+    def _format_examples(self):
+        return "\n".join(map(str, self.examples))
+
+
+@dataclass
+class AnswerSimilarityMetric:
+    definition = (
+        "Answer similarity is evaluated on the degree of semantic similarity of the provided "
+        "output to the provided targets, which is the ground truth. Scores can be assigned based "
+        "on the gradual similarity in meaning and description to the provided targets, where a "
+        "higher score indicates greater alignment between the provided output and provided targets."
+    )
+
+    grading_prompt = (
+        "Answer similarity: Below are the details for different scores:\n"
+        "- Score 1: The output has little to no semantic similarity to the provided targets.\n"
+        "- Score 2: The output displays partial semantic similarity to the provided targets on "
+        "some aspects.\n"
+        "- Score 3: The output has moderate semantic similarity to the provided targets.\n"
+        "- Score 4: The output aligns with the provided targets in most aspects and has "
+        "substantial semantic similarity.\n"
+        "- Score 5: The output closely aligns with the provided targets in all significant aspects."
+    )
+
+    grading_context_columns = ["targets"]
+    parameters = default_parameters
+    default_model = default_model
+
+    example_score_2 = EvaluationExample(
+        input="What is MLflow?",
+        output="MLflow is an open-source platform.",
+        score=2,
+        justification="The provided output is partially similar to the target, as it captures the "
+        "general idea that MLflow is an open-source platform. However, it lacks the comprehensive "
+        "details and context provided in the target about MLflow's purpose, development, and "
+        "challenges it addresses. Therefore, it demonstrates partial, but not complete, "
+        "semantic similarity.",
+        grading_context={
+            "targets": "MLflow is an open-source platform for managing the end-to-end "
+            "machine learning (ML) lifecycle. It was developed by Databricks, a company "
+            "that specializes in big data and machine learning solutions. MLflow is "
+            "designed to address the challenges that data scientists and machine learning "
+            "engineers face when developing, training, and deploying machine learning "
+            "models."
+        },
+    )
+
+    example_score_4 = EvaluationExample(
+        input="What is MLflow?",
+        output="MLflow is an open-source platform for managing machine learning workflows, "
+        "including experiment tracking, model packaging, versioning, and deployment, simplifying "
+        "the ML lifecycle.",
+        score=4,
+        justification="The provided output aligns closely with the target. It covers various key "
+        "aspects mentioned in the target, including managing machine learning workflows, "
+        "experiment tracking, model packaging, versioning, and deployment. While it may not include"
+        " every single detail from the target, it demonstrates substantial semantic similarity.",
+        grading_context={
+            "targets": "MLflow is an open-source platform for managing the end-to-end "
+            "machine learning (ML) lifecycle. It was developed by Databricks, a company "
+            "that specializes in big data and machine learning solutions. MLflow is "
+            "designed to address the challenges that data scientists and machine learning "
+            "engineers face when developing, training, and deploying machine learning "
+            "models."
+        },
+    )
+
+    default_examples = [example_score_2, example_score_4]
+
+
+@dataclass
+class FaithfulnessMetric:
+    definition = (
+        "Faithfulness is only evaluated with the provided output and provided context, please "
+        "ignore the provided input entirely when scoring faithfulness. Faithfulness assesses "
+        "how much of the provided output is factually consistent with the provided context. A "
+        "higher score indicates that a higher proportion of claims present in the output can be "
+        "derived from the provided context. Faithfulness does not consider how much extra "
+        "information from the context is not present in the output."
+    )
+
+    grading_prompt = (
+        "Faithfulness: Below are the details for different scores:\n"
+        "- Score 1: None of the claims in the output can be inferred from the provided context.\n"
+        "- Score 2: Some of the claims in the output can be inferred from the provided context, "
+        "but the majority of the output is missing from, inconsistent with, or contradictory to "
+        "the provided context.\n"
+        "- Score 3: Half or more of the claims in the output can be inferred from the provided "
+        "context.\n"
+        "- Score 4: Most of the claims in the output can be inferred from the provided context, "
+        "with very little information that is not directly supported by the provided context.\n"
+        "- Score 5: All of the claims in the output are directly supported by the provided "
+        "context, demonstrating high faithfulness to the provided context."
+    )
+
+    grading_context_columns = ["context"]
+    parameters = default_parameters
+    default_model = default_model
+
+    example_score_2 = EvaluationExample(
+        input="How is MLflow related to Databricks?",
+        output="Databricks is a company that specializes in big data and machine learning "
+        "solutions. MLflow has nothing to do with Databricks. MLflow is an open-source platform "
+        "for managing the end-to-end machine learning (ML) lifecycle.",
+        score=2,
+        justification='The output claims that "MLflow has nothing to do with Databricks" which is '
+        'contradictory to the provided context that states "It was developed by Databricks". This '
+        'is a major inconsistency. However, the output correctly identifies that "MLflow is an '
+        'open-source platform for managing the end-to-end machine learning (ML) lifecycle" and '
+        '"Databricks is a company that specializes in big data and machine learning solutions", '
+        "which are both supported by the context. Therefore, some of the claims in the output can "
+        "be inferred from the provided context, but the majority of the output is inconsistent "
+        "with the provided context, leading to a faithfulness score of 2.",
+        grading_context={
+            "context": "MLflow is an open-source platform for managing the end-to-end machine "
+            "learning (ML) lifecycle. It was developed by Databricks, a company that specializes "
+            "in big data and machine learning solutions. MLflow is designed to address the "
+            "challenges that data scientists and machine learning engineers face when developing, "
+            "training, and deploying machine learning models."
+        },
+    )
+
+    example_score_5 = EvaluationExample(
+        input="How is MLflow related to Databricks?",
+        output="Databricks is a company that specializes in big data and machine learning "
+        "solutions.",
+        score=5,
+        justification='The output states that "Databricks is a company that specializes in big data'
+        ' and machine learning solutions." This claim is directly supported by the context, which '
+        'states "It was developed by Databricks, a company that specializes in big data and '
+        'machine learning solutions." Therefore, the faithfulness score is 5 as all the claims in '
+        'the output are directly supported by the provided context."',
+        grading_context={
+            "context": "MLflow is an open-source platform for managing the end-to-end "
+            "machine learning (ML) lifecycle. It was developed by Databricks, a company "
+            "that specializes in big data and machine learning solutions. MLflow is "
+            "designed to address the challenges that data scientists and machine learning "
+            "engineers face when developing, training, and deploying machine learning "
+            "models."
+        },
+    )
+
+    default_examples = [example_score_2, example_score_5]
+
+
+@dataclass
+class AnswerCorrectnessMetric:
+    definition = (
+        "Answer correctness is evaluated on the accuracy of the provided output based on the "
+        "provided targets, which is the ground truth. Scores can be assigned based on the degree "
+        "of semantic similarity and factual correctness of the provided output to the provided "
+        "targets, where a higher score indicates higher degree of accuracy."
+    )
+
+    grading_prompt = (
+        "Answer Correctness: Below are the details for different scores:\n"
+        "- Score 1: The output is completely incorrect. It is completely different from or "
+        "contradicts the provided targets.\n"
+        "- Score 2: The output demonstrates some degree of semantic similarity and includes "
+        "partially correct information. However, the output still has significant discrepancies "
+        "with the provided targets or inaccuracies.\n"
+        "- Score 3: The output addresses a couple of aspects of the input accurately, aligning "
+        "with the provided targets. However, there are still omissions or minor inaccuracies.\n"
+        "- Score 4: The output is mostly correct. It provides mostly accurate information, but "
+        "there may be one or more minor omissions or inaccuracies.\n"
+        "- Score 5: The output is correct. It demonstrates a high degree of accuracy and "
+        "semantic similarity to the targets."
+    )
+
+    grading_context_columns = ["targets"]
+    parameters = default_parameters
+    default_model = default_model
+
+    example_score_2 = EvaluationExample(
+        input="How is MLflow related to Databricks?",
+        output="Databricks is a data engineering and analytics platform designed to help "
+        "organizations process and analyze large amounts of data. Databricks is a company "
+        "specializing in big data and machine learning solutions.",
+        score=2,
+        justification="The output provided by the model does demonstrate some degree of semantic "
+        "similarity to the targets, as it correctly identifies Databricks as a company "
+        "specializing in big data and machine learning solutions. However, it fails to address "
+        "the main point of the input question, which is the relationship between MLflow and "
+        "Databricks. The output does not mention MLflow at all, which is a significant discrepancy "
+        "with the provided targets. Therefore, the model's answer_correctness score is 2.",
+        grading_context={
+            "targets": "MLflow is an open-source platform for managing the end-to-end machine "
+            "learning (ML) lifecycle. It was developed by Databricks, a company that specializes "
+            "in big data and machine learning solutions. MLflow is designed to address the "
+            "challenges that data scientists and machine learning engineers face when developing, "
+            "training, and deploying machine learning models."
+        },
+    )
+
+    example_score_4 = EvaluationExample(
+        input="How is MLflow related to Databricks?",
+        output="MLflow is a product created by Databricks to enhance the efficiency of machine "
+        "learning processes.",
+        score=4,
+        justification="The output provided by the model is mostly correct. It correctly identifies "
+        "that MLflow is a product created by Databricks. However, it does not mention that MLflow "
+        "is an open-source platform for managing the end-to-end machine learning lifecycle, which "
+        "is a significant part of its function. Therefore, while the output is mostly accurate, "
+        "it has a minor omission, which is why it gets a score of 4 according to the grading "
+        "rubric.",
+        grading_context={
+            "targets": "MLflow is an open-source platform for managing the end-to-end machine "
+            "learning (ML) lifecycle. It was developed by Databricks, a company that specializes "
+            "in big data and machine learning solutions. MLflow is designed to address the "
+            "challenges that data scientists and machine learning engineers face when developing, "
+            "training, and deploying machine learning models."
+        },
+    )
+
+    default_examples = [example_score_2, example_score_4]
+
+
+@dataclass
+class AnswerRelevanceMetric:
+    definition = (
+        "Answer relevance measures the appropriateness and applicability of the output with "
+        "respect to the input. Scores should reflect the extent to which the output directly "
+        "addresses the question provided in the input, and give lower scores for incomplete or "
+        "redundant output."
+    )
+
+    grading_prompt = (
+        "Answer relevance: Please give a score from 1-5 based on the degree of relevance to the "
+        "input, where the lowest and highest scores are defined as follows:"
+        "- Score 1: The output doesn't mention anything about the question or is completely "
+        "irrelevant to the input.\n"
+        "- Score 5: The output addresses all aspects of the question and all parts of the output "
+        "are meaningful and relevant to the question."
+    )
+
+    parameters = default_parameters
+    default_model = default_model
+
+    example_score_2 = EvaluationExample(
+        input="How is MLflow related to Databricks?",
+        output="Databricks is a company that specializes in big data and machine learning "
+        "solutions.",
+        score=2,
+        justification="The output provided by the model does give some information about "
+        "Databricks, which is part of the input question. However, it does not address the main "
+        "point of the question, which is the relationship between MLflow and Databricks. "
+        "Therefore, while the output is not completely irrelevant, it does not fully answer the "
+        "question, leading to a lower score.",
+    )
+
+    example_score_5 = EvaluationExample(
+        input="How is MLflow related to Databricks?",
+        output="MLflow is a product created by Databricks to enhance the efficiency of machine "
+        "learning processes.",
+        score=5,
+        justification="The output directly addresses the input question by explaining the "
+        "relationship between MLflow and Databricks. It provides a clear and concise answer that "
+        "MLflow is a product created by Databricks, and also adds relevant information about the "
+        "purpose of MLflow, which is to enhance the efficiency of machine learning processes. "
+        "Therefore, the output is highly relevant to the input and deserves a full score.",
+    )
+
+    default_examples = [example_score_2, example_score_5]
+
+
+@dataclass
+class RelevanceMetric:
+    definition = (
+        "Relevance encompasses the appropriateness, significance, and applicability of the output "
+        "with respect to both the input and context. Scores should reflect the extent to which the "
+        "output directly addresses the question provided in the input, given the provided context."
+    )
+
+    grading_prompt = (
+        "Relevance: Below are the details for different scores:"
+        "- Score 1: The output doesn't mention anything about the question or is completely "
+        "irrelevant to the provided context.\n"
+        "- Score 2: The output provides some relevance to the question and is somehow related "
+        "to the provided context.\n"
+        "- Score 3: The output mostly answers the question and is largely consistent with the "
+        "provided context.\n"
+        "- Score 4: The output answers the question and is consistent with the provided context.\n"
+        "- Score 5: The output answers the question comprehensively using the provided context."
+    )
+
+    grading_context_columns = ["context"]
+    parameters = default_parameters
+    default_model = default_model
+
+    example_score_2 = EvaluationExample(
+        input="How is MLflow related to Databricks?",
+        output="Databricks is a data engineering and analytics platform designed to help "
+        "organizations process and analyze large amounts of data. Databricks is a company "
+        "specializing in big data and machine learning solutions.",
+        score=2,
+        justification="The output provides relevant information about Databricks, mentioning it "
+        "as a company specializing in big data and machine learning solutions. However, it doesn't "
+        "directly address how MLflow is related to Databricks, which is the specific question "
+        "asked in the input. Therefore, the output is only somewhat related to the provided "
+        "context.",
+        grading_context={
+            "context": "MLflow is an open-source platform for managing the end-to-end machine "
+            "learning (ML) lifecycle. It was developed by Databricks, a company that specializes "
+            "in big data and machine learning solutions. MLflow is designed to address the "
+            "challenges that data scientists and machine learning engineers face when developing, "
+            "training, and deploying machine learning models."
+        },
+    )
+
+    example_score_4 = EvaluationExample(
+        input="How is MLflow related to Databricks?",
+        output="MLflow is a product created by Databricks to enhance the efficiency of machine "
+        "learning processes.",
+        score=4,
+        justification="The output provides a relevant and accurate statement about the "
+        "relationship between MLflow and Databricks. While it doesn't provide extensive detail, "
+        "it still offers a substantial and meaningful response. To achieve a score of 5, the "
+        "response could be further improved by providing additional context or details about "
+        "how MLflow specifically functions within the Databricks ecosystem.",
+        grading_context={
+            "context": "MLflow is an open-source platform for managing the end-to-end machine "
+            "learning (ML) lifecycle. It was developed by Databricks, a company that specializes "
+            "in big data and machine learning solutions. MLflow is designed to address the "
+            "challenges that data scientists and machine learning engineers face when developing, "
+            "training, and deploying machine learning models."
+        },
+    )
+
+    default_examples = [example_score_2, example_score_4]

venv/lib/python3.9/site-packages/mlflow/metrics/genai/utils.py

@@ -0,0 +1,6 @@
+def _get_latest_metric_version():
+    return "v1"
+
+
+def _get_default_model():
+    return "openai:/gpt-4"