423 lines
21 KiB
Python
423 lines
21 KiB
Python
from dataclasses import dataclass, field
|
|
from typing import Any, Optional
|
|
|
|
from mlflow.metrics.genai.base import EvaluationExample
|
|
from mlflow.metrics.genai.prompt_template import PromptTemplate
|
|
|
|
# TODO: Update the default_mode and default_parameters to the correct values post experimentation
|
|
default_model = "openai:/gpt-4"
|
|
# Default parameters expressed in the OpenAI format
|
|
default_parameters = {
|
|
"temperature": 0.0,
|
|
"max_tokens": 200,
|
|
"top_p": 1.0,
|
|
}
|
|
|
|
grading_system_prompt_template = PromptTemplate(
|
|
[
|
|
"""
|
|
Task:
|
|
You must return the following fields in your response in two lines, one below the other:
|
|
score: Your numerical score for the model's {name} based on the rubric
|
|
justification: Your reasoning about the model's {name} score
|
|
|
|
You are an impartial judge. You will be given an input that was sent to a machine
|
|
learning model, and you will be given an output that the model produced. You
|
|
may also be given additional information that was used by the model to generate the output.
|
|
|
|
Your task is to determine a numerical score called {name} based on the input and output.
|
|
A definition of {name} and a grading rubric are provided below.
|
|
You must use the grading rubric to determine your score. You must also justify your score.
|
|
|
|
Examples could be included below for reference. Make sure to use them as references and to
|
|
understand them before completing the task.""",
|
|
"""
|
|
|
|
Input:
|
|
{input}""",
|
|
"""
|
|
|
|
Output:
|
|
{output}
|
|
|
|
{grading_context_columns}
|
|
|
|
Metric definition:
|
|
{definition}
|
|
|
|
Grading rubric:
|
|
{grading_prompt}
|
|
|
|
{examples}
|
|
|
|
You must return the following fields in your response in two lines, one below the other:
|
|
score: Your numerical score for the model's {name} based on the rubric
|
|
justification: Your reasoning about the model's {name} score
|
|
|
|
Do not add additional new lines. Do not add any other fields.
|
|
""",
|
|
]
|
|
)
|
|
|
|
|
|
@dataclass
|
|
class EvaluationModel:
|
|
"""
|
|
Useful to compute v1 prompt for make_genai_metric
|
|
"""
|
|
|
|
name: str
|
|
definition: str
|
|
grading_prompt: str
|
|
examples: Optional[list[EvaluationExample]] = None
|
|
model: str = default_model
|
|
parameters: dict[str, Any] = field(default_factory=lambda: default_parameters)
|
|
|
|
def to_dict(self):
|
|
examples_str = (
|
|
""
|
|
if self.examples is None or len(self.examples) == 0
|
|
else f"Examples:\n{self._format_examples()}"
|
|
)
|
|
|
|
return {
|
|
"model": self.model,
|
|
"eval_prompt": grading_system_prompt_template.partial_fill(
|
|
name=self.name,
|
|
definition=self.definition,
|
|
grading_prompt=self.grading_prompt,
|
|
examples=examples_str,
|
|
),
|
|
"parameters": self.parameters,
|
|
}
|
|
|
|
def _format_examples(self):
|
|
return "\n".join(map(str, self.examples))
|
|
|
|
|
|
@dataclass
|
|
class AnswerSimilarityMetric:
|
|
definition = (
|
|
"Answer similarity is evaluated on the degree of semantic similarity of the provided "
|
|
"output to the provided targets, which is the ground truth. Scores can be assigned based "
|
|
"on the gradual similarity in meaning and description to the provided targets, where a "
|
|
"higher score indicates greater alignment between the provided output and provided targets."
|
|
)
|
|
|
|
grading_prompt = (
|
|
"Answer similarity: Below are the details for different scores:\n"
|
|
"- Score 1: The output has little to no semantic similarity to the provided targets.\n"
|
|
"- Score 2: The output displays partial semantic similarity to the provided targets on "
|
|
"some aspects.\n"
|
|
"- Score 3: The output has moderate semantic similarity to the provided targets.\n"
|
|
"- Score 4: The output aligns with the provided targets in most aspects and has "
|
|
"substantial semantic similarity.\n"
|
|
"- Score 5: The output closely aligns with the provided targets in all significant aspects."
|
|
)
|
|
|
|
grading_context_columns = ["targets"]
|
|
parameters = default_parameters
|
|
default_model = default_model
|
|
|
|
example_score_2 = EvaluationExample(
|
|
input="What is MLflow?",
|
|
output="MLflow is an open-source platform.",
|
|
score=2,
|
|
justification="The provided output is partially similar to the target, as it captures the "
|
|
"general idea that MLflow is an open-source platform. However, it lacks the comprehensive "
|
|
"details and context provided in the target about MLflow's purpose, development, and "
|
|
"challenges it addresses. Therefore, it demonstrates partial, but not complete, "
|
|
"semantic similarity.",
|
|
grading_context={
|
|
"targets": "MLflow is an open-source platform for managing the end-to-end "
|
|
"machine learning (ML) lifecycle. It was developed by Databricks, a company "
|
|
"that specializes in big data and machine learning solutions. MLflow is "
|
|
"designed to address the challenges that data scientists and machine learning "
|
|
"engineers face when developing, training, and deploying machine learning "
|
|
"models."
|
|
},
|
|
)
|
|
|
|
example_score_4 = EvaluationExample(
|
|
input="What is MLflow?",
|
|
output="MLflow is an open-source platform for managing machine learning workflows, "
|
|
"including experiment tracking, model packaging, versioning, and deployment, simplifying "
|
|
"the ML lifecycle.",
|
|
score=4,
|
|
justification="The provided output aligns closely with the target. It covers various key "
|
|
"aspects mentioned in the target, including managing machine learning workflows, "
|
|
"experiment tracking, model packaging, versioning, and deployment. While it may not include"
|
|
" every single detail from the target, it demonstrates substantial semantic similarity.",
|
|
grading_context={
|
|
"targets": "MLflow is an open-source platform for managing the end-to-end "
|
|
"machine learning (ML) lifecycle. It was developed by Databricks, a company "
|
|
"that specializes in big data and machine learning solutions. MLflow is "
|
|
"designed to address the challenges that data scientists and machine learning "
|
|
"engineers face when developing, training, and deploying machine learning "
|
|
"models."
|
|
},
|
|
)
|
|
|
|
default_examples = [example_score_2, example_score_4]
|
|
|
|
|
|
@dataclass
|
|
class FaithfulnessMetric:
|
|
definition = (
|
|
"Faithfulness is only evaluated with the provided output and provided context, please "
|
|
"ignore the provided input entirely when scoring faithfulness. Faithfulness assesses "
|
|
"how much of the provided output is factually consistent with the provided context. A "
|
|
"higher score indicates that a higher proportion of claims present in the output can be "
|
|
"derived from the provided context. Faithfulness does not consider how much extra "
|
|
"information from the context is not present in the output."
|
|
)
|
|
|
|
grading_prompt = (
|
|
"Faithfulness: Below are the details for different scores:\n"
|
|
"- Score 1: None of the claims in the output can be inferred from the provided context.\n"
|
|
"- Score 2: Some of the claims in the output can be inferred from the provided context, "
|
|
"but the majority of the output is missing from, inconsistent with, or contradictory to "
|
|
"the provided context.\n"
|
|
"- Score 3: Half or more of the claims in the output can be inferred from the provided "
|
|
"context.\n"
|
|
"- Score 4: Most of the claims in the output can be inferred from the provided context, "
|
|
"with very little information that is not directly supported by the provided context.\n"
|
|
"- Score 5: All of the claims in the output are directly supported by the provided "
|
|
"context, demonstrating high faithfulness to the provided context."
|
|
)
|
|
|
|
grading_context_columns = ["context"]
|
|
parameters = default_parameters
|
|
default_model = default_model
|
|
|
|
example_score_2 = EvaluationExample(
|
|
input="How is MLflow related to Databricks?",
|
|
output="Databricks is a company that specializes in big data and machine learning "
|
|
"solutions. MLflow has nothing to do with Databricks. MLflow is an open-source platform "
|
|
"for managing the end-to-end machine learning (ML) lifecycle.",
|
|
score=2,
|
|
justification='The output claims that "MLflow has nothing to do with Databricks" which is '
|
|
'contradictory to the provided context that states "It was developed by Databricks". This '
|
|
'is a major inconsistency. However, the output correctly identifies that "MLflow is an '
|
|
'open-source platform for managing the end-to-end machine learning (ML) lifecycle" and '
|
|
'"Databricks is a company that specializes in big data and machine learning solutions", '
|
|
"which are both supported by the context. Therefore, some of the claims in the output can "
|
|
"be inferred from the provided context, but the majority of the output is inconsistent "
|
|
"with the provided context, leading to a faithfulness score of 2.",
|
|
grading_context={
|
|
"context": "MLflow is an open-source platform for managing the end-to-end machine "
|
|
"learning (ML) lifecycle. It was developed by Databricks, a company that specializes "
|
|
"in big data and machine learning solutions. MLflow is designed to address the "
|
|
"challenges that data scientists and machine learning engineers face when developing, "
|
|
"training, and deploying machine learning models."
|
|
},
|
|
)
|
|
|
|
example_score_5 = EvaluationExample(
|
|
input="How is MLflow related to Databricks?",
|
|
output="Databricks is a company that specializes in big data and machine learning "
|
|
"solutions.",
|
|
score=5,
|
|
justification='The output states that "Databricks is a company that specializes in big data'
|
|
' and machine learning solutions." This claim is directly supported by the context, which '
|
|
'states "It was developed by Databricks, a company that specializes in big data and '
|
|
'machine learning solutions." Therefore, the faithfulness score is 5 as all the claims in '
|
|
'the output are directly supported by the provided context."',
|
|
grading_context={
|
|
"context": "MLflow is an open-source platform for managing the end-to-end "
|
|
"machine learning (ML) lifecycle. It was developed by Databricks, a company "
|
|
"that specializes in big data and machine learning solutions. MLflow is "
|
|
"designed to address the challenges that data scientists and machine learning "
|
|
"engineers face when developing, training, and deploying machine learning "
|
|
"models."
|
|
},
|
|
)
|
|
|
|
default_examples = [example_score_2, example_score_5]
|
|
|
|
|
|
@dataclass
|
|
class AnswerCorrectnessMetric:
|
|
definition = (
|
|
"Answer correctness is evaluated on the accuracy of the provided output based on the "
|
|
"provided targets, which is the ground truth. Scores can be assigned based on the degree "
|
|
"of semantic similarity and factual correctness of the provided output to the provided "
|
|
"targets, where a higher score indicates higher degree of accuracy."
|
|
)
|
|
|
|
grading_prompt = (
|
|
"Answer Correctness: Below are the details for different scores:\n"
|
|
"- Score 1: The output is completely incorrect. It is completely different from or "
|
|
"contradicts the provided targets.\n"
|
|
"- Score 2: The output demonstrates some degree of semantic similarity and includes "
|
|
"partially correct information. However, the output still has significant discrepancies "
|
|
"with the provided targets or inaccuracies.\n"
|
|
"- Score 3: The output addresses a couple of aspects of the input accurately, aligning "
|
|
"with the provided targets. However, there are still omissions or minor inaccuracies.\n"
|
|
"- Score 4: The output is mostly correct. It provides mostly accurate information, but "
|
|
"there may be one or more minor omissions or inaccuracies.\n"
|
|
"- Score 5: The output is correct. It demonstrates a high degree of accuracy and "
|
|
"semantic similarity to the targets."
|
|
)
|
|
|
|
grading_context_columns = ["targets"]
|
|
parameters = default_parameters
|
|
default_model = default_model
|
|
|
|
example_score_2 = EvaluationExample(
|
|
input="How is MLflow related to Databricks?",
|
|
output="Databricks is a data engineering and analytics platform designed to help "
|
|
"organizations process and analyze large amounts of data. Databricks is a company "
|
|
"specializing in big data and machine learning solutions.",
|
|
score=2,
|
|
justification="The output provided by the model does demonstrate some degree of semantic "
|
|
"similarity to the targets, as it correctly identifies Databricks as a company "
|
|
"specializing in big data and machine learning solutions. However, it fails to address "
|
|
"the main point of the input question, which is the relationship between MLflow and "
|
|
"Databricks. The output does not mention MLflow at all, which is a significant discrepancy "
|
|
"with the provided targets. Therefore, the model's answer_correctness score is 2.",
|
|
grading_context={
|
|
"targets": "MLflow is an open-source platform for managing the end-to-end machine "
|
|
"learning (ML) lifecycle. It was developed by Databricks, a company that specializes "
|
|
"in big data and machine learning solutions. MLflow is designed to address the "
|
|
"challenges that data scientists and machine learning engineers face when developing, "
|
|
"training, and deploying machine learning models."
|
|
},
|
|
)
|
|
|
|
example_score_4 = EvaluationExample(
|
|
input="How is MLflow related to Databricks?",
|
|
output="MLflow is a product created by Databricks to enhance the efficiency of machine "
|
|
"learning processes.",
|
|
score=4,
|
|
justification="The output provided by the model is mostly correct. It correctly identifies "
|
|
"that MLflow is a product created by Databricks. However, it does not mention that MLflow "
|
|
"is an open-source platform for managing the end-to-end machine learning lifecycle, which "
|
|
"is a significant part of its function. Therefore, while the output is mostly accurate, "
|
|
"it has a minor omission, which is why it gets a score of 4 according to the grading "
|
|
"rubric.",
|
|
grading_context={
|
|
"targets": "MLflow is an open-source platform for managing the end-to-end machine "
|
|
"learning (ML) lifecycle. It was developed by Databricks, a company that specializes "
|
|
"in big data and machine learning solutions. MLflow is designed to address the "
|
|
"challenges that data scientists and machine learning engineers face when developing, "
|
|
"training, and deploying machine learning models."
|
|
},
|
|
)
|
|
|
|
default_examples = [example_score_2, example_score_4]
|
|
|
|
|
|
@dataclass
|
|
class AnswerRelevanceMetric:
|
|
definition = (
|
|
"Answer relevance measures the appropriateness and applicability of the output with "
|
|
"respect to the input. Scores should reflect the extent to which the output directly "
|
|
"addresses the question provided in the input, and give lower scores for incomplete or "
|
|
"redundant output."
|
|
)
|
|
|
|
grading_prompt = (
|
|
"Answer relevance: Please give a score from 1-5 based on the degree of relevance to the "
|
|
"input, where the lowest and highest scores are defined as follows:"
|
|
"- Score 1: The output doesn't mention anything about the question or is completely "
|
|
"irrelevant to the input.\n"
|
|
"- Score 5: The output addresses all aspects of the question and all parts of the output "
|
|
"are meaningful and relevant to the question."
|
|
)
|
|
|
|
parameters = default_parameters
|
|
default_model = default_model
|
|
|
|
example_score_2 = EvaluationExample(
|
|
input="How is MLflow related to Databricks?",
|
|
output="Databricks is a company that specializes in big data and machine learning "
|
|
"solutions.",
|
|
score=2,
|
|
justification="The output provided by the model does give some information about "
|
|
"Databricks, which is part of the input question. However, it does not address the main "
|
|
"point of the question, which is the relationship between MLflow and Databricks. "
|
|
"Therefore, while the output is not completely irrelevant, it does not fully answer the "
|
|
"question, leading to a lower score.",
|
|
)
|
|
|
|
example_score_5 = EvaluationExample(
|
|
input="How is MLflow related to Databricks?",
|
|
output="MLflow is a product created by Databricks to enhance the efficiency of machine "
|
|
"learning processes.",
|
|
score=5,
|
|
justification="The output directly addresses the input question by explaining the "
|
|
"relationship between MLflow and Databricks. It provides a clear and concise answer that "
|
|
"MLflow is a product created by Databricks, and also adds relevant information about the "
|
|
"purpose of MLflow, which is to enhance the efficiency of machine learning processes. "
|
|
"Therefore, the output is highly relevant to the input and deserves a full score.",
|
|
)
|
|
|
|
default_examples = [example_score_2, example_score_5]
|
|
|
|
|
|
@dataclass
|
|
class RelevanceMetric:
|
|
definition = (
|
|
"Relevance encompasses the appropriateness, significance, and applicability of the output "
|
|
"with respect to both the input and context. Scores should reflect the extent to which the "
|
|
"output directly addresses the question provided in the input, given the provided context."
|
|
)
|
|
|
|
grading_prompt = (
|
|
"Relevance: Below are the details for different scores:"
|
|
"- Score 1: The output doesn't mention anything about the question or is completely "
|
|
"irrelevant to the provided context.\n"
|
|
"- Score 2: The output provides some relevance to the question and is somehow related "
|
|
"to the provided context.\n"
|
|
"- Score 3: The output mostly answers the question and is largely consistent with the "
|
|
"provided context.\n"
|
|
"- Score 4: The output answers the question and is consistent with the provided context.\n"
|
|
"- Score 5: The output answers the question comprehensively using the provided context."
|
|
)
|
|
|
|
grading_context_columns = ["context"]
|
|
parameters = default_parameters
|
|
default_model = default_model
|
|
|
|
example_score_2 = EvaluationExample(
|
|
input="How is MLflow related to Databricks?",
|
|
output="Databricks is a data engineering and analytics platform designed to help "
|
|
"organizations process and analyze large amounts of data. Databricks is a company "
|
|
"specializing in big data and machine learning solutions.",
|
|
score=2,
|
|
justification="The output provides relevant information about Databricks, mentioning it "
|
|
"as a company specializing in big data and machine learning solutions. However, it doesn't "
|
|
"directly address how MLflow is related to Databricks, which is the specific question "
|
|
"asked in the input. Therefore, the output is only somewhat related to the provided "
|
|
"context.",
|
|
grading_context={
|
|
"context": "MLflow is an open-source platform for managing the end-to-end machine "
|
|
"learning (ML) lifecycle. It was developed by Databricks, a company that specializes "
|
|
"in big data and machine learning solutions. MLflow is designed to address the "
|
|
"challenges that data scientists and machine learning engineers face when developing, "
|
|
"training, and deploying machine learning models."
|
|
},
|
|
)
|
|
|
|
example_score_4 = EvaluationExample(
|
|
input="How is MLflow related to Databricks?",
|
|
output="MLflow is a product created by Databricks to enhance the efficiency of machine "
|
|
"learning processes.",
|
|
score=4,
|
|
justification="The output provides a relevant and accurate statement about the "
|
|
"relationship between MLflow and Databricks. While it doesn't provide extensive detail, "
|
|
"it still offers a substantial and meaningful response. To achieve a score of 5, the "
|
|
"response could be further improved by providing additional context or details about "
|
|
"how MLflow specifically functions within the Databricks ecosystem.",
|
|
grading_context={
|
|
"context": "MLflow is an open-source platform for managing the end-to-end machine "
|
|
"learning (ML) lifecycle. It was developed by Databricks, a company that specializes "
|
|
"in big data and machine learning solutions. MLflow is designed to address the "
|
|
"challenges that data scientists and machine learning engineers face when developing, "
|
|
"training, and deploying machine learning models."
|
|
},
|
|
)
|
|
|
|
default_examples = [example_score_2, example_score_4]
|