Creating a Custom LLM Evaluator with a Benchmark Dataset

import os

import pandas as pd
from phoenix.client import Client
from phoenix.client.types import spans

client = Client(api_key=os.getenv("PHOENIX_API_KEY"))
# replace "accuracy" if you chose to annotate on different criteria
query = spans.SpanQuery().where("annotations['accuracy']")
spans_df = client.spans.get_spans_dataframe(query=query, project_identifier="receipt-classification")
annotations_df = client.spans.get_span_annotations_dataframe(spans_dataframe = spans_df, project_identifier="receipt-classification")
full_df = annotations_df.join(spans_df, how = "inner")

# create_dataset serializes the DataFrame as CSV, so a nested column like
# attributes.llm.output_messages is stored as a string and can't be indexed later.
# Flatten the model's response to plain text up front so the evaluator can read it.
def first_message_content(messages):
    # get_spans_dataframe returns output_messages as a list of message dicts;
    # guard against empty rows (and the occasional already-stringified value).
    if isinstance(messages, list) and messages:
        return messages[0].get("message.content", "")
    if isinstance(messages, str):
        return messages
    return ""

full_df["output_text"] = full_df["attributes.llm.output_messages"].apply(first_message_content)

from phoenix.client import Client
dataset = Client().datasets.create_dataset(
    name="annotated-receipts",
    dataframe=full_df,
    input_keys=["attributes.input.value"],
    output_keys=["output_text"],
    metadata_keys=["result.label", "result.score", "result.explanation"],
)

choices = ["accurate", "almost accurate", "inaccurate"]
prompt_template = """You are an evaluator tasked with assessing the quality of a model-generated expense report.
The model was instructed to analyze a receipt image and return a brief summary including: category of expense, total cost, and summary of items.
---
MODEL OUTPUT (Expense Report): {output}

---
Evaluate whether the expense report is complete and well-structured. Assign one of the following labels. Only include the label:
- **"accurate"** – Includes expense category, total cost, and item summary; all information looks reasonable
- **"almost accurate"** – Mostly correct but with small issues (e.g., missing one element or vague category)
- **"inaccurate"** – Substantially wrong or missing critical information
"""

from phoenix.evals import LLM, ClassificationEvaluator

llm = LLM(provider="openai", model="gpt-5.5")
receipt_evaluator = ClassificationEvaluator(
    name="receipt_accuracy",
    prompt_template=prompt_template,
    llm=llm,
    choices=choices,
)

def task_function(input, reference):
    output = reference["output_text"]

    result = receipt_evaluator.evaluate(
        eval_input={"output": output}
    )
    return result[0].label


def evaluate_response(output, metadata):
    expected_label = metadata["result.label"]
    predicted_label = output
    return 1 if expected_label == predicted_label else 0

prompt_template = """You are an evaluator tasked with assessing the quality of a model-generated expense report.
The model was instructed to analyze a receipt image and return a brief summary including: category of expense, total cost, and summary of items.
---
MODEL OUTPUT (Expense Report): {output}

---
Evaluate the following and assign one of the following labels. Only include the label:
- **"accurate"** – Total price, itemized list, and expense category are all present and look reasonable. All three must be present to get this label.
- **"almost accurate"** – Mostly correct but with small issues. For example, expense category is too vague or one element is missing.
- **"inaccurate"** – Substantially wrong or missing critical information. For example, missing total price entirely.
"""

receipt_evaluator = ClassificationEvaluator(
    name="receipt_accuracy",
    prompt_template=prompt_template,
    llm=llm,
    choices=choices,
)