{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ur8xi4C7S06n"
      },
      "outputs": [],
      "source": [
        "# Copyright 2025 Google LLC\n",
        "#\n",
        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
        "# you may not use this file except in compliance with the License.\n",
        "# You may obtain a copy of the License at\n",
        "#\n",
        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
        "#\n",
        "# Unless required by applicable law or agreed to in writing, software\n",
        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
        "# See the License for the specific language governing permissions and\n",
        "# limitations under the License."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JAPoU8Sm5E6e"
      },
      "source": [
        "# Evaluate agent final answer with custom parsing"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "898104bc6188"
      },
      "source": [
        "<table align=\"left\">\n",
        "  <td style=\"text-align: center\">\n",
        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/generative-ai/blob/main/gemini/evaluation/evaltask_approach/evaluate_agent_final_answer_with_custom_parsing.ipynb\">\n",
        "      <img width=\"32px\" src=\"https://www.gstatic.com/pantheon/images/bigquery/welcome_page/colab-logo.svg\" alt=\"Google Colaboratory logo\"><br> Open in Colab\n",
        "    </a>\n",
        "  </td>\n",
        "  <td style=\"text-align: center\">\n",
        "    <a href=\"https://console.cloud.google.com/vertex-ai/colab/import/https:%2F%2Fraw.githubusercontent.com%2FGoogleCloudPlatform%2Fgenerative-ai%2Fmain%2Fgemini%2Fevaluation%2Fevaltask_approach%2Fevaluate_agent_final_answer_with_custom_parsing.ipynb\">\n",
        "      <img width=\"32px\" src=\"https://lh3.googleusercontent.com/JmcxdQi-qOpctIvWKgPtrzZdJJK-J3sWE1RsfjZNwshCFgE_9fULcNpuXYTilIR2hjwN\" alt=\"Google Cloud Colab Enterprise logo\"><br> Open in Colab Enterprise\n",
        "    </a>\n",
        "  </td>\n",
        "  <td style=\"text-align: center\">\n",
        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/generative-ai/main/gemini/evaluation/evaltask_approach/evaluate_agent_final_answer_with_custom_parsing.ipynb\">\n",
        "      <img src=\"https://www.gstatic.com/images/branding/gcpiconscolors/vertexai/v1/32px.svg\" alt=\"Vertex AI logo\"><br> Open in Vertex AI Workbench\n",
        "    </a>\n",
        "  </td>\n",
        "  <td style=\"text-align: center\">\n",
        "    <a href=\"https://github.com/GoogleCloudPlatform/generative-ai/blob/main/gemini/evaluation/evaltask_approach/evaluate_agent_final_answer_with_custom_parsing.ipynb\">\n",
        "      <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\"><br> View on GitHub\n",
        "    </a>\n",
        "  </td>\n",
        "</table>\n",
        "\n",
        "<div style=\"clear: both;\"></div>\n",
        "\n",
        "<b>Share to:</b>\n",
        "\n",
        "<a href=\"https://www.linkedin.com/sharing/share-offsite/?url=https%3A//github.com/GoogleCloudPlatform/generative-ai/blob/main/gemini/evaluation/evaltask_approach/evaluate_agent_final_answer_with_custom_parsing.ipynb\" target=\"_blank\">\n",
        "  <img width=\"20px\" src=\"https://upload.wikimedia.org/wikipedia/commons/8/81/LinkedIn_icon.svg\" alt=\"LinkedIn logo\">\n",
        "</a>\n",
        "\n",
        "<a href=\"https://bsky.app/intent/compose?text=https%3A//github.com/GoogleCloudPlatform/generative-ai/blob/main/gemini/evaluation/evaltask_approach/evaluate_agent_final_answer_with_custom_parsing.ipynb\" target=\"_blank\">\n",
        "  <img width=\"20px\" src=\"https://upload.wikimedia.org/wikipedia/commons/7/7a/Bluesky_Logo.svg\" alt=\"Bluesky logo\">\n",
        "</a>\n",
        "\n",
        "<a href=\"https://twitter.com/intent/tweet?url=https%3A//github.com/GoogleCloudPlatform/generative-ai/blob/main/gemini/evaluation/evaltask_approach/evaluate_agent_final_answer_with_custom_parsing.ipynb\" target=\"_blank\">\n",
        "  <img width=\"20px\" src=\"https://upload.wikimedia.org/wikipedia/commons/5/5a/X_icon_2.svg\" alt=\"X logo\">\n",
        "</a>\n",
        "\n",
        "<a href=\"https://reddit.com/submit?url=https%3A//github.com/GoogleCloudPlatform/generative-ai/blob/main/gemini/evaluation/evaltask_approach/evaluate_agent_final_answer_with_custom_parsing.ipynb\" target=\"_blank\">\n",
        "  <img width=\"20px\" src=\"https://redditinc.com/hubfs/Reddit%20Inc/Brand/Reddit_Logo.png\" alt=\"Reddit logo\">\n",
        "</a>\n",
        "\n",
        "<a href=\"https://www.facebook.com/sharer/sharer.php?u=https%3A//github.com/GoogleCloudPlatform/generative-ai/blob/main/gemini/evaluation/evaltask_approach/evaluate_agent_final_answer_with_custom_parsing.ipynb\" target=\"_blank\">\n",
        "  <img width=\"20px\" src=\"https://upload.wikimedia.org/wikipedia/commons/5/51/Facebook_f_logo_%282019%29.svg\" alt=\"Facebook logo\">\n",
        "</a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UBAgFfnqvxy0"
      },
      "source": [
        " | | |\n",
        " |-|-|\n",
        " |Author(s): | [Greg Breard](https://github.com/gregbreard) |"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tvgnzT1CKxrO"
      },
      "source": [
        "## Overview\n",
        "\n",
        "This Colab shows how to assess the quality of agent final answers from generative models, evaluating their ability to generate responses that are valid compared to the golden response (human answer) expected from the agent. Evaluating the validity of final agent answers with custom parsing, as demonstrated here, provides a more advanced assessment than standard [metric prompt templates](https://cloud.google.com/vertex-ai/generative-ai/docs/models/metrics-templates), enabling detailed analysis of correctness and expected output adherence.\n",
        "\n",
        "Agents often interface with Retrieval-Augmented Generation (RAG) APIs to respond to a query. In the following example, the autorater checks whether the answer the agent model gave is valid (or invalid) by checking it against the human response. If the response is invalid, the autorater will rewrite the response so that it is true.\n",
        "\n",
        "This is accomplished using the Vertex Gen AI Evaluation SDK which supports custom output parsing. The prompt instructs the autorater model to return a structured (JSON) output which is then parsed seamlessly by providing the parsing method in the metric definition. The parsed output is appended to the evaluation result data frame.\n",
        "\n",
        "## Objective\n",
        "\n",
        "1. Generate structured output from an autorater.\n",
        "2. Use custom parsing for advanced evaluation.\n",
        "\n",
        "## Steps\n",
        "\n",
        "1. Set up the environment.\n",
        "2. Define helper functions, prompt templates, and metric.\n",
        "3. Prepare the dataset for evaluation.\n",
        "4. Run the evaluation (including model inference).\n",
        "\n",
        "## Costs\n",
        "This tutorial uses billable components of Google Cloud:\n",
        "\n",
        "- Vertex AI\n",
        "\n",
        "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing) and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "61RBz8LLbxCR"
      },
      "source": [
        "# Get started"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "No17Cw5hgx12"
      },
      "source": [
        "## Install Vertex AI SDK for Python and other required packages\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "tFy3H3aPgx12"
      },
      "outputs": [],
      "source": [
        "%pip install --upgrade --quiet google-cloud-aiplatform"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "R5Xep4W9lq-Z"
      },
      "source": [
        "## Restart runtime (Colab only)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NTGpLRykv37-"
      },
      "source": [
        "To use the newly installed packages, you must restart the runtime on Google Colab."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "XRvKdaPDTznN"
      },
      "outputs": [],
      "source": [
        "import sys\n",
        "\n",
        "if \"google.colab\" in sys.modules:\n",
        "    import IPython\n",
        "\n",
        "    app = IPython.Application.instance()\n",
        "    app.kernel.do_shutdown(True)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SbmM4z7FOBpM"
      },
      "source": [
        "<div class=\"alert alert-block alert-warning\">\n",
        "<b>⚠️ The kernel is going to restart. Wait until it's finished before continuing to the next step. ⚠️</b>\n",
        "</div>\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dmWOrTJ3gx13"
      },
      "source": [
        "## Authenticate your notebook environment (Colab only)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7pAMQL6bwGKg"
      },
      "source": [
        "Authenticate your environment on Google Colab."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "NyKGtVQjgx13"
      },
      "outputs": [],
      "source": [
        "import sys\n",
        "\n",
        "if \"google.colab\" in sys.modules:\n",
        "    from google.colab import auth\n",
        "\n",
        "    auth.authenticate_user()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DF4l8DTdWgPY"
      },
      "source": [
        "## Set Google Cloud project information and initialize Vertex AI SDK for Python"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xdZEC-KuwKOM"
      },
      "source": [
        "To get started using Vertex AI, you must have an existing Google Cloud project and [enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com). Learn more about [setting up a project and a development environment](https://cloud.google.com/vertex-ai/docs/start/cloud-environment)."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Nqwi-5ufWp_B"
      },
      "outputs": [],
      "source": [
        "PROJECT_ID = \"your-project-id\"  # @param {type:\"string\"}\n",
        "LOCATION = \"us-central1\"  # @param {type:\"string\"}\n",
        "\n",
        "\n",
        "import vertexai\n",
        "\n",
        "vertexai.init(project=PROJECT_ID, location=LOCATION)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EdvJRUWRNGHE"
      },
      "source": [
        "## Import libraries"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UwS2qQaxNTyy"
      },
      "source": [
        "# Set up evaluation"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "e-TDamAG9mkM"
      },
      "source": [
        "## Helper functions"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1pjVIACIMYne"
      },
      "source": [
        "The following functions provide support for extracting JSON objects from the results returned by the autorater and computing the model response score. Additionally, there are pretty printing methods to improve the readability of the evaluation results."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GfLmFxkP-Kvr"
      },
      "outputs": [],
      "source": [
        "import json\n",
        "import re\n",
        "from typing import Any\n",
        "\n",
        "import pandas as pd\n",
        "from vertexai.preview.evaluation import (\n",
        "    AutoraterConfig,\n",
        "    CustomOutputConfig,\n",
        "    EvalTask,\n",
        "    PointwiseMetric,\n",
        ")\n",
        "\n",
        "_TABLE_STYLE = [\n",
        "    {\n",
        "        \"selector\": \"th\",\n",
        "        \"props\": [\n",
        "            (\"background-color\", \"#f2f2f2\"),\n",
        "            (\"border\", \"1px solid gray\"),\n",
        "            (\"color\", \"black\"),\n",
        "            (\"font-size\", \"11pt\"),\n",
        "            (\"text-align\", \"center\"),\n",
        "            (\"word-break\", \"break-all\"),\n",
        "        ],\n",
        "    },\n",
        "    {\n",
        "        \"selector\": \"td\",\n",
        "        \"props\": [\n",
        "            (\"border\", \"1px solid gray\"),\n",
        "            (\"color\", \"black\"),\n",
        "            (\"min-width\", \"100px\"),\n",
        "            (\"text-align\", \"center\"),\n",
        "        ],\n",
        "    },\n",
        "    {\"selector\": \"tr:nth-child(even)\", \"props\": [(\"background-color\", \"#f9f9f9\")]},\n",
        "    {\"selector\": \"tr:nth-child(odd)\", \"props\": [(\"background-color\", \"white\")]},\n",
        "    {\"selector\": \"tr:hover\", \"props\": [(\"background-color\", \"#94e6ff\")]},\n",
        "    {\"selector\": \"td:hover\", \"props\": [(\"background-color\", \"#ffffb3\")]},\n",
        "]\n",
        "\n",
        "\n",
        "def compute_metric_score(df: \"pd.DataFrame\") -> float:\n",
        "    valid = 0.0\n",
        "    for idx, row in df.iterrows():\n",
        "        result = row[\"agent_final_answer/result\"]\n",
        "        if isinstance(result, dict):\n",
        "            if result[\"is_the_agent_response_valid\"] == \"valid\":\n",
        "                valid += 1\n",
        "    return valid / len(df)\n",
        "\n",
        "\n",
        "def parse_response_to_json(responses: list[str]) -> dict[str, Any]:\n",
        "    response = re.sub(\n",
        "        r\"(.*```json|```.*)\",\n",
        "        \"\",\n",
        "        responses[0].strip(),\n",
        "    )\n",
        "    result = None\n",
        "    try:\n",
        "        result = json.loads(response)\n",
        "    except Exception as e:\n",
        "        print(f\"Failed to parse JSON response: {e!s}\")\n",
        "    return {\"result\": result}\n",
        "\n",
        "\n",
        "def pretty_print_df(df: \"pd.DataFrame\") -> \"pd.Styler\":\n",
        "    styled_df = df.copy()\n",
        "    for col in df.columns:\n",
        "        if isinstance(df[col][0], dict):\n",
        "            styled_df[col] = styled_df[col].apply(lambda x: _dict_to_html_table(x))\n",
        "    return styled_df.style.hide(axis=\"index\").set_table_styles(_TABLE_STYLE)\n",
        "\n",
        "\n",
        "def _dict_to_html_table(data: dict[str, Any]) -> str:\n",
        "    if not data:\n",
        "        return \"<i>No data to display.</i>\"\n",
        "    html_table = \"<table style='border-collapse: collapse'>\"\n",
        "    for key, value in data.items():\n",
        "        html_table += (\n",
        "            f\"<tr><td style='font-weight: bold'>{key}</td><td>{value}</td></tr>\"\n",
        "        )\n",
        "    html_table += \"</tbody></table>\"\n",
        "    return html_table"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tLn3FteP9ttL"
      },
      "source": [
        "## Prompt Templates"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5--M9Kcz-LFt"
      },
      "outputs": [],
      "source": [
        "AGENT_FINAL_ANSWER_PROMPT = \"\"\"\n",
        "You are an expert rater for an AI agent. The AI agent is going to call an API to\n",
        "answer the user query and generate API tool use code based for the choice of the\n",
        "API and API arguments. The ideal model response should be a function call that\n",
        "fulfills user query, or a natural language response hedges or asks users for\n",
        "further clarification if a function call does not apply.\n",
        "The primary focus of this rating task is to check correctness of the model\n",
        "responses. If the model response is not correct, you need to write a corrected\n",
        "response.\n",
        "\n",
        "The data consists of:\n",
        "- A set of python function definitions available to the agent.\n",
        "- A user query.\n",
        "- A model generated response for the prompt. The responses can consist of:\n",
        "  - Natural language, when the model is asking for clarification, or tells the\n",
        "    user it does not possess the requested functionality / option.\n",
        "  - Code, in the form of one or multiple python function calls, and additional\n",
        "    code as needed, for when the model is fulfilling the user request.\n",
        "You can use the help from a reference response annotated by a human rater. This\n",
        "reference response is of high quality. You can compare the agent's response with\n",
        "the reference response and decide if the agent's response is valid.\n",
        "\n",
        "You should follow the constitutions below to rate the model response:\n",
        "- Allow flexibility of format even when reference code only uses one of the\n",
        "  possible format, unless API spec or user prompt has explicit format\n",
        "  requirement\n",
        "  - e.g. For state name, allow both abbreviation and full name unless API spec\n",
        "    has explicit requirement. e.g. both 'tx' and 'Texas' should be allowed in\n",
        "    the agent response even when reference code only uses one of them.\n",
        "  - e.g. If a reference response list outputs in a list format, the agent\n",
        "    response is allowed to use sentence format unless user prompt explicitly\n",
        "    asks for a list.\n",
        "- The model shouldn't assume that it doesn't have access to according data or\n",
        "  incapable of answering the question if reference response is able to find a\n",
        "  legit answer.\n",
        "- If the model response contains the correct final answer, rate it as valid even\n",
        "  when the model response contains more information than the reference response.\n",
        "- If the user prompt has csv or other table format data, don't read it yourself.\n",
        "  Trust the reference response final answer instead.\n",
        "- Be mindful about unit of numbers. For example, if the reference response says\n",
        "  100 miles, but the model response says 100 km, it is invalid.\n",
        "\n",
        "Below are the inputs:\n",
        "{\n",
        "  \"User prompt\": {prompt},\n",
        "  \"Agent response\": {response},\n",
        "  \"Reference response\": {golden},\n",
        "}\n",
        "\n",
        "The answer should be a json alone which follows the json structure below:\n",
        "{\n",
        "  \"is_the_agent_response_valid\": \"[valid or invalid]\",\n",
        "  \"reasoning\":\n",
        "  \"rewritten response\":\n",
        "}\n",
        "Answer with assertiveness:\n",
        "\"\"\""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Fdtcd-WF9ymb"
      },
      "source": [
        "## Define the metric"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CeA0mrWi-LX1"
      },
      "outputs": [],
      "source": [
        "agent_final_answer_metric = PointwiseMetric(\n",
        "    metric=\"agent_final_answer\",\n",
        "    metric_prompt_template=AGENT_FINAL_ANSWER_PROMPT,\n",
        "    custom_output_config=CustomOutputConfig(\n",
        "        return_raw_output=True,\n",
        "        parsing_fn=parse_response_to_json,\n",
        "    ),\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nHjCRree99Px"
      },
      "source": [
        "# Prepare the dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "U37FNLD8-Lp5"
      },
      "outputs": [],
      "source": [
        "prompts = [\n",
        "    \"Considering countries population in 2022, which country ranked 10th in population for this year? Respond in the format 'Country_Name: Population'\",\n",
        "    \"Calculate the difference between the average highest and lowest temperatures recorded in the USA between January and December 2023. Express your answer as a single number, which represents degrees Fahrenheit, rounded to 1 decimal place.\",\n",
        "    \"How many total medals did the USA win in the Rio Olympics and what percentage of these were gold medals?\",\n",
        "    \"In F1 2021, Lewis Hamilton lost the World Championship to Max Verstappen in the last race. How many times did both of the drivers retire during the same race that year?\",\n",
        "    \"What is the volume in milliliters of a system comprised of 0.312 kg Freon-12 refrigerant when placed at the bottom of the Mariana Trench and allowed to stabilize at the Trench's peak temperature, rounded to the nearest mL? Provide your answer as just an integer value.\",\n",
        "]\n",
        "responses = [\n",
        "    \"Mexico: 129,150,971\",\n",
        "    \"40.5\",\n",
        "    \"The USA won a total of 121 medals at the Rio Olympics.  38.02% of these medals were gold.\",\n",
        "    \"Lewis Hamilton and Max Verstappen both retired from the same race once in the 2021 F1 season, at the Italian Grand Prix.\",\n",
        "    \"55\",\n",
        "]\n",
        "goldens = [\n",
        "    \"Mexico: 129,150,971\",\n",
        "    \"23.5\",\n",
        "    \"121, 38.02\",\n",
        "    \"1\",\n",
        "    \"55\",\n",
        "]\n",
        "\n",
        "eval_dataset = pd.DataFrame(\n",
        "    {\n",
        "        \"prompt\": prompts,\n",
        "        \"response\": responses,\n",
        "        \"golden\": goldens,\n",
        "    }\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qm6mHoPc-DS0"
      },
      "source": [
        "# Run evaluation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "mMuDxiwy-MTt"
      },
      "outputs": [],
      "source": [
        "eval_task = EvalTask(\n",
        "    dataset=eval_dataset,\n",
        "    metrics=[agent_final_answer_metric],\n",
        "    autorater_config=AutoraterConfig(sampling_count=1),\n",
        ")\n",
        "eval_result = eval_task.evaluate()\n",
        "\n",
        "# Calculate overall score for metric.\n",
        "compute_metric_score(eval_result.metrics_table)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "e93FLC-xjtZU"
      },
      "outputs": [],
      "source": [
        "pretty_print_df(eval_result.metrics_table)"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "name": "evaluate_agent_final_answer_with_custom_parsing.ipynb",
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}
