{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bkIF-qKfOvFl"
      },
      "outputs": [],
      "source": [
        "# Copyright 2024 Google LLC\n",
        "#\n",
        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
        "# you may not use this file except in compliance with the License.\n",
        "# You may obtain a copy of the License at\n",
        "#\n",
        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
        "#\n",
        "# Unless required by applicable law or agreed to in writing, software\n",
        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
        "# See the License for the specific language governing permissions and\n",
        "# limitations under the License."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KNc5B1-pOvFn"
      },
      "source": [
        "# Use Gen AI Evaluation SDK to Evaluate Models in Vertex AI Studio, Model Garden, and Model Registry\n",
        "\n",
        "<table align=\"left\">\n",
        "  <td style=\"text-align: center\">\n",
        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/generative-ai/blob/main/gemini/evaluation/evaluate_models_in_vertex_ai_studio_and_model_garden.ipynb\">\n",
        "      <img width=\"32px\" src=\"https://www.gstatic.com/pantheon/images/bigquery/welcome_page/colab-logo.svg\" alt=\"Google Colaboratory logo\"><br> Run in Colab\n",
        "    </a>\n",
        "  </td>\n",
        "  <td style=\"text-align: center\">\n",
        "    <a href=\"https://console.cloud.google.com/vertex-ai/colab/import/https:%2F%2Fraw.githubusercontent.com%2FGoogleCloudPlatform%2Fgenerative-ai%2Fmain%2Fgemini%2Fevaluation%2Fevaluate_models_in_vertex_ai_studio_and_model_garden.ipynb\">\n",
        "      <img width=\"32px\" src=\"https://lh3.googleusercontent.com/JmcxdQi-qOpctIvWKgPtrzZdJJK-J3sWE1RsfjZNwshCFgE_9fULcNpuXYTilIR2hjwN\" alt=\"Google Cloud Colab Enterprise logo\"><br> Run in Colab Enterprise\n",
        "    </a>\n",
        "  </td>\n",
        "  <td style=\"text-align: center\">\n",
        "    <a href=\"https://github.com/GoogleCloudPlatform/generative-ai/blob/main/gemini/evaluation/evaluate_models_in_vertex_ai_studio_and_model_garden.ipynb\">\n",
        "      <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\"><br> View on GitHub\n",
        "    </a>\n",
        "  </td>\n",
        "  <td style=\"text-align: center\">\n",
        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/generative-ai/main/gemini/evaluation/evaluate_models_in_vertex_ai_studio_and_model_garden.ipynb\">\n",
        "      <img src=\"https://www.gstatic.com/images/branding/gcpiconscolors/vertexai/v1/32px.svg\" alt=\"Vertex AI logo\"><br> Open in Vertex AI Workbench\n",
        "    </a>\n",
        "  </td>    \n",
        "</table>\n",
        "\n",
        "<div style=\"clear: both;\"></div>\n",
        "\n",
        "<b>Share to:</b>\n",
        "\n",
        "<a href=\"https://www.linkedin.com/sharing/share-offsite/?url=https%3A//github.com/GoogleCloudPlatform/generative-ai/blob/main/gemini/evaluation/evaluate_models_in_vertex_ai_studio_and_model_garden.ipynb\" target=\"_blank\">\n",
        "  <img width=\"20px\" src=\"https://upload.wikimedia.org/wikipedia/commons/8/81/LinkedIn_icon.svg\" alt=\"LinkedIn logo\">\n",
        "</a>\n",
        "\n",
        "<a href=\"https://bsky.app/intent/compose?text=https%3A//github.com/GoogleCloudPlatform/generative-ai/blob/main/gemini/evaluation/evaluate_models_in_vertex_ai_studio_and_model_garden.ipynb\" target=\"_blank\">\n",
        "  <img width=\"20px\" src=\"https://upload.wikimedia.org/wikipedia/commons/7/7a/Bluesky_Logo.svg\" alt=\"Bluesky logo\">\n",
        "</a>\n",
        "\n",
        "<a href=\"https://twitter.com/intent/tweet?url=https%3A//github.com/GoogleCloudPlatform/generative-ai/blob/main/gemini/evaluation/evaluate_models_in_vertex_ai_studio_and_model_garden.ipynb\" target=\"_blank\">\n",
        "  <img width=\"20px\" src=\"https://upload.wikimedia.org/wikipedia/commons/5/5a/X_icon_2.svg\" alt=\"X logo\">\n",
        "</a>\n",
        "\n",
        "<a href=\"https://reddit.com/submit?url=https%3A//github.com/GoogleCloudPlatform/generative-ai/blob/main/gemini/evaluation/evaluate_models_in_vertex_ai_studio_and_model_garden.ipynb\" target=\"_blank\">\n",
        "  <img width=\"20px\" src=\"https://redditinc.com/hubfs/Reddit%20Inc/Brand/Reddit_Logo.png\" alt=\"Reddit logo\">\n",
        "</a>\n",
        "\n",
        "<a href=\"https://www.facebook.com/sharer/sharer.php?u=https%3A//github.com/GoogleCloudPlatform/generative-ai/blob/main/gemini/evaluation/evaluate_models_in_vertex_ai_studio_and_model_garden.ipynb\" target=\"_blank\">\n",
        "  <img width=\"20px\" src=\"https://upload.wikimedia.org/wikipedia/commons/5/51/Facebook_f_logo_%282019%29.svg\" alt=\"Facebook logo\">\n",
        "</a>            "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kPgYxQc1OvFn"
      },
      "source": [
        "| | |\n",
        "|-|-|\n",
        "|Author(s) | [Jason Dai](https://github.com/jsondai) |"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vQ_7WkHE3gNO"
      },
      "source": [
        "This notebook demonstrates how to get started with using the *Vertex AI Python SDK for Gen AI Evaluation Service* for generative models in Vertex AI Studio, Model Garden, and Model Registry.\n",
        "\n",
        "Gen AI Evaluation Service empowers you to comprehensively assess and enhance your generative AI models and applications. Whether you're selecting the ideal model, optimizing prompt templates, or evaluating fine-tuned checkpoints, this service provides the tools and insights you need.\n",
        "\n",
        "In this Colab tutorial, we'll explore three major use cases:\n",
        "\n",
        "1.  Run Evaluation on 1P Models\n",
        "  *   Learn how to evaluate `Gemini` models in Vertex AI Studio using the *Gen AI Evaluation Service SDK*.\n",
        "\n",
        "  *   Explore different evaluation metrics and techniques for assessing performance on various tasks.\n",
        "\n",
        "  *   Discover how to leverage the SDK for in-depth analysis and comparison of `Gemini` model variants.\n",
        "\n",
        "\n",
        "2.  Run Evaluation on 3P Models\n",
        "  *   Learn how to evaluate third-party open models, such as a pretrained `Llama 3.1` model, or a fine-tuned `Llama 3` model deployed in Vertex Model Garden, using the *Gen AI Evaluation Service SDK*.\n",
        "\n",
        "  *   Learn how to evaluate third-party closed model APIs, such as Anthropic's `Claude 3.5 Sonnet` model hosted on Vertex AI, using the *Gen AI Evaluation Service SDK*.\n",
        "\n",
        "  *   Gain insights into conducting controlled experiments by maintaining the same `EvalTask` configuration with fixed dataset and evaluation metrics while evaluating various model architectures and capabilities.\n",
        "\n",
        "\n",
        "3.  Prompt Engineering\n",
        "\n",
        "  *   Explore the impact of prompt design on model performance.\n",
        "  *   Utilize the SDK to systematically evaluate and refine your prompts.\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fHGZmbkw6GgM"
      },
      "source": [
        "For additional use cases and advanced features, refer to our public documentation and notebook tutorials for evaluation use cases:\n",
        "\n",
        "* https://cloud.google.com/vertex-ai/generative-ai/docs/models/evaluation-overview#notebooks_for_evaluation_use_cases\n",
        "\n",
        "* https://cloud.google.com/vertex-ai/generative-ai/docs/models/run-evaluation\n",
        "\n",
        "Let's get started!"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mN5IHo-aOvFo"
      },
      "source": [
        "**_NOTE_**: This notebook has been tested in the following environment:\n",
        "\n",
        "* Python version = 3.10"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3XZf_4VEOvFo"
      },
      "source": [
        "## Getting Started"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kE20na1OOvFo"
      },
      "source": [
        "### Install Vertex AI SDK for Gen AI Evaluation Service"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "abLuRgBzOvFp"
      },
      "outputs": [],
      "source": [
        "%pip install -U -q google-cloud-aiplatform[evaluation]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kYJVHBVSZgTX"
      },
      "source": [
        "### Install other required packages"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "squkG3h9ZZs8"
      },
      "outputs": [],
      "source": [
        "%pip install -U -q datasets anthropic[vertex] openai"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "x1oLkh17OvFp"
      },
      "source": [
        "### Authenticate your notebook environment (Colab only)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9ygOCeYoOvFp"
      },
      "outputs": [],
      "source": [
        "import sys\n",
        "\n",
        "if \"google.colab\" in sys.modules:\n",
        "    from google.colab import auth\n",
        "\n",
        "    auth.authenticate_user()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8wyNclIAOvFp"
      },
      "source": [
        "### Set Google Cloud project information and initialize Vertex AI SDK"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GTL_YzF9OvFq"
      },
      "outputs": [],
      "source": [
        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}\n",
        "LOCATION = \"us-central1\"  # @param {type:\"string\"}\n",
        "\n",
        "if not PROJECT_ID or PROJECT_ID == \"[your-project-id]\":\n",
        "    raise ValueError(\"Please set your PROJECT_ID\")\n",
        "\n",
        "\n",
        "import vertexai\n",
        "\n",
        "vertexai.init(project=PROJECT_ID, location=LOCATION)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UBQgjn5wOvFq"
      },
      "source": [
        "### Import libraries"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-TmyCxUSOvFq"
      },
      "outputs": [],
      "source": [
        "import openai\n",
        "import pandas as pd\n",
        "from anthropic import AnthropicVertex"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gXkKKzk8OvFq"
      },
      "source": [
        "## Load an evaluation dataset\n",
        "\n",
        "Load a subset of the `OpenOrca` dataset using the `huggingface/datasets` library. We will use 10 samples from the first 100 rows of the \"train\" split of `OpenOrca` dataset to demonstrate evaluating prompts and model responses in this Colab.\n",
        "\n",
        "### Dataset Summary\n",
        "\n",
        "The OpenOrca dataset is a collection of augmented [FLAN Collection data](https://arxiv.org/abs/2301.13688). Currently ~1M GPT-4 completions, and ~3.2M GPT-3.5 completions. It is tabularized in alignment with the distributions presented in the ORCA paper and currently represents a partial completion of the full intended dataset, with ongoing generation to expand its scope. The data is primarily used for training and evaluation in the field of natural language processing.\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "w-j7l4Qd0Ull"
      },
      "outputs": [],
      "source": [
        "from datasets import load_dataset\n",
        "from google.auth import default, transport\n",
        "from vertexai.evaluation import (\n",
        "    EvalTask,\n",
        "    MetricPromptTemplateExamples,\n",
        "    PairwiseMetric,\n",
        "    PointwiseMetric,\n",
        "    PointwiseMetricPromptTemplate,\n",
        ")\n",
        "from vertexai.generative_models import GenerativeModel\n",
        "from vertexai.preview.evaluation import notebook_utils\n",
        "\n",
        "ds = (\n",
        "    load_dataset(\n",
        "        \"Open-Orca/OpenOrca\",\n",
        "        data_files=\"1M-GPT4-Augmented.parquet\",\n",
        "        split=\"train[:100]\",\n",
        "    )\n",
        "    .to_pandas()\n",
        "    .drop(columns=[\"id\"])\n",
        "    .rename(columns={\"response\": \"reference\"})\n",
        ")\n",
        "\n",
        "dataset = ds.sample(n=10)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kOeVlflg06F3"
      },
      "source": [
        "#### Preview the dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "zAHGlmkEelhm"
      },
      "outputs": [],
      "source": [
        "dataset.head()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kgrS1Ljo0VVQ"
      },
      "source": [
        "## Run evaluation on 1P models\n",
        "\n",
        "The *Gen AI Evaluation Service SDK* includes native support for evaluating Gemini models. This streamlines the evaluation process for Google's latest and most capable family of large language models. With minimal coding effort, you can leverage pre-defined metrics and workflows to assess the performance of Gemini models on various tasks. You can also customize your own model-based metrics based on your specific evaluation criteria.\n",
        "\n",
        "This enhanced support enables you to:\n",
        "\n",
        "- **Quickly evaluate Gemini models:** Effortlessly assess the performance of Gemini models using the SDK's streamlined workflows.\n",
        "- **Compare models side-by-side:** Benchmark Gemini against other models to understand relative strengths and weaknesses.\n",
        "- **Analyze prompt templates:** Evaluate the effectiveness of different prompt designs for optimizing Gemini's performance.\n",
        "\n",
        "This native integration simplifies the evaluation process, allowing you to focus on understanding and improving the capabilities of Gemini."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "011966ad51c4"
      },
      "source": [
        "#### Understand the `EvalTask` class\n",
        "\n",
        "The `EvalTask` class is a core component of the *Gen AI Evaluation Service SDK* framework. It allows you to define and run evaluation jobs against your Gen AI models/applications, providing a structured way to measure their performance on specific tasks. Think of an `EvalTask` as a blueprint for your evaluation process.\n",
        "\n",
        "`EvalTask` class requires an evaluation dataset and a list of metrics. Supported metrics are documented on the Generative AI on Vertex AI [Define your evaluation metrics](https://cloud.google.com/vertex-ai/generative-ai/docs/models/determine-eval) page. The dataset can be an `pandas.DataFrame`, Python dictionary or a file path URI and can contain default column names such as `prompt`, `reference`, `response`, and `baseline_model_response`. \n",
        "\n",
        "* **Bring-your-own-response (BYOR):** You already have the data that you want to evaluate stored in the dataset. You can customize the response column names for both your model and the baseline model using parameters like `response_column_name` and `baseline_model_response_column_name` or through the `metric_column_mapping`.\n",
        "\n",
        "* **Perform model inference without a prompt template:** You have a dataset containing the input prompts to the model and want to perform model inference before evaluation. A column named `prompt` is required in the evaluation dataset and is used directly as input to the model.\n",
        "\n",
        "* **Perform model inference with a prompt template:** You have a dataset containing the input variables to the prompt template and want to assemble the prompts for model inference. Evaluation dataset must contain column names corresponding to the variable names in the prompt template. For example, if prompt template is \"Instruction: {instruction}, context: {context}\", the dataset must contain `instruction` and `context` columns.\n",
        "\n",
        "\n",
        "\n",
        "`EvalTask` supports extensive evaluation scenarios including BYOR, model inference with Gemini models, 3P models endpoints/SDK clients, or custom model generation functions, using computation-based metrics, model-based pointwise and pairwise metrics. The `evaluate()` method triggers the evaluation process, optionally taking a model, prompt template, experiment logging configurations, and other evaluation run configurations. You can view the SDK reference documentation for [Gen AI Evaluation package](https://cloud.google.com/vertex-ai/generative-ai/docs/reference/python/latest/vertexai.evaluation) for more details."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "noHGa6rD6-ks"
      },
      "source": [
        "### Define a model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ol1fa0hI7Irr"
      },
      "outputs": [],
      "source": [
        "# Model to be evaluated\n",
        "model = GenerativeModel(\n",
        "    \"gemini-2.0-flash\",\n",
        "    generation_config={\"temperature\": 0.6, \"max_output_tokens\": 256, \"top_k\": 1},\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xuvdzrUIayPp"
      },
      "source": [
        "### Use computation-based metrics\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JrY-NLGg0TVS"
      },
      "outputs": [],
      "source": [
        "# Define an EvalTask with ROUGE-L-SUM metric\n",
        "rouge_eval_task = EvalTask(\n",
        "    dataset=dataset,\n",
        "    metrics=[\"rouge_l_sum\"],\n",
        ")\n",
        "rouge_result = rouge_eval_task.evaluate(\n",
        "    model=model,\n",
        "    prompt_template=\"# System_prompt\\n{system_prompt} # Question\\n{question}\",\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "FSMRnDMixwLS"
      },
      "outputs": [],
      "source": [
        "notebook_utils.display_eval_result(rouge_result)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gx9pKjiSbHag"
      },
      "source": [
        "### Use model-based pointwise metrics"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WuRlYbj0HvBB"
      },
      "outputs": [],
      "source": [
        "# @title Select a pointwise metric to use\n",
        "\n",
        "import ipywidgets as widgets\n",
        "\n",
        "pointwise_single_turn_metrics = [\n",
        "    metric\n",
        "    for metric in MetricPromptTemplateExamples.list_example_metric_names()\n",
        "    if not metric.startswith(\"pairwise\") and not metric.startswith(\"multi_turn\")\n",
        "]\n",
        "\n",
        "dropdown = widgets.Dropdown(\n",
        "    options=pointwise_single_turn_metrics,\n",
        "    description=\"Select a metric:\",\n",
        "    font_weight=\"bold\",\n",
        "    style={\"description_width\": \"initial\"},\n",
        ")\n",
        "\n",
        "\n",
        "def dropdown_eventhandler(change):\n",
        "    global POINTWISE_METRIC\n",
        "    if change[\"type\"] == \"change\" and change[\"name\"] == \"value\":\n",
        "        POINTWISE_METRIC = change.new\n",
        "        print(\"Selected:\", change.new)\n",
        "\n",
        "\n",
        "POINTWISE_METRIC = dropdown.value\n",
        "dropdown.observe(dropdown_eventhandler, names=\"value\")\n",
        "display(dropdown)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "kBnyzogabOmp"
      },
      "outputs": [],
      "source": [
        "pointwise_result = EvalTask(\n",
        "    dataset=dataset,\n",
        "    metrics=[POINTWISE_METRIC],\n",
        ").evaluate(\n",
        "    model=model,\n",
        "    prompt_template=\"# System_prompt\\n{system_prompt} # Question\\n{question}\",\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0EwLaPusyeL9"
      },
      "outputs": [],
      "source": [
        "notebook_utils.display_eval_result(pointwise_result)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "qsPD-YUWoRYo"
      },
      "outputs": [],
      "source": [
        "notebook_utils.display_explanations(pointwise_result, num=1, metrics=[POINTWISE_METRIC])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9AzW8Zg71Igs"
      },
      "source": [
        "#### Build your own pointwise metric\n",
        "\n",
        "For more information about metric customization, see this [notebook tutorial](https://colab.research.google.com/github/GoogleCloudPlatform/generative-ai/blob/main/gemini/evaluation/customize_model_based_metrics.ipynb)."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1MJdLUJz6H5F"
      },
      "outputs": [],
      "source": [
        "pointwise_result = EvalTask(\n",
        "    dataset=dataset,\n",
        "    metrics=[\n",
        "        # Create a unique model-based metric for your own use cases\n",
        "        PointwiseMetric(\n",
        "            metric=\"linguistic_acceptability\",\n",
        "            metric_prompt_template=PointwiseMetricPromptTemplate(\n",
        "                criteria={\n",
        "                    \"Proper Grammar\": \"The language's grammar rules are correctly followed, including but not limited to sentence structures, verb tenses, subject-verb agreement, proper punctuation, and capitalization.\",\n",
        "                    \"Appropriate word choice\": \"Words chosen are appropriate and purposeful given their relative context and positioning in the text. Vocabulary demonstrates prompt understanding.\",\n",
        "                    \"Reference Alignment\": \"The response is consistent and aligned with the reference.\",\n",
        "                },\n",
        "                rating_rubric={\n",
        "                    \"5\": \"Excellent: The writing is grammatically correct, uses appropriate vocabulary and aligns perfectly with the reference.\",\n",
        "                    \"4\": \"Good: The writing is generally grammatically correct, uses appropriate vocabulary and aligns well with the reference.\",\n",
        "                    \"3\": \"Satisfactory: The writing may have minor grammatical errors or use less-appropriate vocabulary, but it aligns reasonably well with the reference.\",\n",
        "                    \"2\": \"Unsatisfactory: The writing has significant grammatical errors, uses inappropriate vocabulary, deviates significantly from the reference.\",\n",
        "                    \"1\": \"Poor: The writing is riddled with grammatical errors, uses highly inappropriate vocabulary, is completely unrelated to the reference.\",\n",
        "                },\n",
        "                input_variables=[\"prompt\", \"reference\"],\n",
        "            ),\n",
        "        )\n",
        "    ],\n",
        ").evaluate(\n",
        "    model=model,\n",
        "    prompt_template=\"# System_prompt\\n{system_prompt} # Question\\n{question}\",\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "UW5DXIGv7oIJ"
      },
      "outputs": [],
      "source": [
        "notebook_utils.display_eval_result(pointwise_result)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sFHRuWjv86uG"
      },
      "source": [
        "### Use model-based pairwise metrics\n",
        "\n",
        "Evaluate two Gen AI models side-by-side (SxS) with model-based pairwise metrics."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "E7Eqh9e9MDsa"
      },
      "outputs": [],
      "source": [
        "# @title Select a pairwise metric to use\n",
        "\n",
        "import ipywidgets as widgets\n",
        "from IPython.display import display\n",
        "\n",
        "pairwise_single_turn_metrics = [\n",
        "    metric\n",
        "    for metric in MetricPromptTemplateExamples.list_example_metric_names()\n",
        "    if metric.startswith(\"pairwise\") and \"multi_turn\" not in metric\n",
        "]\n",
        "\n",
        "dropdown = widgets.Dropdown(\n",
        "    options=pairwise_single_turn_metrics,\n",
        "    description=\"Select a metric:\",\n",
        "    font_weight=\"bold\",\n",
        "    style={\"description_width\": \"initial\"},\n",
        ")\n",
        "\n",
        "\n",
        "def dropdown_eventhandler(change):\n",
        "    global PAIRWISE_METRIC_NAME\n",
        "    if change[\"type\"] == \"change\" and change[\"name\"] == \"value\":\n",
        "        PAIRWISE_METRIC_NAME = change.new\n",
        "        print(\"Selected:\", change.new)\n",
        "\n",
        "\n",
        "PAIRWISE_METRIC_NAME = dropdown.value\n",
        "dropdown.observe(dropdown_eventhandler, names=\"value\")\n",
        "display(dropdown)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rJd6sF7re5gC"
      },
      "source": [
        "#### Run SxS evaluation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "sDAaYDqY9FBu"
      },
      "outputs": [],
      "source": [
        "pairwise_result = EvalTask(\n",
        "    dataset=dataset,\n",
        "    metrics=[\n",
        "        PairwiseMetric(\n",
        "            metric=PAIRWISE_METRIC_NAME,\n",
        "            metric_prompt_template=MetricPromptTemplateExamples.get_prompt_template(\n",
        "                PAIRWISE_METRIC_NAME\n",
        "            ),\n",
        "            # Define a baseline model to compare against\n",
        "            baseline_model=GenerativeModel(\"gemini-2.0-flash-lite\"),\n",
        "        )\n",
        "    ],\n",
        ").evaluate(\n",
        "    # Specify a candidate model for pairwise comparison\n",
        "    model=model,\n",
        "    prompt_template=\"# System_prompt\\n{system_prompt} # Question\\n{question}\",\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bPAoJu53-mfx"
      },
      "outputs": [],
      "source": [
        "notebook_utils.display_eval_result(\n",
        "    pairwise_result,\n",
        "    title=\"Gemini-2.0-Flash vs. Gemini-2.0-Flash-Lite SxS Pairwise Evaluation Results\",\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "olbyG8ZVOvFq"
      },
      "source": [
        "## Run evaluation on 3P Models\n",
        "\n",
        "The *Vertex Gen AI Evaluation Service SDK* provides robust support for evaluating third-party (3P) models, allowing you to assess and compare models from various sources. The *Gen AI Evaluation Service SDK* allows you to provide a generic Python function as input to specify how the model/application should be invoked for batch inference, which could be done through an endpoint or an SDK. This flexible approach accommodates a wide range of open and closed models.\n",
        "\n",
        "**Open Models:**\n",
        "\n",
        "Evaluate open models like a pre-trained `Llama 3.1` or a fine-tuned `Llama 3` models deployed with Vertex AI Model Garden using the *Gen AI Evaluation Service SDK*. This enables you to assess the performance of these models and understand how they align with your specific requirements.\n",
        "\n",
        "**Closed Models:**\n",
        "\n",
        "Evaluate the performance of closed model APIs, such as Anthropic's `Claude 3.5 Sonnet`, hosted on Vertex AI. This allows you to compare the capabilities of different closed models and make informed decisions about which best suits your needs."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HXfM_yJwiuh3"
      },
      "outputs": [],
      "source": [
        "# Define an EvalTask with a list of metrics\n",
        "pointwise_eval_task = EvalTask(\n",
        "    dataset=dataset,\n",
        "    metrics=[\n",
        "        \"coherence\",\n",
        "        \"fluency\",\n",
        "        \"instruction_following\",\n",
        "        \"text_quality\",\n",
        "        \"rouge_l_sum\",\n",
        "    ],\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UGokrtdiIHrX"
      },
      "source": [
        "### Run Evaluation on Llama 3.1 API Endpoint\n",
        "\n",
        "You can experiment with various supported Llama models.\n",
        "\n",
        "This tutorial use Llama 3 8B Instruct, 70B Instruct, and 405B Instruct using Model-as-a-Service (MaaS). Using Model-as-a-Service (MaaS), you can access Llama 3.1 models in just a few clicks without any setup or infrastructure hassles. Model-as-a-Service (MaaS) integrates [Llama Guard](https://huggingface.co/meta-llama/Llama-Guard-3-8B) as a safety filter. It is switched on by default and can be switched off. Llama Guard enables us to safeguard model inputs and outputs. If a response is filtered, it will be populated with a `finish_reason` field (with value `content_filtered`) and a `refusal` field (stating the filtering reason).\n",
        "\n",
        "You can also access Llama models for self-service in Vertex AI Model Garden, allowing you to choose your preferred infrastructure. [Check out Llama 3.1 model card](https://console.cloud.google.com/vertex-ai/publishers/meta/model-garden/llama3_1?_ga=2.31261500.2048242469.1721714335-1107467625.1721655511) to learn how to deploy a Llama 3.1 models on Vertex AI."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "r7OhyH46H2H5"
      },
      "outputs": [],
      "source": [
        "# fmt: off\n",
        "MODEL_ID = \"meta/llama-3.1-8b-instruct-maas\"  # @param {type:\"string\"} [\"meta/llama-3.1-8b-instruct-maas\", \"meta/llama-3.1-70b-instruct-maas\", \"meta/llama-3.1-405b-instruct-maas\"]\n",
        "# fmt: on"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KUgMmggOa_TG"
      },
      "source": [
        "#### Authentication\n",
        "\n",
        "You can request an access token from the default credentials for the current environment. Note that the access token lives for [1 hour by default](https://cloud.google.com/docs/authentication/token-types#at-lifetime); after expiration, it must be refreshed.\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OT03q0Oqa86s"
      },
      "outputs": [],
      "source": [
        "credentials, _ = default()\n",
        "auth_request = transport.requests.Request()\n",
        "credentials.refresh(auth_request)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PtpB2ig9bCxs"
      },
      "source": [
        "Then configure the OpenAI SDK to point to the Llama 3.1 API endpoint.\n",
        "\n",
        "Note: only `us-central1` is supported region for Llama 3.1 models using Model-as-a-Service (MaaS)."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5TazdE-CbEdW"
      },
      "outputs": [],
      "source": [
        "MODEL_LOCATION = \"us-central1\"\n",
        "\n",
        "client = openai.OpenAI(\n",
        "    base_url=f\"https://{MODEL_LOCATION}-aiplatform.googleapis.com/v1beta1/projects/{PROJECT_ID}/locations/{MODEL_LOCATION}/endpoints/openapi/chat/completions?\",\n",
        "    api_key=credentials.token,\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zMPNCDTsbqwc"
      },
      "source": [
        "#### Set model configurations for Llama 3.1\n",
        "\n",
        "Use the following parameters to generate different answers:\n",
        "\n",
        "*   `temperature` to control the randomness of the response\n",
        "*   `max_tokens` to limit the response length\n",
        "*   `top_p` to control the quality of the response"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "id": "VxQbGCHHbnot"
      },
      "outputs": [],
      "source": [
        "temperature = 1.0  # @param {type:\"number\"}\n",
        "max_tokens = 50  # @param {type:\"integer\"}\n",
        "top_p = 1.0  # @param {type:\"number\"}\n",
        "apply_llama_guard = True  # @param {type:\"boolean\"}\n",
        "\n",
        "response = client.chat.completions.create(\n",
        "    model=MODEL_ID,\n",
        "    messages=[{\"role\": \"user\", \"content\": \"Hello, Llama 3.1!\"}],\n",
        "    extra_body={\n",
        "        \"extra_body\": {\n",
        "            \"google\": {\n",
        "                \"model_safety_settings\": {\n",
        "                    \"enabled\": apply_llama_guard,\n",
        "                    \"llama_guard_settings\": {},\n",
        "                }\n",
        "            }\n",
        "        }\n",
        "    },\n",
        ")\n",
        "print(response.choices[0].message.content)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wqoSsvMlcYPM"
      },
      "source": [
        "#### Define the Llama 3.1 Model Function"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CdiP_zMWbzp0"
      },
      "outputs": [],
      "source": [
        "def llama_model_fn(prompt: str) -> str:\n",
        "    response = client.chat.completions.create(\n",
        "        model=MODEL_ID,\n",
        "        messages=[{\"role\": \"user\", \"content\": prompt}],\n",
        "        temperature=temperature,\n",
        "        max_tokens=max_tokens,\n",
        "        top_p=top_p,\n",
        "        extra_body={\n",
        "            \"extra_body\": {\n",
        "                \"google\": {\n",
        "                    \"model_safety_settings\": {\n",
        "                        \"enabled\": apply_llama_guard,\n",
        "                        \"llama_guard_settings\": {},\n",
        "                    }\n",
        "                }\n",
        "            }\n",
        "        },\n",
        "    )\n",
        "    return response.choices[0].message.content"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nmtUTC1jTZT2"
      },
      "source": [
        "#### Run evaluation on Llama 3.1 API Service"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1are_m2DTZT2"
      },
      "outputs": [],
      "source": [
        "llama_result = pointwise_eval_task.evaluate(\n",
        "    model=llama_model_fn,\n",
        "    prompt_template=\"# System_prompt\\n{system_prompt} # Question\\n{question}\",\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gD3zu0LLTZT2"
      },
      "outputs": [],
      "source": [
        "notebook_utils.display_eval_result(\n",
        "    llama_result, title=\"Llama 3.1 API Service Evaluation Results\"\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BUWSXVToTZT2"
      },
      "outputs": [],
      "source": [
        "notebook_utils.display_explanations(llama_result, num=1)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qS2BpbzMIpZc"
      },
      "source": [
        "### Run Evaluation on Claude 3\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "id": "4j9taDaPJHy9"
      },
      "outputs": [],
      "source": [
        "# fmt: off\n",
        "MODEL = \"claude-3-5-sonnet@20240620\"  # @param [\"claude-3-5-sonnet@20240620\", \"claude-3-opus@20240229\", \"claude-3-haiku@20240307\", \"claude-3-sonnet@20240229\" ]\n",
        "# fmt: on\n",
        "if MODEL == \"claude-3-5-sonnet@20240620\":\n",
        "    available_regions = [\"europe-west1\", \"us-east5\"]\n",
        "elif MODEL == \"claude-3-opus@20240229\":\n",
        "    available_regions = [\"us-east5\"]\n",
        "elif MODEL == \"claude-3-haiku@20240307\":\n",
        "    available_regions = [\"us-east5\", \"europe-west1\"]\n",
        "elif MODEL == \"claude-3-sonnet@20240229\":\n",
        "    available_regions = [\"us-east5\"]\n",
        "\n",
        "dropdown = widgets.Dropdown(\n",
        "    options=available_regions,\n",
        "    description=\"Select a location:\",\n",
        "    font_weight=\"bold\",\n",
        "    style={\"description_width\": \"initial\"},\n",
        ")\n",
        "\n",
        "\n",
        "def dropdown_eventhandler(change):\n",
        "    global LOCATION\n",
        "    if change[\"type\"] == \"change\" and change[\"name\"] == \"value\":\n",
        "        LOCATION = change.new\n",
        "        print(\"Selected:\", change.new)\n",
        "\n",
        "\n",
        "LOCATION = dropdown.value\n",
        "dropdown.observe(dropdown_eventhandler, names=\"value\")\n",
        "display(dropdown)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lVbFk9pSd6L0"
      },
      "source": [
        "#### Define the Claude 3 Model Function"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jKLjm0o3KwEI"
      },
      "outputs": [],
      "source": [
        "def anthropic_claude_model_fn(prompt):\n",
        "    client = AnthropicVertex(region=LOCATION, project_id=PROJECT_ID)\n",
        "    message = client.messages.create(\n",
        "        max_tokens=1024,\n",
        "        messages=[\n",
        "            {\n",
        "                \"role\": \"user\",\n",
        "                \"content\": f\"{prompt}\",\n",
        "            }\n",
        "        ],\n",
        "        model=MODEL,\n",
        "    )\n",
        "    response = message.content[0].text\n",
        "    return response"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "N73OcARGgMUT"
      },
      "source": [
        "#### Run evaluation on Claude 3 Model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gdGUsc7gIpZi"
      },
      "outputs": [],
      "source": [
        "claude_result = pointwise_eval_task.evaluate(\n",
        "    model=anthropic_claude_model_fn,\n",
        "    prompt_template=\"# System_prompt\\n{system_prompt} # Question\\n{question}\",\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_URkaZhYIpZi"
      },
      "outputs": [],
      "source": [
        "notebook_utils.display_eval_result(claude_result, title=\"Claude 3 Evaluation Results\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "38IJUEXthXsE"
      },
      "outputs": [],
      "source": [
        "notebook_utils.display_explanations(claude_result, num=1)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yLHuqe2D-x81"
      },
      "source": [
        "## Prompt Engineering"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cd393d80e252"
      },
      "source": [
        "The *Vertex AI Gen AI Evaluation Service SDK* simplifies prompt engineering by streamlining the process of creating and evaluating multiple prompt templates. It allows you to efficiently test different prompts against a chosen dataset and compare their performance using comprehensive evaluation metrics. This empowers you to identify the most effective prompts for your specific use case and optimize your generative AI applications."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5261f69b615a"
      },
      "source": [
        "###  Compare and optimize prompt template design"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7d01f70c3163"
      },
      "source": [
        "#### Define an evaluation dataset\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2157353f3309"
      },
      "source": [
        "To perform pointwise inference, the evaluation dataset is required to contain the following fields:\n",
        "\n",
        "* Instruction: Part of the input user prompt. It refers to the inference instruction that is sent to your LLM.\n",
        "* Context: User input for the Gen AI model or application in the current turn.\n",
        "* Reference: The ground truth to compare your LLM response to.\n",
        "\n",
        "Your dataset must include a minimum of one evaluation example. We recommend around 100 examples to ensure high-quality aggregated metrics and statistically significant results."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "972ebabb2712"
      },
      "outputs": [],
      "source": [
        "instruction = \"Summarize the following article: \\n\"\n",
        "\n",
        "context = [\n",
        "    \"Typhoon Phanfone has killed at least one person, a US airman on Okinawa who was washed away by high waves. Thousands of households have lost power and Japan's two largest airlines have suspended many flights. The storm also forced the suspension of the search for people missing after last week's volcanic eruption. The storm-tracking website Tropical Storm Risk forecasts that Phanfone will rapidly lose power over the next few hours as it goes further into the Pacific Ocean. Typhoon Phanfone was downgraded from an earlier status of a super typhoon, but the Japan Meteorological Agency had warned it was still a dangerous storm. Japan averages 11 typhoons a year, according to its weather agency. The typhoon made landfall on Monday morning near the central city of Hamamatsu, with winds of up to 180 km/h (112 mph). The airman was one of three US military personnel swept away by high waves whipped up by the typhoon off southern Okinawa island, where the US has a large military base. The remaining two are still missing. A police spokesman said they had been taking photographs of the sea. A university student who was surfing off the seas of Kanagawa Prefecture, south of Tokyo, was also missing, national broadcast NHK reports. It said at least 10 people had been injured and 9,500 houses were without power. The storm was expected to deposit about 100mm of rain on Tokyo over 24 hours, according to the Transport Ministry website. Many schools were closed on Monday and two car companies in Japan halted production at some plants ahead of the storm. More than 174 domestic flights were affected nationwide, NHK state broadcaster said on Sunday. On Sunday, heavy rain delayed the Japanese Formula One Grand Prix in Suzaka. French driver Jules Bianchi lost control in the wet conditions and crashed, sustaining a severe head injury.\",\n",
        "    \"The blaze started at the detached building in Drivers End in Codicote, near Welwyn, during the morning. There was another fire at the building 20 years ago, after which fire-proof foil was placed under the thatch, which is protecting the main building. More than 15 fire engines and support vehicles were called to tackle the blaze. Roads in the area were closed and traffic diverted.\",\n",
        "    'The 18-year-old fell at the New Charter Academy on Broadoak Road in Ashton-under-Lyne at about 09:10 BST, Greater Manchester Police (GMP) said. GMP said he had gone to Manchester Royal Infirmary and his condition was \"serious\". Principal Jenny Langley said the school would remain \"fully open\" while police investigated. \"Our thoughts are with the family and we\\'re doing everything we can to support them along with staff and pupils,\" she said.',\n",
        "    'But Belgian-born Dutchman Max Verstappen was unable to drive a car legally on his own in either country. That all changed on Wednesday when the youngster turned 18 and passed his driving test at the first attempt. Despite having competed in 14 grands prix since his debut in Australia in March, Verstappen admitted to feeling the pressure during his test. \"It\\'s a relief,\" said the Toro Rosso driver, who finished ninth in Japan on Sunday and had only started driving lessons a week ago. \"I was a bit nervous to make mistakes, but the exam went well.\" A bonus of turning 18 is that Verstappen will now be able to drink the champagne if he ever makes it onto the podium.',\n",
        "]\n",
        "\n",
        "reference = [\n",
        "    \"A powerful typhoon has brought many parts of Japan to a standstill and briefly battered Tokyo before heading out to sea.\",\n",
        "    \"A major fire has been burning in the thatched roof of a large property in Hertfordshire.\",\n",
        "    \"A student has been taken to hospital after falling from a balcony at a Greater Manchester school.\",\n",
        "    \"He is Formula 1's youngest ever driver and in charge of a car that can reach over 200mph.\",\n",
        "]\n",
        "\n",
        "response = [\n",
        "    \"Typhoon Phanfone, while downgraded from super typhoon status, caused significant disruption and tragedy in Japan. One US airman died after being swept away by high waves, with two more missing. The storm caused power outages for thousands, flight cancellations, and the suspension of rescue efforts for missing volcano victims. Heavy rain and strong winds led to school and factory closures, transportation disruptions, and at least 10 injuries. The typhoon is expected to weaken as it moves over the Pacific Ocean.\",\n",
        "    \"A large fire broke out in a detached thatched building in Codicote, near Welwyn. This is the second fire at the building in 20 years. Thankfully, fire-proof foil installed after the previous fire is protecting the main building. Over 15 fire engines and support vehicles responded, closing roads and diverting traffic in the area.\",\n",
        "    \"An 18-year-old student at New Charter Academy in Ashton-under-Lyne suffered a serious fall and was hospitalized. The incident is under investigation by Greater Manchester Police, but the school remains open. The principal expressed support for the student's family and the school community.\",\n",
        "    \"Max Verstappen, a Formula One driver, was finally able to get his driver's license at age 18. Despite already competing in 14 Grand Prix races, he was not of legal driving age in his native countries. He admitted to being nervous but passed the test on his first attempt.  As an added bonus of turning 18, Verstappen can now enjoy champagne on the podium if he places.\",\n",
        "]\n",
        "\n",
        "eval_dataset = pd.DataFrame(\n",
        "    {\n",
        "        \"instruction\": instruction,\n",
        "        \"context\": context,\n",
        "        \"reference\": reference,\n",
        "    }\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "c3c1121684f5"
      },
      "source": [
        "#### Define an EvalTask"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "052ec86e5777"
      },
      "outputs": [],
      "source": [
        "EXPERIMENT_NAME = \"eval-sdk-prompt-engineering\"  # @param {type:\"string\"}\n",
        "\n",
        "summarization_eval_task = EvalTask(\n",
        "    dataset=eval_dataset,\n",
        "    metrics=[\n",
        "        \"rouge_l_sum\",\n",
        "        \"bleu\",\n",
        "        \"fluency\",\n",
        "        \"coherence\",\n",
        "        \"safety\",\n",
        "        \"groundedness\",\n",
        "        \"summarization_quality\",\n",
        "        \"verbosity\",\n",
        "        \"instruction_following\",\n",
        "        \"text_quality\",\n",
        "    ],\n",
        "    experiment=EXPERIMENT_NAME,\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "515fe8c3652f"
      },
      "source": [
        "#### Run Evaluation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2b8422213915"
      },
      "outputs": [],
      "source": [
        "# Define prompt templates to compare\n",
        "prompt_templates = [\n",
        "    \"Instruction: {instruction} such that you're explaining it to a 5 year old. Article: {context}. Summary:\",\n",
        "    \"Article: {context}. Complete this task: {instruction}. Summary:\",\n",
        "    \"Goal: {instruction} and give me a TL;DR in five words. Here's an article: {context}. Summary:\",\n",
        "    \"Article: {context}. Reference Summary: {reference}. {instruction} to be more concise and verbose than the reference.\",\n",
        "]\n",
        "\n",
        "eval_results = []\n",
        "for i, prompt_template in enumerate(prompt_templates):\n",
        "    eval_result = summarization_eval_task.evaluate(\n",
        "        prompt_template=prompt_template,\n",
        "        model=GenerativeModel(\n",
        "            \"gemini-2.0-flash\",\n",
        "            generation_config={\n",
        "                \"temperature\": 0.3,\n",
        "                \"max_output_tokens\": 256,\n",
        "                \"top_k\": 1,\n",
        "            },\n",
        "        ),\n",
        "        # Customize eval service rate limit to improve speed.\n",
        "        # See more details in https://cloud.google.com/vertex-ai/generative-ai/docs/models/run-evaluation#increase-quota\n",
        "        evaluation_service_qps=5,\n",
        "    )\n",
        "\n",
        "    eval_results.append((f\"Prompt Template #{i + 1}\", eval_result))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ec6e97cf9f27"
      },
      "source": [
        "#### Display Evaluation report and explanations"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "d3724b416b39"
      },
      "outputs": [],
      "source": [
        "for title, eval_result in eval_results:\n",
        "    notebook_utils.display_eval_result(title=title, eval_result=eval_result)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0615d0925469"
      },
      "outputs": [],
      "source": [
        "for title, eval_result in eval_results:\n",
        "    notebook_utils.display_explanations(\n",
        "        eval_result, metrics=[\"summarization_quality\"], num=2\n",
        "    )"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GzWSUPj2oV-_"
      },
      "source": [
        "#### Visualize Results"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "SSDyxE2aoYMH"
      },
      "outputs": [],
      "source": [
        "notebook_utils.display_radar_plot(\n",
        "    eval_results,\n",
        "    metrics=[\"instruction_following\", \"fluency\", \"coherence\", \"text_quality\"],\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Id5sjYHboZHh"
      },
      "outputs": [],
      "source": [
        "notebook_utils.display_bar_plot(\n",
        "    eval_results,\n",
        "    metrics=[\"instruction_following\", \"fluency\", \"coherence\", \"text_quality\"],\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ed966819648e"
      },
      "source": [
        "####  View Experiment log for evaluation runs"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "363c1b2553b9"
      },
      "outputs": [],
      "source": [
        "summarization_eval_task.display_runs()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "d7cce28cc97e"
      },
      "source": [
        "## Cleaning up"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ef73672573e1"
      },
      "outputs": [],
      "source": [
        "delete_experiment = True\n",
        "\n",
        "# Please set your LOCATION to the same one used during Vertex AI SDK initialization.\n",
        "LOCATION = \"us-central1\"  # @param {type:\"string\"}\n",
        "\n",
        "if delete_experiment:\n",
        "    from google.cloud import aiplatform\n",
        "\n",
        "    aiplatform.init(project=PROJECT_ID, location=LOCATION)\n",
        "    experiment = aiplatform.Experiment(EXPERIMENT_NAME)\n",
        "    experiment.delete()"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [
        "tfQ7sPtOjZOw",
        "F_Gw6YLeOvFq"
      ],
      "name": "evaluate_models_in_vertex_ai_studio_and_model_garden.ipynb",
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}
