{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "f010346b5720"
      },
      "outputs": [],
      "source": [
        "# Copyright 2024 Google LLC\n",
        "#\n",
        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
        "# you may not use this file except in compliance with the License.\n",
        "# You may obtain a copy of the License at\n",
        "#\n",
        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
        "#\n",
        "# Unless required by applicable law or agreed to in writing, software\n",
        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
        "# See the License for the specific language governing permissions and\n",
        "# limitations under the License.\n",
        "# Getting Started with Vertex AI Python SDK for Gen AI Evaluation Service"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "de06e24084b6"
      },
      "source": [
        "<table align=\"left\">\n",
        "  <td style=\"text-align: center\">\n",
        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/generative-ai/blob/main/gemini/evaluation/multi_agent_evals_with_arize_and_crewai.ipynb\">\n",
        "      <img width=\"32px\" src=\"https://www.gstatic.com/pantheon/images/bigquery/welcome_page/colab-logo.svg\" alt=\"Google Colaboratory logo\"><br> Open in Colab\n",
        "    </a>\n",
        "  </td>\n",
        "  <td style=\"text-align: center\">\n",
        "    <a href=\"https://console.cloud.google.com/vertex-ai/colab/import/https:%2F%2Fraw.githubusercontent.com%2FGoogleCloudPlatform%2Fgenerative-ai%2Fmain%2Fgemini%2Fevaluation%2Fmulti_agent_evals_with_arize_and_crewai.ipynb\">\n",
        "      <img width=\"32px\" src=\"https://lh3.googleusercontent.com/JmcxdQi-qOpctIvWKgPtrzZdJJK-J3sWE1RsfjZNwshCFgE_9fULcNpuXYTilIR2hjwN\" alt=\"Google Cloud Colab Enterprise logo\"><br> Open in Colab Enterprise\n",
        "    </a>\n",
        "  </td>\n",
        "  <td style=\"text-align: center\">\n",
        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/generative-ai/main/gemini/evaluation/multi_agent_evals_with_arize_and_crewai.ipynb\">\n",
        "      <img src=\"https://www.gstatic.com/images/branding/gcpiconscolors/vertexai/v1/32px.svg\" alt=\"Vertex AI logo\"><br> Open in Vertex AI Workbench\n",
        "    </a>\n",
        "  </td>\n",
        "  <td style=\"text-align: center\">\n",
        "    <a href=\"https://github.com/GoogleCloudPlatform/generative-ai/blob/main/gemini/evaluation/multi_agent_evals_with_arize_and_crewai.ipynb\">\n",
        "      <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\"><br> View on GitHub\n",
        "    </a>\n",
        "  </td>\n",
        "</table>\n",
        "\n",
        "<div style=\"clear: both;\"></div>\n",
        "\n",
        "<b>Share to:</b>\n",
        "\n",
        "<a href=\"https://www.linkedin.com/sharing/share-offsite/?url=https%3A//github.com/GoogleCloudPlatform/generative-ai/blob/main/gemini/evaluation/multi_agent_evals_with_arize_and_crewai.ipynb\" target=\"_blank\">\n",
        "  <img width=\"20px\" src=\"https://upload.wikimedia.org/wikipedia/commons/8/81/LinkedIn_icon.svg\" alt=\"LinkedIn logo\">\n",
        "</a>\n",
        "\n",
        "<a href=\"https://bsky.app/intent/compose?text=https%3A//github.com/GoogleCloudPlatform/generative-ai/blob/main/gemini/evaluation/multi_agent_evals_with_arize_and_crewai.ipynb\" target=\"_blank\">\n",
        "  <img width=\"20px\" src=\"https://upload.wikimedia.org/wikipedia/commons/7/7a/Bluesky_Logo.svg\" alt=\"Bluesky logo\">\n",
        "</a>\n",
        "\n",
        "<a href=\"https://twitter.com/intent/tweet?url=https%3A//github.com/GoogleCloudPlatform/generative-ai/blob/main/gemini/evaluation/multi_agent_evals_with_arize_and_crewai.ipynb\" target=\"_blank\">\n",
        "  <img width=\"20px\" src=\"https://upload.wikimedia.org/wikipedia/commons/5/5a/X_icon_2.svg\" alt=\"X logo\">\n",
        "</a>\n",
        "\n",
        "<a href=\"https://reddit.com/submit?url=https%3A//github.com/GoogleCloudPlatform/generative-ai/blob/main/gemini/evaluation/multi_agent_evals_with_arize_and_crewai.ipynb\" target=\"_blank\">\n",
        "  <img width=\"20px\" src=\"https://redditinc.com/hubfs/Reddit%20Inc/Brand/Reddit_Logo.png\" alt=\"Reddit logo\">\n",
        "</a>\n",
        "\n",
        "<a href=\"https://www.facebook.com/sharer/sharer.php?u=https%3A//github.com/GoogleCloudPlatform/generative-ai/blob/main/gemini/evaluation/multi_agent_evals_with_arize_and_crewai.ipynb\" target=\"_blank\">\n",
        "  <img width=\"20px\" src=\"https://upload.wikimedia.org/wikipedia/commons/5/51/Facebook_f_logo_%282019%29.svg\" alt=\"Facebook logo\">\n",
        "</a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fa15a6e36d28"
      },
      "source": [
        "| Author(s) | [John Gilhuly](https://github.com/jgilhuly),  [Ivan Nardini](https://github.com/inardini), [Naveksha Sood](https://github.com/navekshasood) |\n",
        "|-----------|----------------------------------------------|"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1561a9f3aa99"
      },
      "source": [
        "# Overview\n",
        "\n",
        "This notebook demonstrates how to evaluate multi-agent systems using [Arize Phoenix](https://phoenix.arize.com), Google Evals, and [CrewAI](https://www.crewai.com/). It shows how to:\n",
        "\n",
        "1. Set up a multi-agent system using CrewAI for collaborative AI agents\n",
        "2. Instrument the agents with Phoenix for tracing and monitoring\n",
        "3. Evaluate agent performance and interactions using Google Gen AI\n",
        "4. Analyze the results using Arize's observability platform\n",
        "\n",
        "The notebook uses a practical example of agents working together to solve a task, with detailed tracing of their interactions and performance metrics. This enables:\n",
        "\n",
        "- Monitoring agent behavior and decision-making\n",
        "- Identifying bottlenecks and inefficiencies  \n",
        "- Measuring system performance and reliability\n",
        "- Gaining insights into agent collaboration patterns\n",
        " \n",
        "## Key Technologies\n",
        " \n",
        "- **CrewAI**: For orchestrating multi-agent systems\n",
        "- **Arize Phoenix**: For observability and tracing\n",
        "- **Google Cloud Vertex AI**: For model hosting and execution\n",
        "- **OpenAI**: For agent LLM capabilities"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "90954c9a0213"
      },
      "source": [
        "# Install dependencies and set API keys\n",
        "\n",
        "Create an account on [Arize Phoenix](https://phoenix.arize.com) to access a free hosted instance. Alternatively, you can [self-host](https://docs.arize.com/phoenix/self-hosting) the platform."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "4f79f0c45c4a"
      },
      "outputs": [],
      "source": [
        "%pip install -q arize-phoenix crewai crewai_tools openinference-instrumentation-crewai"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "id": "be48bc9db049"
      },
      "outputs": [],
      "source": [
        "import getpass\n",
        "import os\n",
        "\n",
        "# Prompt the user for their API keys if they haven't been set\n",
        "serper_key = os.getenv(\"SERPER_API_KEY\", \"SERPER_API_KEY\")\n",
        "phoenix_api_key = os.getenv(\"PHOENIX_API_KEY\", \"PHOENIX_API_KEY\")\n",
        "gemini_api_key = os.getenv(\"GEMINI_API_KEY\", \"GEMINI_API_KEY\")\n",
        "\n",
        "if serper_key == \"SERPER_API_KEY\":\n",
        "    serper_key = getpass.getpass(\"Please enter your SERPER_API_KEY: \")\n",
        "\n",
        "if phoenix_api_key == \"PHOENIX_API_KEY\":\n",
        "    phoenix_api_key = getpass.getpass(\"Please enter your PHOENIX_API_KEY: \")\n",
        "\n",
        "if gemini_api_key == \"GEMINI_API_KEY\":\n",
        "    gemini_api_key = getpass.getpass(\"Please enter your GEMINI_API_KEY: \")\n",
        "\n",
        "# Set the environment variables with the provided keys\n",
        "os.environ[\"SERPER_API_KEY\"] = serper_key\n",
        "os.environ[\"GEMINI_API_KEY\"] = gemini_api_key\n",
        "os.environ[\"PHOENIX_API_KEY\"] = phoenix_api_key\n",
        "os.environ[\"PHOENIX_COLLECTOR_ENDPOINT\"] = \"https://app.phoenix.arize.com/\"\n",
        "os.environ[\"PHOENIX_CLIENT_HEADERS\"] = f\"api_key={phoenix_api_key}\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "248a6f1d8c42"
      },
      "outputs": [],
      "source": [
        "!gcloud auth login"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1207d8a80e34"
      },
      "source": [
        "# Connect to Phoenix for Tracing\n",
        "\n",
        "The auto_instrument parameter below will automatically call any `openinference-instrumentation-xxx` packages you have installed to capture traces from the corresponding packages. In this example, the call below will automatically capture any calls to CrewAI."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "b5b394a242ae"
      },
      "outputs": [],
      "source": [
        "from phoenix.otel import register\n",
        "\n",
        "tracer_provider = register(auto_instrument=True)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "b1161ddf211a"
      },
      "source": [
        "# Define your CrewAI Crew of Agents\n",
        "\n",
        "This crew consists of three specialized agents working together to analyze and report on a given topic:\n",
        "1. A Senior Research Analyst who uncovers and identifies emerging trends\n",
        "2. A Specialist Fact Checker who verifies claims and assesses implications\n",
        "3. A Writer who transforms the verified information into compelling content\n",
        "Together, these agents form a collaborative team that researches, validates, and communicates insights on the specified topic.\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "id": "ddfd2ddf6a14"
      },
      "outputs": [],
      "source": [
        "from crewai import Agent, Crew, Process, Task\n",
        "from crewai_tools import SerperDevTool\n",
        "\n",
        "search_tool = SerperDevTool()\n",
        "llm = \"gemini-2.0-flash\"\n",
        "\n",
        "\n",
        "def create_research_crew(topic=\"AI and data science advancements in 2024\"):\n",
        "    \"\"\"\n",
        "    Creates a crew of agents to analyze a given topic.\n",
        "\n",
        "    Args:\n",
        "        topic (str): The topic to analyze. Defaults to \"AI and data science advancements in 2024\".\n",
        "\n",
        "    Returns:\n",
        "        Crew: A crew instance configured to analyze the given topic.\n",
        "    \"\"\"\n",
        "    # Define your agents with roles and goals\n",
        "    researcher = Agent(\n",
        "        role=\"Senior Research Analyst\",\n",
        "        goal=f\"Uncover cutting-edge developments in {topic}\",\n",
        "        backstory=\"\"\"You work at a leading tech think tank.\n",
        "      Your expertise lies in identifying emerging trends.\n",
        "      You have a knack for dissecting complex data and presenting actionable insights.\"\"\",\n",
        "        verbose=True,\n",
        "        allow_delegation=False,\n",
        "        tools=[search_tool],\n",
        "        llm=llm,\n",
        "    )\n",
        "\n",
        "    fact_checker = Agent(\n",
        "        role=\"Specialist Fact Checker\",\n",
        "        goal=f\"Verify claims and assess implications of {topic}\",\n",
        "        backstory=\"\"\"You are a respected authority with a background in both technical and ethical aspects.\n",
        "      You scrutinize developments for potential societal impacts and ethical concerns.\n",
        "      Your analysis helps ensure balanced reporting on technological advancements.\"\"\",\n",
        "        verbose=True,\n",
        "        allow_delegation=True,\n",
        "        tools=[search_tool],\n",
        "        llm=llm,\n",
        "    )\n",
        "\n",
        "    writer = Agent(\n",
        "        role=\"Writer\",\n",
        "        goal=f\"Craft compelling content on {topic}\",\n",
        "        backstory=\"\"\"You are a renowned Writer, known for your insightful and engaging articles.\n",
        "      You transform complex concepts into compelling narratives while maintaining factual accuracy.\n",
        "      You can only work with confirmed facts, not speculations.\n",
        "      You require that your input is confirmed by the fact checker.\"\"\",\n",
        "        verbose=True,\n",
        "        allow_delegation=True,\n",
        "        llm=llm,\n",
        "    )\n",
        "\n",
        "    # Create tasks for your agents with explicit context\n",
        "    conduct_analysis_task = Task(\n",
        "        description=f\"\"\"Conduct a comprehensive analysis of the latest developments in {topic}.\n",
        "      Identify key trends, breakthrough technologies, and potential industry impacts.\n",
        "      Focus on both research breakthroughs and commercial applications.\"\"\",\n",
        "        expected_output=\"Full analysis report in bullet points with citations to sources\",\n",
        "        agent=researcher,\n",
        "        context=[],  # Explicitly set empty context\n",
        "    )\n",
        "\n",
        "    fact_checking_task = Task(\n",
        "        description=f\"\"\"Review the research findings and verify the accuracy of claims about {topic}.\n",
        "      Identify any potential ethical concerns or societal implications.\n",
        "      Highlight areas where hype may exceed reality and provide a balanced assessment.\n",
        "      Suggest frameworks that should be considered for each major advancement.\"\"\",\n",
        "        expected_output=\"Fact-checking report with verification status for each major claim\",\n",
        "        agent=fact_checker,\n",
        "        context=[conduct_analysis_task],  # Set context to previous task\n",
        "    )\n",
        "\n",
        "    writer_task = Task(\n",
        "        description=f\"\"\"Using the insights provided from research, fact-checking, and market assessment,\n",
        "      develop a comprehensive and engaging blog post that presents a holistic view of {topic}.\n",
        "      Your post should be informative yet accessible, catering to a knowledgeable audience.\n",
        "      Include balanced perspectives on both the potential and limitations.\n",
        "      Make it sound cool, avoid complex words so it doesn't sound like AI.\n",
        "      Incorporate ethical considerations and market projections to provide readers with a complete picture.\"\"\",\n",
        "        expected_output=\"Full blog post of at least 6 paragraphs with sections covering innovations, considerations, and market outlook\",\n",
        "        agent=writer,\n",
        "        context=[fact_checking_task],  # Set context to previous task\n",
        "    )\n",
        "\n",
        "    # Instantiate your crew with a sequential process\n",
        "    crew = Crew(\n",
        "        agents=[researcher, fact_checker, writer],\n",
        "        tasks=[conduct_analysis_task, fact_checking_task, writer_task],\n",
        "        verbose=False,\n",
        "        process=Process.sequential,\n",
        "    )\n",
        "\n",
        "    return crew"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "be935d71a45e"
      },
      "source": [
        "# Evaluating your Crew of Agents\n",
        "\n",
        "Now you'll built an experiment set to test your CrewAI crew with Phoenix and Google Gen AI evals.\n",
        "\n",
        "Experiments are made up of three pieces:\n",
        "- A **Dataset** to house your inputs and expected outputs, and to centralize your tests.\n",
        "- A **Task** to perform on each row of the dataset. Usually this is an invokation of your agent or crew of agents.\n",
        "- A set of **Evaluators** to measure the outputs of your task on each row. These can be of your own creation, or pulled from a framework like Google Gen AI evals.\n",
        "\n",
        "When run, an Experiment will send each row of your dataset through your task, then apply each of your evaluators to the result. All traces and metrics will then be stored in Phoenix for reference and comparison.\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8d92f4328122"
      },
      "source": [
        "## Define Dataset of Test Cases\n",
        "\n",
        "First define a set of test inputs for your crew and their expected agent trajectories."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "92db3497e8f7"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import phoenix as px\n",
        "\n",
        "reference_trajectory = {\n",
        "    # example 1 - AI and data science advancements\n",
        "    \"AI and data science advancements in 2024\": [\n",
        "        {\n",
        "            \"tool_name\": \"Senior Research Analyst\",\n",
        "            \"tool_input\": \"Conduct comprehensive research on AI and data science advancements in 2024, analyzing key innovations, trends, and potential impacts.\",\n",
        "        },\n",
        "        {\n",
        "            \"tool_name\": \"Specialist Fact Checker\",\n",
        "            \"tool_input\": \"Verify the accuracy of research findings on AI and data science advancements in 2024, ensuring all claims are supported by credible sources.\",\n",
        "        },\n",
        "        {\n",
        "            \"tool_name\": \"Writer\",\n",
        "            \"tool_input\": \"Develop a comprehensive blog post about AI and data science advancements in 2024, incorporating verified research and balanced perspectives.\",\n",
        "        },\n",
        "    ],\n",
        "    # example 2 - Quantum computing breakthroughs\n",
        "    \"Quantum computing breakthroughs\": [\n",
        "        {\n",
        "            \"tool_name\": \"Senior Research Analyst\",\n",
        "            \"tool_input\": \"Conduct comprehensive research on quantum computing breakthroughs, analyzing key innovations, trends, and potential impacts.\",\n",
        "        },\n",
        "        {\n",
        "            \"tool_name\": \"Specialist Fact Checker\",\n",
        "            \"tool_input\": \"Verify the accuracy of research findings on quantum computing breakthroughs, ensuring all claims are supported by credible sources.\",\n",
        "        },\n",
        "        {\n",
        "            \"tool_name\": \"Writer\",\n",
        "            \"tool_input\": \"Develop a comprehensive blog post about quantum computing breakthroughs, incorporating verified research and balanced perspectives.\",\n",
        "        },\n",
        "    ],\n",
        "    # example 3 - Climate tech innovations\n",
        "    \"Climate tech innovations\": [\n",
        "        {\n",
        "            \"tool_name\": \"Senior Research Analyst\",\n",
        "            \"tool_input\": \"Conduct comprehensive research on climate tech innovations, analyzing key developments, trends, and potential impacts.\",\n",
        "        },\n",
        "        {\n",
        "            \"tool_name\": \"Specialist Fact Checker\",\n",
        "            \"tool_input\": \"Verify the accuracy of research findings on climate tech innovations, ensuring all claims are supported by credible sources.\",\n",
        "        },\n",
        "        {\n",
        "            \"tool_name\": \"Writer\",\n",
        "            \"tool_input\": \"Develop a comprehensive blog post about climate tech innovations, incorporating verified research and balanced perspectives.\",\n",
        "        },\n",
        "    ],\n",
        "    # example 4 - Biotechnology and gene editing\n",
        "    \"Biotechnology and gene editing progress\": [\n",
        "        {\n",
        "            \"tool_name\": \"Senior Research Analyst\",\n",
        "            \"tool_input\": \"Conduct comprehensive research on biotechnology and gene editing progress, analyzing key innovations, trends, and potential impacts.\",\n",
        "        },\n",
        "        {\n",
        "            \"tool_name\": \"Specialist Fact Checker\",\n",
        "            \"tool_input\": \"Verify the accuracy of research findings on biotechnology and gene editing progress, ensuring all claims are supported by credible sources.\",\n",
        "        },\n",
        "        {\n",
        "            \"tool_name\": \"Writer\",\n",
        "            \"tool_input\": \"Develop a comprehensive blog post about biotechnology and gene editing progress, incorporating verified research and balanced perspectives.\",\n",
        "        },\n",
        "    ],\n",
        "    # example 5 - Renewable energy solutions\n",
        "    \"Renewable energy solutions\": [\n",
        "        {\n",
        "            \"tool_name\": \"Senior Research Analyst\",\n",
        "            \"tool_input\": \"Conduct comprehensive research on renewable energy solutions, analyzing key innovations, trends, and potential impacts.\",\n",
        "        },\n",
        "        {\n",
        "            \"tool_name\": \"Specialist Fact Checker\",\n",
        "            \"tool_input\": \"Verify the accuracy of research findings on renewable energy solutions, ensuring all claims are supported by credible sources.\",\n",
        "        },\n",
        "        {\n",
        "            \"tool_name\": \"Writer\",\n",
        "            \"tool_input\": \"Develop a comprehensive blog post about renewable energy solutions, incorporating verified research and balanced perspectives.\",\n",
        "        },\n",
        "    ],\n",
        "}\n",
        "\n",
        "df = pd.DataFrame(\n",
        "    [\n",
        "        {\n",
        "            \"topic\": \"AI and data science advancements in 2024\",\n",
        "            \"reference_trajectory\": reference_trajectory[\n",
        "                \"AI and data science advancements in 2024\"\n",
        "            ],\n",
        "        },\n",
        "        {\n",
        "            \"topic\": \"Quantum computing breakthroughs\",\n",
        "            \"reference_trajectory\": reference_trajectory[\n",
        "                \"Quantum computing breakthroughs\"\n",
        "            ],\n",
        "        },\n",
        "        {\n",
        "            \"topic\": \"Climate tech innovations\",\n",
        "            \"reference_trajectory\": reference_trajectory[\"Climate tech innovations\"],\n",
        "        },\n",
        "        {\n",
        "            \"topic\": \"Biotechnology and gene editing progress\",\n",
        "            \"reference_trajectory\": reference_trajectory[\n",
        "                \"Biotechnology and gene editing progress\"\n",
        "            ],\n",
        "        },\n",
        "        {\n",
        "            \"topic\": \"Renewable energy solutions\",\n",
        "            \"reference_trajectory\": reference_trajectory[\"Renewable energy solutions\"],\n",
        "        },\n",
        "    ]\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6b9fa92d3908"
      },
      "source": [
        "Create a dataset in Phoenix to track your various experiments on that dataset."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0538af77f7a8"
      },
      "outputs": [],
      "source": [
        "phoenix_client = px.Client()\n",
        "try:\n",
        "    dataset = phoenix_client.get_dataset(name=\"crewai-researcher-test-topics\")\n",
        "except ValueError:\n",
        "    dataset = phoenix_client.upload_dataset(\n",
        "        dataframe=df,\n",
        "        dataset_name=\"crewai-researcher-test-topics\",\n",
        "        input_keys=[\"topic\"],\n",
        "        output_keys=[\"reference_trajectory\"],\n",
        "    )"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3dd5e7b83235"
      },
      "source": [
        "## Define your Experiment Task\n",
        "\n",
        "This method will be run on each row of your test cases dataset:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "id": "0fd72d4bf1e2"
      },
      "outputs": [],
      "source": [
        "def call_crew_with_topic(input):\n",
        "    crew = create_research_crew(topic=input.get(\"topic\"))\n",
        "    result = crew.kickoff()\n",
        "    return result"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "90f854fe126e"
      },
      "source": [
        "## Define your Evaluators\n",
        "\n",
        "Define as many evaluators as you'd need to evaluate your agent. In this case, you'll use Google Gen AI's eval library to evaluate the crew's trajectory."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "f1aa145beee1"
      },
      "outputs": [],
      "source": [
        "from vertexai.preview.evaluation import EvalTask\n",
        "\n",
        "\n",
        "def create_trajectory_from_response(response):\n",
        "    tasks = response.get(\"tasks_output\")\n",
        "    actual_trajectory = []\n",
        "    if tasks is not None:  # Check if tasks is not None and iterable\n",
        "        for task in tasks:\n",
        "            if isinstance(task, dict):\n",
        "                agent_name = task.get(\"agent\", \"Unknown Agent\")\n",
        "                description = task.get(\"description\", \"No description\")\n",
        "                new_entry = {\"tool_name\": agent_name, \"tool_input\": description}\n",
        "                actual_trajectory.append(new_entry)\n",
        "    return actual_trajectory\n",
        "\n",
        "\n",
        "def eval_trajectory_with_google_gen_ai(\n",
        "    output, expected, metric_name=\"trajectory_exact_match\"\n",
        ") -> float:\n",
        "    eval_dataset = pd.DataFrame(\n",
        "        {\n",
        "            \"predicted_trajectory\": [create_trajectory_from_response(output)],\n",
        "            \"reference_trajectory\": [expected.get(\"reference_trajectory\")],\n",
        "        }\n",
        "    )\n",
        "    eval_task = EvalTask(\n",
        "        dataset=eval_dataset,\n",
        "        metrics=[metric_name],\n",
        "    )\n",
        "    eval_result = eval_task.evaluate()\n",
        "    metric_value = eval_result.summary_metrics.get(f\"{metric_name}/mean\")\n",
        "    if metric_value is None:\n",
        "        return 0.0\n",
        "    return metric_value\n",
        "\n",
        "\n",
        "def trajectory_exact_match(output, expected):\n",
        "    return eval_trajectory_with_google_gen_ai(\n",
        "        output, expected, metric_name=\"trajectory_exact_match\"\n",
        "    )\n",
        "\n",
        "\n",
        "def trajectory_precision(output, expected):\n",
        "    return eval_trajectory_with_google_gen_ai(\n",
        "        output, expected, metric_name=\"trajectory_precision\"\n",
        "    )\n",
        "\n",
        "\n",
        "def trajectory_in_order_match(output, expected):\n",
        "    return eval_trajectory_with_google_gen_ai(\n",
        "        output, expected, metric_name=\"trajectory_in_order_match\"\n",
        "    )\n",
        "\n",
        "\n",
        "def trajectory_any_order_match(output, expected):\n",
        "    return eval_trajectory_with_google_gen_ai(\n",
        "        output, expected, metric_name=\"trajectory_any_order_match\"\n",
        "    )\n",
        "\n",
        "\n",
        "# This final evaluator is a custom code based check:\n",
        "def agent_names_match(output, expected):\n",
        "    predicted_trajectory = create_trajectory_from_response(output)\n",
        "    reference_trajectory = expected.get(\"reference_trajectory\")\n",
        "\n",
        "    # Check if the predicted agents appear in the same order as in the reference\n",
        "    if reference_trajectory is None or len(predicted_trajectory) != len(\n",
        "        reference_trajectory\n",
        "    ):\n",
        "        return 0\n",
        "\n",
        "    for i, predicted_task in enumerate(predicted_trajectory):\n",
        "        if predicted_task[\"tool_name\"] != reference_trajectory[i][\"tool_name\"]:\n",
        "            return 0\n",
        "    return 1"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "b40a92818dae"
      },
      "source": [
        "## Run your Experiment and Visualize Results in Phoenix"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "21f513245d44"
      },
      "outputs": [],
      "source": [
        "import nest_asyncio\n",
        "from phoenix.experiments import run_experiment\n",
        "\n",
        "nest_asyncio.apply()\n",
        "\n",
        "experiment = run_experiment(\n",
        "    dataset,\n",
        "    call_crew_with_topic,\n",
        "    experiment_name=\"agent-experiment\",\n",
        "    evaluators=[\n",
        "        trajectory_exact_match,\n",
        "        trajectory_precision,\n",
        "        trajectory_in_order_match,\n",
        "        trajectory_any_order_match,\n",
        "        agent_names_match,\n",
        "    ],\n",
        ")"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "name": "multi_agent_evals_with_arize_and_crewai.ipynb",
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}
